dynamic-skill-compiler 0.1.0__py3-none-any.whl

dynamic_skill_compiler/__init__.py

@@ -0,0 +1,66 @@
+from dynamic_skill_compiler.models import (
+    CompilationMetrics,
+    CompiledSkill,
+    CompiledSkillPackage,
+    CompilerPassTrace,
+    LocalEnvironment,
+    QueryPlan,
+    SkillAsset,
+    SkillFragment,
+    SkillGraph,
+    SkillRelation,
+    Subgoal,
+)
+from dynamic_skill_compiler.decompose import TaskDecomposer
+from dynamic_skill_compiler.fragments import FragmentMatcher, SkillFragmentExtractor
+from dynamic_skill_compiler.graph import (
+    DEFAULT_GRAPH_PASSES,
+    GRAPH_PASS_PRESETS,
+    LEGACY_DEFAULT_GRAPH_PASSES,
+    SLIM_GRAPH_PASSES,
+    SUPPORTED_GRAPH_PASSES,
+)
+from dynamic_skill_compiler.grounding import EnvironmentGrounder
+from dynamic_skill_compiler.pipeline import CompilerConfig, DynamicSkillCompiler
+from dynamic_skill_compiler.semantic import SemanticSoftMatcher
+from dynamic_skill_compiler.query import QueryOptimizer
+from dynamic_skill_compiler.retriever import (
+    CompositeSkillRetriever,
+    InMemorySkillRetriever,
+    LocalSkillLibraryRetriever,
+    SkillRetriever,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "CompilationMetrics",
+    "CompiledSkill",
+    "CompiledSkillPackage",
+    "CompilerPassTrace",
+    "CompilerConfig",
+    "CompositeSkillRetriever",
+    "DEFAULT_GRAPH_PASSES",
+    "DynamicSkillCompiler",
+    "EnvironmentGrounder",
+    "FragmentMatcher",
+    "GRAPH_PASS_PRESETS",
+    "InMemorySkillRetriever",
+    "LEGACY_DEFAULT_GRAPH_PASSES",
+    "LocalSkillLibraryRetriever",
+    "LocalEnvironment",
+    "QueryOptimizer",
+    "QueryPlan",
+    "SkillAsset",
+    "SkillFragment",
+    "SkillFragmentExtractor",
+    "SkillGraph",
+    "SkillRelation",
+    "SkillRetriever",
+    "SLIM_GRAPH_PASSES",
+    "Subgoal",
+    "SemanticSoftMatcher",
+    "SUPPORTED_GRAPH_PASSES",
+    "TaskDecomposer",
+    "__version__",
+]

dynamic_skill_compiler/cli.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from dataclasses import asdict, is_dataclass
+from typing import Any
+
+from dynamic_skill_compiler.models import LocalEnvironment
+from dynamic_skill_compiler.pipeline import CompilerConfig, DynamicSkillCompiler
+from dynamic_skill_compiler.retriever import LocalSkillLibraryRetriever
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="dsc",
+        description="Compile a local skill library into a task-specific DSC package.",
+    )
+    parser.add_argument(
+        "query",
+        nargs="?",
+        help="Task query to compile. If omitted, the query is read from stdin.",
+    )
+    parser.add_argument(
+        "--skills-dir",
+        required=True,
+        help="Directory containing local skill folders with SKILL.md files.",
+    )
+    parser.add_argument(
+        "--benchmark",
+        default="generic",
+        help="Optional environment/benchmark label used by adaptive compiler profiles.",
+    )
+    parser.add_argument(
+        "--cwd",
+        default=".",
+        help="Execution working directory used for localization.",
+    )
+    parser.add_argument(
+        "--workspace-root",
+        default=".",
+        help="Workspace root used for localization.",
+    )
+    parser.add_argument(
+        "--python-bin",
+        default=sys.executable or "python",
+        help="Python executable name/path used for localization.",
+    )
+    parser.add_argument(
+        "--min-relevance",
+        type=float,
+        default=CompilerConfig.min_relevance,
+        help="Minimum utility score required for skill selection.",
+    )
+    parser.add_argument(
+        "--preserve-top-k",
+        type=int,
+        default=CompilerConfig.preserve_top_k,
+        help="Always preserve at least this many top-scored skills.",
+    )
+    parser.add_argument(
+        "--max-selected-skills",
+        type=int,
+        default=CompilerConfig.max_selected_skills,
+        help="Hard cap for selected skills. Use 0 for no explicit cap.",
+    )
+    parser.add_argument(
+        "--pretty",
+        action="store_true",
+        help="Pretty-print JSON output.",
+    )
+    parser.add_argument(
+        "--include-instructions",
+        action="store_true",
+        help="Include localized selected instructions in the JSON output.",
+    )
+    return parser
+
+
+def compile_from_args(args: argparse.Namespace) -> dict[str, Any]:
+    query = args.query if args.query is not None else sys.stdin.read().strip()
+    if not query:
+        raise SystemExit("A task query is required, either as an argument or on stdin.")
+
+    compiler = DynamicSkillCompiler(
+        retriever=LocalSkillLibraryRetriever(args.skills_dir),
+        config=CompilerConfig(
+            min_relevance=args.min_relevance,
+            preserve_top_k=args.preserve_top_k,
+            max_selected_skills=args.max_selected_skills,
+        ),
+    )
+    compiled = compiler.compile(
+        query,
+        environment=LocalEnvironment(
+            cwd=args.cwd,
+            workspace_root=args.workspace_root,
+            python_bin=args.python_bin,
+            benchmark=args.benchmark,
+        ),
+    )
+    summary = compiler.summarize(compiled)
+    summary["metrics"] = _to_jsonable(compiled.metrics)
+    if args.include_instructions:
+        summary["compiled_skills"] = [
+            {
+                "name": item.asset.name,
+                "skill_id": item.asset.skill_id,
+                "selected_reason": item.selected_reason,
+                "utility_score": item.utility_score,
+                "localized_instructions": item.localized_instructions,
+                "selected_fragments": [_to_jsonable(fragment) for fragment in item.selected_fragments],
+            }
+            for item in compiled.compiled_skills
+        ]
+    return summary
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    summary = compile_from_args(args)
+    indent = 2 if args.pretty else None
+    print(json.dumps(summary, ensure_ascii=False, indent=indent))
+    return 0
+
+
+def _to_jsonable(value: Any) -> Any:
+    if is_dataclass(value):
+        return _to_jsonable(asdict(value))
+    if isinstance(value, dict):
+        return {str(key): _to_jsonable(item) for key, item in value.items()}
+    if isinstance(value, (list, tuple)):
+        return [_to_jsonable(item) for item in value]
+    if isinstance(value, set):
+        return sorted(_to_jsonable(item) for item in value)
+    return value
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

dynamic_skill_compiler/decompose.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import List, Set
+
+from dynamic_skill_compiler.models import QueryPlan, Subgoal
+from dynamic_skill_compiler.query import QueryOptimizer
+
+
+CONNECTORS = {"and", "then", "after", "before", "finally", "otherwise", "else"}
+LEADING_PHASE_MARKERS = {
+    "first",
+    "next",
+    "then",
+    "after",
+    "before",
+    "finally",
+    "if",
+    "when",
+    "while",
+    "until",
+}
+
+@dataclass
+class TaskDecomposer:
+    optimizer: QueryOptimizer = field(default_factory=QueryOptimizer)
+
+    def decompose(self, query_plan: QueryPlan) -> List[Subgoal]:
+        raw = self.optimizer.normalize_text(
+            query_plan.raw_query or query_plan.normalized_query,
+            preserve_delimiters=True,
+        )
+        clauses = self._split_into_clauses(raw)
+        if not clauses:
+            clauses = [query_plan.normalized_query]
+
+        subgoals: List[Subgoal] = []
+        previous_id = None
+        for index, clause in enumerate(clauses):
+            tokens = self.optimizer.extract_content_terms(clause)
+            token_set = set(tokens)
+            required = {
+                token
+                for token in tokens
+                if token in query_plan.required_capabilities or token in query_plan.optional_capabilities
+            }
+            if not required:
+                required = set(tokens[:3]) if tokens else set(clause.split()[:3])
+            optional = set(tokens[3:]) if len(tokens) > 3 else set()
+            required |= self._augment_required_capabilities(clause, token_set)
+            optional |= self._augment_optional_capabilities(clause, token_set)
+            subgoal_id = f"sg_{index + 1}"
+            hints = self._infer_environment_hints(clause, required | optional)
+            subgoals.append(
+                Subgoal(
+                    subgoal_id=subgoal_id,
+                    description=clause,
+                    required_capabilities=required,
+                    optional_capabilities=optional,
+                    depends_on=[previous_id] if previous_id else [],
+                    priority=index,
+                    environment_hints=hints,
+                )
+            )
+            previous_id = subgoal_id
+        return subgoals
+
+    def _split_into_clauses(self, text: str) -> List[str]:
+        normalized = re.sub(r"\s+", " ", text).strip()
+        if not normalized:
+            return []
+
+        for marker in ("first", "next", "then", "finally", "otherwise", "else", "if", "when", "while", "until"):
+            normalized = re.sub(rf"\b{marker}\b", f". {marker}", normalized)
+
+        sentence_candidates = []
+        for segment in re.split(r"[.;!?]", normalized):
+            segment = segment.strip()
+            if segment:
+                sentence_candidates.append(segment)
+
+        clauses: List[str] = []
+        for sentence in sentence_candidates or [normalized]:
+            tokens = sentence.split()
+            current: List[str] = []
+            for token in tokens:
+                lowered = token.lower().strip(",")
+                if lowered in CONNECTORS or lowered in LEADING_PHASE_MARKERS:
+                    if current:
+                        clauses.append(" ".join(current).strip())
+                        current = []
+                    # Keep conditional markers so the clause still carries its branch meaning.
+                    if lowered in {"if", "when", "while", "until"}:
+                        current.append(lowered)
+                    continue
+                current.append(token)
+            if current:
+                clauses.append(" ".join(current).strip())
+        return [clause for clause in clauses if clause]
+
+    def _infer_environment_hints(self, clause: str, capabilities: Set[str]) -> dict:
+        lowered = clause.lower()
+        caps = {cap.lower() for cap in capabilities}
+
+        scienceworld_terms = {
+            "conductive",
+            "conductivity",
+            "battery",
+            "wire",
+            "thermometer",
+            "workshop",
+            "focus",
+            "box",
+            "boil",
+            "melt",
+            "freeze",
+            "combust",
+            "temperature",
+            "substance",
+        }
+        strong_scienceworld_terms = {
+            "conductive",
+            "conductivity",
+            "battery",
+            "wire",
+            "thermometer",
+            "workshop",
+            "boil",
+            "melt",
+            "freeze",
+            "combust",
+        }
+        scienceworld_phrases = (
+            "state of matter",
+            "degrees celsius",
+            "electrically conductive",
+        )
+        alfworld_terms = {
+            "stoveburner",
+            "microwave",
+            "fridge",
+            "sinkbasin",
+            "cabinet",
+            "drawer",
+            "countertop",
+            "cool",
+            "heat",
+            "clean",
+            "put",
+            "move",
+            "take",
+        }
+        webshop_terms = {
+            "search",
+            "buy",
+            "price",
+            "size",
+            "color",
+            "click",
+            "product",
+            "results",
+            "listing",
+        }
+
+        token_text = " ".join(sorted(caps)) + " " + lowered
+
+        if any(term in token_text for term in webshop_terms):
+            return {"domain": "webshop"}
+        if any(phrase in lowered for phrase in scienceworld_phrases):
+            return {"domain": "scienceworld"}
+        if any(term in lowered for term in {"stoveburner", "microwave", "fridge", "sinkbasin"}):
+            return {"domain": "alfworld"}
+        if (
+            caps & {"cool", "heat", "clean", "wash", "slice"}
+            and not any(term in token_text for term in strong_scienceworld_terms)
+        ):
+            return {"domain": "alfworld"}
+        if any(term in token_text for term in scienceworld_terms):
+            return {"domain": "scienceworld"}
+        if any(term in token_text for term in alfworld_terms):
+            return {"domain": "alfworld"}
+        return {"domain": "generic"} if capabilities else {}
+
+    def _augment_required_capabilities(self, clause: str, tokens: Set[str]) -> Set[str]:
+        augmented: Set[str] = set()
+        if "focus" in tokens:
+            augmented.add("focus")
+        if self._is_phase_change_clause(clause, tokens):
+            augmented |= {"heat", "contain", "apparatu"}
+        if tokens & {"measure", "temperature"}:
+            augmented |= {"measure", "monit"}
+        return augmented
+
+    def _augment_optional_capabilities(self, clause: str, tokens: Set[str]) -> Set[str]:
+        augmented = set(self.optimizer.infer_structural_capabilities(tokens))
+        if self._is_phase_change_clause(clause, tokens):
+            augmented |= {"monit", "wait", "state", "chang", "substance"}
+        if "combust" in tokens:
+            augmented |= {"heat", "fallback"}
+        return augmented
+
+    def _is_phase_change_clause(self, clause: str, tokens: Set[str]) -> bool:
+        if tokens & {"boil", "melt", "freeze", "heat", "cool", "combust"}:
+            return True
+        lowered = clause.lower()
+        return "state of matter" in lowered or ("change" in lowered and "state" in lowered)

dynamic_skill_compiler/fragments.py

@@ -0,0 +1,205 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Dict, Iterable, List
+
+from dynamic_skill_compiler.models import SkillAsset, SkillFragment, Subgoal
+
+
+LINE_SPLIT = re.compile(r"[.\n]")
+ACTION_QUOTED = re.compile(r"`([^`]+)`")
+ACTION_PREFIXES = (
+    "teleport to ",
+    "pick up ",
+    "go to ",
+    "take ",
+    "move ",
+    "open ",
+    "close ",
+    "use ",
+    "look at ",
+    "examine ",
+    "turn on ",
+    "turn off ",
+    "deactivate ",
+    "wait1",
+    "wait",
+    "clean ",
+    "heat ",
+    "cool ",
+    "focus on ",
+    "activate ",
+    "pour ",
+    "mix ",
+    "measure ",
+    "search ",
+    "click ",
+    "select ",
+    "buy now",
+    "add to cart",
+    "filter ",
+    "sort ",
+)
+LOW_SIGNAL_PREFIXES = (
+    "purpose",
+    "when to use",
+    "notes",
+    "key parameters",
+    "key considerations",
+    "integration with other skills",
+)
+
+
+@dataclass
+class SkillFragmentExtractor:
+    max_fragments_per_skill: int = 6
+
+    def extract(self, skills: Iterable[SkillAsset]) -> Dict[str, List[SkillFragment]]:
+        fragment_map: Dict[str, List[SkillFragment]] = {}
+        for skill in skills:
+            fragments = self._extract_for_skill(skill)
+            fragment_map[skill.skill_id] = fragments
+        return fragment_map
+
+    def _extract_for_skill(self, skill: SkillAsset) -> List[SkillFragment]:
+        indexed_lines: List[tuple[int, str]] = []
+        for instruction in skill.instructions:
+            for line in LINE_SPLIT.split(instruction):
+                cleaned = line.strip(" -*\t")
+                if cleaned:
+                    indexed_lines.append((len(indexed_lines), cleaned))
+        lines = [line for _, line in indexed_lines]
+        if not lines:
+            lines.append(skill.description)
+            indexed_lines = [(0, skill.description)]
+
+        selected_lines = self._select_fragment_lines(indexed_lines)
+
+        fragments: List[SkillFragment] = []
+        for index, line in enumerate(selected_lines):
+            capabilities = {
+                token.strip(".,:;()[]{}").lower()
+                for token in line.split()
+                if len(token.strip(".,:;()[]{}")) > 2
+            }
+            example_actions = self._extract_actions(line)
+            preconditions = self._extract_preconditions(line)
+            postconditions = self._extract_postconditions(line)
+            action_schema = self._extract_action_schema(line, example_actions)
+            fragments.append(
+                SkillFragment(
+                    fragment_id=f"{skill.skill_id}::frag_{index + 1}",
+                    skill_id=skill.skill_id,
+                    title=f"{skill.name} fragment {index + 1}",
+                    content=line,
+                    capabilities=capabilities or skill.normalized_capabilities(),
+                    action_schema=action_schema,
+                    preconditions=preconditions,
+                    postconditions=postconditions,
+                    example_actions=example_actions,
+                    token_cost=max(1.0, len(line.split()) / 20.0),
+                    metadata={"source_skill": skill.name},
+                )
+            )
+        return fragments
+
+    def _select_fragment_lines(self, indexed_lines: List[tuple[int, str]]) -> List[str]:
+        if len(indexed_lines) <= self.max_fragments_per_skill:
+            return [line for _, line in indexed_lines]
+
+        ranked = sorted(
+            indexed_lines,
+            key=lambda item: (
+                self._line_priority(item[1]),
+                -item[0],
+            ),
+            reverse=True,
+        )
+        chosen = sorted(
+            ranked[: self.max_fragments_per_skill],
+            key=lambda item: item[0],
+        )
+        return [line for _, line in chosen]
+
+    def _line_priority(self, line: str) -> int:
+        lowered = line.lower()
+        priority = 0
+        if self._extract_actions(line):
+            priority += 8
+        if self._extract_preconditions(line):
+            priority += 3
+        if self._extract_postconditions(line):
+            priority += 2
+        if any(marker in lowered for marker in ("command", "action pattern", "execute", "retry", "verify")):
+            priority += 2
+        if any(lowered.startswith(prefix) for prefix in LOW_SIGNAL_PREFIXES):
+            priority -= 2
+        return priority
+
+    def _extract_actions(self, line: str) -> List[str]:
+        matches = ACTION_QUOTED.findall(line)
+        if matches:
+            return matches
+        lowered = line.lower()
+        for marker in ("action:", "command:", "action pattern:", "example:"):
+            if marker in lowered:
+                candidate = line[lowered.find(marker) + len(marker):].strip()
+                if candidate:
+                    return [candidate]
+        if "action:" in lowered:
+            return [line.split(":", 1)[1].strip()]
+        for prefix in ACTION_PREFIXES:
+            idx = lowered.find(prefix)
+            if idx != -1:
+                return [line[idx:].strip()]
+        return []
+
+    def _extract_preconditions(self, line: str) -> List[str]:
+        lowered = line.lower()
+        if lowered.startswith(("if ", "ensure ", "verify ", "when ", "requires ")):
+            return [line]
+        if "before " in lowered or "must " in lowered:
+            return [line]
+        return []
+
+    def _extract_postconditions(self, line: str) -> List[str]:
+        lowered = line.lower()
+        if lowered.startswith(("output", "result", "return", "the agent receives", "the specified object is")):
+            return [line]
+        if "output" in lowered or "confirm" in lowered or "transferred" in lowered:
+            return [line]
+        return []
+
+    def _extract_action_schema(self, line: str, example_actions: List[str]) -> str | None:
+        if example_actions:
+            return example_actions[0]
+        lowered = line.lower()
+        for prefix in ACTION_PREFIXES:
+            if prefix in lowered:
+                start = lowered.find(prefix)
+                return line[start:].strip()
+        return None
+
+
+@dataclass
+class FragmentMatcher:
+    subgoal_weight: float = 0.75
+    optional_weight: float = 0.25
+
+    def match(self, subgoal: Subgoal, fragments: List[SkillFragment]) -> List[SkillFragment]:
+        ranked = sorted(
+            fragments,
+            key=lambda fragment: self.score(subgoal, fragment),
+            reverse=True,
+        )
+        return [fragment for fragment in ranked if self.score(subgoal, fragment) > 0]
+
+    def score(self, subgoal: Subgoal, fragment: SkillFragment) -> float:
+        required = subgoal.required_capabilities
+        optional = subgoal.optional_capabilities
+        frag_caps = fragment.capabilities
+        required_hit = len(required & frag_caps) / max(len(required), 1)
+        optional_hit = len(optional & frag_caps) / max(len(optional), 1) if optional else 0.0
+        action_bonus = 0.15 if fragment.example_actions else 0.0
+        return self.subgoal_weight * required_hit + self.optional_weight * optional_hit + action_bonus