opencomb 0.2.0__py3-none-any.whl

opencomb/combiner.py

@@ -0,0 +1,232 @@
+"""Core combining logic for code and configuration files."""
+
+from __future__ import annotations
+
+import ast
+import json
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+try:
+    import tomllib  # Python 3.11+
+except ImportError:
+    import tomli as tomllib  # type: ignore
+
+import tomli_w
+
+
+class CodeCombiner:
+    """Combine multiple Python source files / snippets into one coherent module."""
+
+    def __init__(self, separator: str = "\n\n# --- OpenComb separator ---\n\n"):
+        self.separator = separator
+
+    def combine_files(
+        self,
+        files: list[str | Path],
+        *,
+        add_headers: bool = True,
+        deduplicate_imports: bool = True,
+    ) -> str:
+        """
+        Combine multiple Python files into a single source string.
+
+        Args:
+            files: List of file paths to combine.
+            add_headers: Whether to add a comment header for each file.
+            deduplicate_imports: Try to keep only unique import statements at the top.
+
+        Returns:
+            Combined Python source code as a string.
+        """
+        contents: list[str] = []
+        all_imports: list[str] = []
+        seen_imports: set[str] = set()
+
+        for file_path in files:
+            path = Path(file_path)
+            if not path.exists():
+                raise FileNotFoundError(f"File not found: {path}")
+
+            source = path.read_text(encoding="utf-8")
+
+            if deduplicate_imports:
+                imports, body = self._split_imports(source)
+                for imp in imports:
+                    normalized = imp.strip()
+                    if normalized and normalized not in seen_imports:
+                        seen_imports.add(normalized)
+                        all_imports.append(normalized)
+                source_body = body
+            else:
+                source_body = source
+
+            if add_headers:
+                header = f"# === Source: {path.name} ===\n"
+                contents.append(header + source_body.strip())
+            else:
+                contents.append(source_body.strip())
+
+        result_parts: list[str] = []
+        if all_imports:
+            result_parts.append("\n".join(all_imports))
+            result_parts.append("")
+
+        result_parts.append(self.separator.join(contents))
+        return "\n".join(result_parts).strip() + "\n"
+
+    def combine_snippets(
+        self,
+        snippets: list[str],
+        *,
+        names: list[str] | None = None,
+    ) -> str:
+        """Combine in-memory code snippets."""
+        if names is None:
+            names = [f"snippet_{i}" for i in range(len(snippets))]
+
+        if len(names) != len(snippets):
+            raise ValueError("names and snippets must have the same length")
+
+        parts = []
+        for name, snippet in zip(names, snippets):
+            parts.append(f"# === {name} ===\n{snippet.strip()}")
+
+        return self.separator.join(parts) + "\n"
+
+    @staticmethod
+    def _split_imports(source: str) -> tuple[list[str], str]:
+        """Naive but useful import splitter using AST when possible."""
+        try:
+            tree = ast.parse(source)
+        except SyntaxError:
+            # Fallback: treat everything as body
+            return [], source
+
+        imports: list[str] = []
+        lines = source.splitlines(keepends=True)
+
+        import_end = 0
+        for node in tree.body:
+            if isinstance(node, (ast.Import, ast.ImportFrom)):
+                # Collect the original source lines for this import
+                start = node.lineno - 1
+                end = getattr(node, "end_lineno", node.lineno)
+                import_text = "".join(lines[start:end]).rstrip()
+                imports.append(import_text)
+                import_end = max(import_end, end)
+            else:
+                break
+
+        body = "".join(lines[import_end:]).lstrip("\n")
+        return imports, body
+
+
+class ConfigMerger:
+    """Intelligently merge configuration files (YAML, JSON, TOML)."""
+
+    SUPPORTED = {".yaml", ".yml", ".json", ".toml"}
+
+    def merge_files(
+        self,
+        files: list[str | Path],
+        *,
+        strategy: str = "deep",
+    ) -> dict[str, Any]:
+        """
+        Merge multiple config files.
+
+        Args:
+            files: Config files in order (later files override earlier ones).
+            strategy: "deep" (recursive merge) or "shallow" (top-level only).
+
+        Returns:
+            Merged dictionary.
+        """
+        result: dict[str, Any] = {}
+
+        for file_path in files:
+            path = Path(file_path)
+            if not path.exists():
+                raise FileNotFoundError(f"Config file not found: {path}")
+
+            data = self._load(path)
+            if not isinstance(data, dict):
+                raise ValueError(f"Config must be a mapping/object: {path}")
+
+            if strategy == "deep":
+                result = self._deep_merge(result, data)
+            else:
+                result.update(data)
+
+        return result
+
+    def merge_dicts(
+        self,
+        *dicts: dict[str, Any],
+        strategy: str = "deep",
+    ) -> dict[str, Any]:
+        """Merge multiple dictionaries."""
+        result: dict[str, Any] = {}
+        for d in dicts:
+            if strategy == "deep":
+                result = self._deep_merge(result, d)
+            else:
+                result.update(d)
+        return result
+
+    def save(
+        self,
+        data: dict[str, Any],
+        path: str | Path,
+        *,
+        format: str | None = None,
+    ) -> None:
+        """Save merged config to a file."""
+        path = Path(path)
+        fmt = format or path.suffix.lower().lstrip(".")
+
+        if fmt in ("yaml", "yml"):
+            path.write_text(
+                yaml.dump(data, default_flow_style=False, allow_unicode=True, sort_keys=False),
+                encoding="utf-8",
+            )
+        elif fmt == "json":
+            path.write_text(
+                json.dumps(data, indent=2, ensure_ascii=False) + "\n",
+                encoding="utf-8",
+            )
+        elif fmt == "toml":
+            path.write_bytes(tomli_w.dumps(data).encode("utf-8"))
+        else:
+            raise ValueError(f"Unsupported format: {fmt}. Use yaml, json or toml.")
+
+    def _load(self, path: Path) -> Any:
+        suffix = path.suffix.lower()
+        content = path.read_text(encoding="utf-8")
+
+        if suffix in (".yaml", ".yml"):
+            return yaml.safe_load(content) or {}
+        if suffix == ".json":
+            return json.loads(content)
+        if suffix == ".toml":
+            return tomllib.loads(content)
+
+        raise ValueError(f"Unsupported config format: {suffix}")
+
+    @staticmethod
+    def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+        """Recursively merge two dictionaries. Lists are replaced, not merged."""
+        result = base.copy()
+        for key, value in override.items():
+            if (
+                key in result
+                and isinstance(result[key], dict)
+                and isinstance(value, dict)
+            ):
+                result[key] = ConfigMerger._deep_merge(result[key], value)
+            else:
+                result[key] = value
+        return result

opencomb/prompt.py

@@ -0,0 +1,149 @@
+"""Prompt combining and management for LLM workflows."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+
+class PromptCombiner:
+    """
+    Combine system prompts, user prompts, few-shot examples and context
+    into a single well-structured prompt.
+    """
+
+    def __init__(
+        self,
+        system_separator: str = "\n\n",
+        example_header: str = "### Example {n}\n",
+        section_header: str = "## {title}\n\n",
+    ):
+        self.system_separator = system_separator
+        self.example_header = example_header
+        self.section_header = section_header
+
+    def combine(
+        self,
+        *,
+        system: str | list[str] | None = None,
+        instruction: str | None = None,
+        context: str | list[str] | None = None,
+        examples: list[dict[str, str]] | None = None,
+        user: str | None = None,
+        extra_sections: dict[str, str] | None = None,
+    ) -> str:
+        """
+        Build a complete prompt from components.
+
+        Args:
+            system: One or more system messages.
+            instruction: Main task instruction.
+            context: Background information / retrieved documents.
+            examples: List of {"input": ..., "output": ...} few-shot examples.
+            user: The actual user query.
+            extra_sections: Arbitrary named sections.
+
+        Returns:
+            Fully combined prompt string.
+        """
+        parts: list[str] = []
+
+        # System
+        if system:
+            if isinstance(system, str):
+                systems = [system]
+            else:
+                systems = system
+            system_text = self.system_separator.join(s.strip() for s in systems if s.strip())
+            if system_text:
+                parts.append(self.section_header.format(title="System") + system_text)
+
+        # Instruction
+        if instruction and instruction.strip():
+            parts.append(self.section_header.format(title="Instruction") + instruction.strip())
+
+        # Context
+        if context:
+            if isinstance(context, str):
+                contexts = [context]
+            else:
+                contexts = context
+            ctx_text = "\n\n".join(c.strip() for c in contexts if c.strip())
+            if ctx_text:
+                parts.append(self.section_header.format(title="Context") + ctx_text)
+
+        # Few-shot examples
+        if examples:
+            example_blocks = []
+            for i, ex in enumerate(examples, 1):
+                header = self.example_header.format(n=i)
+                block = header
+                if "input" in ex:
+                    block += f"**Input:**\n{ex['input'].strip()}\n\n"
+                if "output" in ex:
+                    block += f"**Output:**\n{ex['output'].strip()}\n"
+                example_blocks.append(block.strip())
+            if example_blocks:
+                parts.append(
+                    self.section_header.format(title="Examples")
+                    + "\n\n".join(example_blocks)
+                )
+
+        # Extra sections
+        if extra_sections:
+            for title, content in extra_sections.items():
+                if content and content.strip():
+                    parts.append(
+                        self.section_header.format(title=title) + content.strip()
+                    )
+
+        # User query
+        if user and user.strip():
+            parts.append(self.section_header.format(title="User") + user.strip())
+
+        return "\n\n".join(parts).strip() + "\n"
+
+    def from_files(
+        self,
+        *,
+        system_files: list[str | Path] | None = None,
+        instruction_file: str | Path | None = None,
+        context_files: list[str | Path] | None = None,
+        examples_file: str | Path | None = None,
+        user_file: str | Path | None = None,
+    ) -> str:
+        """Load components from files and combine them."""
+        system = None
+        if system_files:
+            system = [Path(f).read_text(encoding="utf-8") for f in system_files]
+
+        instruction = None
+        if instruction_file:
+            instruction = Path(instruction_file).read_text(encoding="utf-8")
+
+        context = None
+        if context_files:
+            context = [Path(f).read_text(encoding="utf-8") for f in context_files]
+
+        examples = None
+        if examples_file:
+            import yaml
+            data = yaml.safe_load(Path(examples_file).read_text(encoding="utf-8"))
+            if isinstance(data, list):
+                examples = data
+
+        user = None
+        if user_file:
+            user = Path(user_file).read_text(encoding="utf-8")
+
+        return self.combine(
+            system=system,
+            instruction=instruction,
+            context=context,
+            examples=examples,
+            user=user,
+        )
+
+    def save(self, prompt: str, path: str | Path) -> None:
+        """Save the combined prompt to a file."""
+        Path(path).write_text(prompt, encoding="utf-8")

opencomb/recipe.py

@@ -0,0 +1,165 @@
+"""Recipe system – declarative combining of code, configs, prompts and templates."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from opencomb.combiner import CodeCombiner, ConfigMerger
+from opencomb.combinatorial import CombinatorialGenerator
+from opencomb.prompt import PromptCombiner
+from opencomb.template import TemplateRenderer
+
+
+class RecipeRunner:
+    """
+    Execute OpenComb recipes defined in YAML.
+
+    A recipe can combine code, merge configs, render templates,
+    build prompts and generate combinatorial sets in one go.
+    """
+
+    def __init__(self, base_dir: str | Path | None = None):
+        self.base_dir = Path(base_dir) if base_dir else Path.cwd()
+
+    def load(self, recipe_path: str | Path) -> dict[str, Any]:
+        """Load a recipe YAML file."""
+        path = Path(recipe_path)
+        if not path.is_absolute():
+            path = self.base_dir / path
+        data = yaml.safe_load(path.read_text(encoding="utf-8"))
+        if not isinstance(data, dict):
+            raise ValueError("Recipe must be a YAML mapping")
+        return data
+
+    def run(self, recipe: dict[str, Any] | str | Path, *, dry_run: bool = False) -> dict[str, Any]:
+        """
+        Execute a recipe.
+
+        Returns a dict with results of each step.
+        """
+        if not isinstance(recipe, dict):
+            recipe = self.load(recipe)
+
+        results: dict[str, Any] = {}
+        name = recipe.get("name", "unnamed")
+        results["_recipe"] = name
+
+        # 1. Code combining
+        if "combine" in recipe:
+            step = recipe["combine"]
+            files = [self._resolve(f) for f in step.get("files", [])]
+            output = step.get("output")
+            combiner = CodeCombiner()
+            combined = combiner.combine_files(
+                files,
+                add_headers=step.get("headers", True),
+                deduplicate_imports=step.get("dedupe_imports", True),
+            )
+            results["combine"] = combined
+            if output and not dry_run:
+                out_path = self._resolve(output)
+                out_path.parent.mkdir(parents=True, exist_ok=True)
+                out_path.write_text(combined, encoding="utf-8")
+                results["combine_output"] = str(out_path)
+
+        # 2. Config merging
+        if "merge" in recipe:
+            step = recipe["merge"]
+            files = [self._resolve(f) for f in step.get("files", [])]
+            output = step.get("output")
+            strategy = step.get("strategy", "deep")
+            merger = ConfigMerger()
+            merged = merger.merge_files(files, strategy=strategy)
+            results["merge"] = merged
+            if output and not dry_run:
+                out_path = self._resolve(output)
+                out_path.parent.mkdir(parents=True, exist_ok=True)
+                fmt = step.get("format")
+                merger.save(merged, out_path, format=fmt)
+                results["merge_output"] = str(out_path)
+
+        # 3. Template rendering
+        if "template" in recipe:
+            step = recipe["template"]
+            template_dirs = step.get("dirs", ["."])
+            renderer = TemplateRenderer(
+                [self._resolve(d) for d in template_dirs],
+                strict=step.get("strict", True),
+            )
+            data = step.get("data", {})
+            if "file" in step:
+                rendered = renderer.render_file(step["file"], **data)
+            elif "files" in step:
+                rendered = renderer.render_files(
+                    step["files"],
+                    separator=step.get("separator", "\n\n"),
+                    **data,
+                )
+            elif "string" in step:
+                rendered = renderer.render_string(step["string"], **data)
+            else:
+                raise ValueError("template step needs 'file', 'files' or 'string'")
+            results["template"] = rendered
+            if "output" in step and not dry_run:
+                out_path = self._resolve(step["output"])
+                out_path.parent.mkdir(parents=True, exist_ok=True)
+                out_path.write_text(rendered, encoding="utf-8")
+                results["template_output"] = str(out_path)
+
+        # 4. Prompt building
+        if "prompt" in recipe:
+            step = recipe["prompt"]
+            combiner = PromptCombiner()
+            prompt = combiner.combine(
+                system=step.get("system"),
+                instruction=step.get("instruction"),
+                context=step.get("context"),
+                examples=step.get("examples"),
+                user=step.get("user"),
+                extra_sections=step.get("extra_sections"),
+            )
+            results["prompt"] = prompt
+            if "output" in step and not dry_run:
+                out_path = self._resolve(step["output"])
+                out_path.parent.mkdir(parents=True, exist_ok=True)
+                out_path.write_text(prompt, encoding="utf-8")
+                results["prompt_output"] = str(out_path)
+
+        # 5. Combinatorial generation
+        if "generate" in recipe:
+            step = recipe["generate"]
+            params = step.get("params", {})
+            method = step.get("method", "cartesian")
+            limit = step.get("limit")
+            seed = step.get("seed")
+            gen = CombinatorialGenerator(seed=seed)
+
+            if method == "cartesian":
+                combos = gen.cartesian(params, limit=limit)
+            elif method == "pairwise":
+                combos = gen.pairwise(params, limit=limit)
+            elif method == "sample":
+                combos = gen.sample(params, n=limit or 10)
+            else:
+                raise ValueError(f"Unknown generate method: {method}")
+
+            results["generate"] = combos
+            if "output" in step and not dry_run:
+                import json
+                out_path = self._resolve(step["output"])
+                out_path.parent.mkdir(parents=True, exist_ok=True)
+                with out_path.open("w", encoding="utf-8") as f:
+                    for c in combos:
+                        f.write(json.dumps(c, ensure_ascii=False) + "\n")
+                results["generate_output"] = str(out_path)
+
+        return results
+
+    def _resolve(self, path: str | Path) -> Path:
+        p = Path(path)
+        if p.is_absolute():
+            return p
+        return self.base_dir / p

opencomb/template.py

@@ -0,0 +1,68 @@
+"""Jinja2-powered template rendering and combining."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from jinja2 import Environment, FileSystemLoader, StrictUndefined, Template, select_autoescape
+
+
+class TemplateRenderer:
+    """Render and combine Jinja2 templates with data."""
+
+    def __init__(
+        self,
+        template_dirs: list[str | Path] | None = None,
+        *,
+        strict: bool = True,
+    ):
+        search_paths = [str(p) for p in (template_dirs or ["."])]
+        self.env = Environment(
+            loader=FileSystemLoader(search_paths),
+            autoescape=select_autoescape(enabled_extensions=("html", "xml")),
+            undefined=StrictUndefined if strict else None,
+            trim_blocks=True,
+            lstrip_blocks=True,
+            keep_trailing_newline=True,
+        )
+
+    def render_string(self, template_str: str, **context: Any) -> str:
+        """Render a template from a string."""
+        tmpl = self.env.from_string(template_str)
+        return tmpl.render(**context)
+
+    def render_file(self, template_name: str, **context: Any) -> str:
+        """Render a template file (relative to template_dirs)."""
+        tmpl = self.env.get_template(template_name)
+        return tmpl.render(**context)
+
+    def render_files(
+        self,
+        template_names: list[str],
+        *,
+        separator: str = "\n\n",
+        **context: Any,
+    ) -> str:
+        """Render multiple templates and join them."""
+        parts = [self.render_file(name, **context) for name in template_names]
+        return separator.join(parts)
+
+    def combine_and_render(
+        self,
+        templates: list[str | Path],
+        *,
+        data: dict[str, Any] | None = None,
+        separator: str = "\n\n# --- template separator ---\n\n",
+    ) -> str:
+        """
+        Load multiple template files, concatenate them, then render once.
+        Useful when you want shared macros / blocks across files.
+        """
+        contents = []
+        for t in templates:
+            path = Path(t)
+            contents.append(path.read_text(encoding="utf-8"))
+
+        combined = separator.join(contents)
+        return self.render_string(combined, **(data or {}))