llm-conductor 1.0.0__py3-none-any.whl

llm_conductor/__init__.py

@@ -0,0 +1,4 @@
+# -*- coding: utf-8 -*-
+"""LLM-Conductor: JSON localization with language models."""
+
+__version__ = "1.0.0"

llm_conductor/__main__.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+"""Allow ``python -m llm_conductor``."""
+
+from llm_conductor.cli.main import main
+
+if __name__ == "__main__":
+    main()

llm_conductor/cli/exit_status.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+"""
+Exit code of a run that may hit several problems at once.
+
+Every problem is recorded with its code; the most severe one by
+EXIT_CODE_PRIORITY becomes the process exit code.
+"""
+import threading
+from typing import Iterable, List, NamedTuple
+
+from llm_conductor.constants import EXIT_CODE_PRIORITY, EXIT_SUCCESS
+
+
+def _checked(codes: Iterable[int]) -> List[int]:
+    """Reject codes outside the contract - an unknown code has no defined severity."""
+    checked = list(codes)
+    unknown = [code for code in checked if code not in EXIT_CODE_PRIORITY]
+    if unknown:
+        raise ValueError(f"exit codes {unknown} are not part of EXIT_CODE_PRIORITY")
+    return checked
+
+
+def most_severe(codes: Iterable[int]) -> int:
+    """
+    The code that wins by EXIT_CODE_PRIORITY.
+
+    Args:
+        codes: Exit codes; each must be listed in EXIT_CODE_PRIORITY.
+
+    Returns:
+        The most severe code, EXIT_SUCCESS for no codes.
+
+    Raises:
+        ValueError: On a code missing from EXIT_CODE_PRIORITY.
+    """
+    ranks = [EXIT_CODE_PRIORITY.index(code) for code in _checked(codes)]
+    return EXIT_CODE_PRIORITY[min(ranks)] if ranks else EXIT_SUCCESS
+
+
+class ExitRecord(NamedTuple):
+    """One recorded problem of a run."""
+
+    code: int
+    reason: str
+
+
+class ExitStatus:
+    """
+    Thread-safe accumulator of the problems of one run.
+
+    record() may be called from any thread; code and records are consistent
+    snapshots taken under the same lock.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._records: List[ExitRecord] = []
+
+    def record(self, code: int, reason: str) -> None:
+        """
+        Record a problem.
+
+        Args:
+            code: Its exit code from EXIT_CODE_PRIORITY.
+            reason: Human-readable description for the final log line.
+
+        Raises:
+            ValueError: On a code missing from EXIT_CODE_PRIORITY.
+        """
+        _checked([code])
+        with self._lock:
+            self._records.append(ExitRecord(code, reason))
+
+    @property
+    def code(self) -> int:
+        """The exit code of the run: the most severe recorded one, EXIT_SUCCESS for none."""
+        with self._lock:
+            return most_severe(record.code for record in self._records)
+
+    @property
+    def records(self) -> List[ExitRecord]:
+        """A copy of the recorded problems, in recording order."""
+        with self._lock:
+            return list(self._records)

llm_conductor/cli/glossary.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+"""
+Where the glossary of a run comes from.
+
+The lookup chain has two levels: --glossary PATH (a missing file is an
+error), then glossary.json next to the target (file or manifest), then no
+glossary at all. A glossary is project data, so there is no shared fallback
+next to the executable - it would leak one project's terms into another.
+"""
+import os
+from dataclasses import dataclass, field
+from typing import Dict, Optional
+
+from llm_conductor.cli.targets import RunTarget
+from llm_conductor.constants import (
+    GLOSSARY_FILENAME,
+    GLOSSARY_ORIGIN_OPTION,
+    GLOSSARY_ORIGIN_TARGET,
+)
+from llm_conductor.core.io import safe_read_file
+from llm_conductor.exceptions import FileOperationError, TargetError
+from llm_conductor.interfaces import FileOperationsProtocol, Logger
+
+
+@dataclass(frozen=True)
+class GlossarySource:
+    """The glossary of a run and where it was found."""
+
+    terms: Dict[str, str] = field(default_factory=dict)
+    path: Optional[str] = None
+    origin: Optional[str] = None
+
+    def describe(self) -> str:
+        """One log line naming the source, its level of the chain and the number of terms."""
+        if self.path is None:
+            return "Glossary: none given and no glossary.json next to the target - working without terms"
+        return f"Glossary: {self.path} ({self.origin}), {len(self.terms)} term(s)"
+
+
+class GlossaryResolver:
+    """Walks the lookup chain and loads the glossary it finds."""
+
+    def __init__(self, file_ops: FileOperationsProtocol, logger: Logger):
+        """
+        Args:
+            file_ops: File operations for reading the glossary.
+            logger: Logger of the run.
+        """
+        self._file_ops = file_ops
+        self._logger = logger
+
+    def resolve(self, explicit_path: Optional[str], target: RunTarget) -> GlossarySource:
+        """
+        Find and load the glossary of a run.
+
+        Args:
+            explicit_path: Value of --glossary, relative to the working directory.
+            target: The run target; its directory holds the implicit glossary.json.
+
+        Returns:
+            The loaded glossary, or an empty one when the chain finds nothing.
+
+        Raises:
+            TargetError: If the explicit glossary is missing, or a found
+                glossary is unreadable or not an object of strings.
+        """
+        if explicit_path:
+            path = os.path.abspath(explicit_path)
+            if not os.path.isfile(path):
+                raise TargetError(f"Glossary {path} not found")
+            return self._load(path, GLOSSARY_ORIGIN_OPTION)
+
+        candidate = os.path.join(target.directory, GLOSSARY_FILENAME)
+        if os.path.isfile(candidate):
+            return self._load(candidate, GLOSSARY_ORIGIN_TARGET)
+        return GlossarySource()
+
+    def _load(self, path: str, origin: str) -> GlossarySource:
+        """
+        Read and check one glossary file.
+
+        Raises:
+            TargetError: If it is unreadable or not a {term: translation} object of strings.
+        """
+        try:
+            data = safe_read_file(path, f"glossary {path}", self._file_ops, self._logger,
+                                  critical=True, as_text=False)
+        except FileOperationError as e:
+            raise TargetError(str(e)) from e
+
+        if not isinstance(data, dict) or not all(
+                isinstance(term, str) and isinstance(translation, str) for term, translation in data.items()):
+            raise TargetError(f"Glossary {path} must be a JSON object mapping terms to translation strings")
+        return GlossarySource(data, path, origin)

llm_conductor/cli/heartbeat.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+"""
+Heartbeat JSON snapshot of a run ({cache_dir}/progress_state.json).
+
+External monitors read this file; it is a status report, not a lock and not
+the source of truth for the process exit code (that remains sys.exit in main).
+Write failures degrade to a WARNING so the pipeline keeps going.
+"""
+from __future__ import annotations
+
+import json
+import os
+import time
+from datetime import UTC, datetime
+from typing import Any, Dict, List, Optional, Sequence
+
+from llm_conductor.cli.exit_status import ExitRecord
+from llm_conductor.cli.glossary import GlossarySource
+from llm_conductor.cli.targets import RunTarget, TargetEntry
+from llm_conductor.constants import (
+    HEARTBEAT_SCHEMA_VERSION,
+    HEARTBEAT_STATUS_COMPLETED,
+    HEARTBEAT_STATUS_INTERRUPTED,
+    HEARTBEAT_STATUS_RUNNING,
+    PROGRESS_STATE_FILENAME,
+)
+from llm_conductor.core.io import safe_write_file
+from llm_conductor.exceptions import FileOperationError
+from llm_conductor.interfaces import FileOperationsProtocol, Logger
+
+
+class HeartbeatWriter:
+    """
+    Builds and rewrites the compact progress_state.json of one run.
+
+    Implements HeartbeatProgressProtocol (begin_stage / note_leaf) for the
+    processor; file and final boundaries are driven by TargetRunner.
+    """
+
+    def __init__(
+        self,
+        cache_dir: str,
+        interval: int,
+        file_ops: FileOperationsProtocol,
+        logger: Logger,
+        target: RunTarget,
+        profile_name: str,
+        stages: Sequence[str],
+        glossary: GlossarySource,
+    ):
+        """
+        Args:
+            cache_dir: Resolved [cache].dir (created on first write if needed).
+            interval: Leaf rewrite cadence; ``<= 0`` skips leaf-only flushes.
+            file_ops: Atomic text writer.
+            logger: Run logger (WARNING on a failed rewrite).
+            target: Resolved run target.
+            profile_name: ``[meta].name`` of the active llm.toml.
+            stages: Operation types that will run, in pipeline order.
+            glossary: Glossary source of the run (same facts as the INFO log).
+        """
+        self._path = os.path.join(cache_dir, PROGRESS_STATE_FILENAME)
+        self._cache_dir = cache_dir
+        self._interval = interval
+        self._file_ops = file_ops
+        self._logger = logger
+
+        self._started = False
+        self._final = False
+        self._started_at = 0.0
+        self._stage_started_at = 0.0
+        self._leaves_since_flush = 0
+
+        self._status = HEARTBEAT_STATUS_RUNNING
+        self._exit_code: Optional[int] = None
+        self._problems: List[Dict[str, Any]] = []
+
+        self._target = {
+            'path': target.path,
+            'name': target.name,
+            'kind': 'manifest' if target.is_manifest else 'file',
+        }
+        self._profile = {'name': profile_name}
+        self._stages = list(stages)
+        self._glossary = {
+            'origin': glossary.origin,
+            'path': glossary.path,
+            'term_count': len(glossary.terms),
+        }
+
+        self._files_total = len(target.entries)
+        self._all_relative = [entry.relative_path for entry in target.entries]
+        self._done: List[str] = []
+        self._failed: List[Dict[str, str]] = []
+        self._pending: List[str] = list(self._all_relative)
+
+        self._file_index = 0
+        self._current_file: Optional[str] = None
+        self._stage: Optional[str] = None
+        self._leaves_done = 0
+        self._leaves_total = 0
+
+    @property
+    def path(self) -> str:
+        """Absolute or relative path of progress_state.json."""
+        return self._path
+
+    @property
+    def started(self) -> bool:
+        """True after the first successful start() call."""
+        return self._started
+
+    @property
+    def finalized(self) -> bool:
+        """True after finalize() has written the terminal snapshot."""
+        return self._final
+
+    def start(self) -> None:
+        """Mark the run as running and write the initial snapshot."""
+        if self._final:
+            return
+        self._started = True
+        self._started_at = time.monotonic()
+        self._status = HEARTBEAT_STATUS_RUNNING
+        self._exit_code = None
+        self._flush()
+
+    def on_file_start(self, entry: TargetEntry, file_index: int) -> None:
+        """
+        Record that a file is about to be processed.
+
+        Args:
+            entry: The file being started.
+            file_index: 1-based index among target entries.
+        """
+        if not self._started or self._final:
+            return
+        self._file_index = file_index
+        self._current_file = entry.relative_path
+        if entry.relative_path in self._pending:
+            self._pending = [path for path in self._pending if path != entry.relative_path]
+        self._stage = None
+        self._leaves_done = 0
+        self._leaves_total = 0
+        self._flush()
+
+    def on_file_done(self, relative_path: str, status_name: str) -> None:
+        """
+        Record the outcome of one file and rewrite the snapshot.
+
+        Args:
+            relative_path: Path under the result stages.
+            status_name: FileStatus value (done / failed / crashed / aborted).
+        """
+        if not self._started or self._final:
+            return
+        if status_name == 'done':
+            if relative_path not in self._done:
+                self._done.append(relative_path)
+        else:
+            self._failed.append({'path': relative_path, 'status': status_name})
+        self._current_file = None
+        self._stage = None
+        self._leaves_done = 0
+        self._leaves_total = 0
+        self._flush()
+
+    def begin_stage(self, stage: str, leaves_total: int) -> None:
+        """
+        Start a translation or verification stage for the current file.
+
+        Args:
+            stage: OPERATION_TRANSLATION or OPERATION_VERIFICATION.
+            leaves_total: Pre-scanned string-leaf count for this stage.
+        """
+        if not self._started or self._final:
+            return
+        self._stage = stage
+        self._leaves_total = max(0, leaves_total)
+        self._leaves_done = 0
+        self._leaves_since_flush = 0
+        self._stage_started_at = time.monotonic()
+        self._flush()
+
+    def note_leaf(self) -> None:
+        """Count one finished string leaf; flush every ``interval`` leaves."""
+        if not self._started or self._final:
+            return
+        self._leaves_done += 1
+        if self._interval <= 0:
+            return
+        self._leaves_since_flush += 1
+        if self._leaves_since_flush >= self._interval:
+            self._leaves_since_flush = 0
+            self._flush()
+
+    def set_problems(self, records: Sequence[ExitRecord]) -> None:
+        """Mirror ExitStatus.records into the snapshot (usually before finalize)."""
+        self._problems = [{'code': record.code, 'reason': record.reason} for record in records]
+
+    def finalize(self, status: str, exit_code: int) -> None:
+        """
+        Write the terminal snapshot.
+
+        Args:
+            status: HEARTBEAT_STATUS_COMPLETED or HEARTBEAT_STATUS_INTERRUPTED.
+            exit_code: Same code the process will exit with.
+        """
+        if not self._started or self._final:
+            return
+        if status not in (HEARTBEAT_STATUS_COMPLETED, HEARTBEAT_STATUS_INTERRUPTED):
+            status = HEARTBEAT_STATUS_COMPLETED
+        self._status = status
+        self._exit_code = exit_code
+        self._current_file = None
+        self._stage = None
+        self._final = True
+        self._flush()
+
+    def _files_done_count(self) -> int:
+        """Number of files that already have an outcome."""
+        return len(self._done) + len(self._failed)
+
+    def _leaf_frac(self) -> float:
+        if self._leaves_total <= 0:
+            return 0.0
+        return min(self._leaves_done / self._leaves_total, 1.0)
+
+    def _percent(self) -> float:
+        total = self._files_total
+        if total <= 0:
+            return 0.0
+        return 100.0 * (self._files_done_count() + self._leaf_frac()) / total
+
+    def _speed_leaves_per_min(self) -> float:
+        if self._leaves_done <= 0 or self._stage_started_at <= 0:
+            return 0.0
+        elapsed = time.monotonic() - self._stage_started_at
+        if elapsed <= 0:
+            return 0.0
+        return self._leaves_done / (elapsed / 60.0)
+
+    def _elapsed_sec(self) -> float:
+        if self._started_at <= 0:
+            return 0.0
+        return max(0.0, time.monotonic() - self._started_at)
+
+    def _snapshot(self) -> Dict[str, Any]:
+        return {
+            'schema_version': HEARTBEAT_SCHEMA_VERSION,
+            'status': self._status,
+            'exit_code': self._exit_code,
+            'updated_at': datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ'),
+            'elapsed_sec': round(self._elapsed_sec(), 3),
+            'target': dict(self._target),
+            'profile': dict(self._profile),
+            'stages': list(self._stages),
+            'glossary': dict(self._glossary),
+            'progress': {
+                'percent': round(self._percent(), 2),
+                'files_done': self._files_done_count(),
+                'files_total': self._files_total,
+                'file_index': self._file_index,
+                'current_file': self._current_file,
+                'stage': self._stage,
+                'leaves_done': self._leaves_done,
+                'leaves_total': self._leaves_total,
+                'speed_leaves_per_min': round(self._speed_leaves_per_min(), 2),
+            },
+            'files': {
+                'done': list(self._done),
+                'failed': [dict(item) for item in self._failed],
+                'pending': list(self._pending),
+            },
+            'problems': [dict(item) for item in self._problems],
+        }
+
+    def _flush(self) -> None:
+        """Atomically rewrite progress_state.json; never raise into the pipeline."""
+        try:
+            os.makedirs(self._cache_dir, exist_ok=True)
+            payload = json.dumps(self._snapshot(), ensure_ascii=False, separators=(',', ':'))
+            wrote = safe_write_file(
+                self._path, payload, 'heartbeat progress state',
+                self._file_ops, self._logger,
+            )
+            if not wrote:
+                self._logger.warning(
+                    f"Heartbeat write to {self._path} failed; continuing the run"
+                )
+        except FileOperationError as e:
+            self._logger.warning(
+                f"Heartbeat write to {self._path} failed ({e}); continuing the run"
+            )
+        except OSError as e:
+            self._logger.warning(
+                f"Heartbeat write to {self._path} failed ({e}); continuing the run"
+            )

llm_conductor/cli/init.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+"""
+The 'init' command: commented templates of conductor.toml and llm.toml.
+
+Templates are rendered from the schema (llm_conductor/schema.py), so every key the
+loader accepts appears in them with its description and default.
+"""
+import json
+import math
+import os
+from typing import Any, List, Optional, Tuple
+
+from llm_conductor.core.io import FileOperations, safe_write_file
+from llm_conductor.exceptions import ConfigValidationError, FileOperationError
+from llm_conductor.interfaces import FileOperationsProtocol
+from llm_conductor.schema import CONDUCTOR_SCHEMA, LLM_SCHEMA, FileSchema, Setting
+
+_LITERAL_DELIMITER = "'''"
+_DEL = "\x7f"
+
+
+def format_toml_value(value: Any) -> str:
+    """
+    Render a scalar or a list as a TOML value.
+
+    Strings become basic (double-quoted) strings: JSON escapes are a subset
+    of TOML ones, and DEL, which JSON leaves raw but TOML forbids, is
+    escaped separately.
+
+    Args:
+        value: bool, int, float, str, or a list/tuple of those.
+
+    Returns:
+        The TOML representation.
+
+    Raises:
+        TypeError: For any other type (tables are rendered by the caller).
+    """
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        if math.isnan(value):
+            return "nan"
+        if math.isinf(value):
+            return "inf" if value > 0 else "-inf"
+        return repr(value)
+    if isinstance(value, str):
+        return json.dumps(value, ensure_ascii=False).replace(_DEL, "\\u007f")
+    if isinstance(value, (list, tuple)):
+        return "[" + ", ".join(format_toml_value(item) for item in value) + "]"
+    raise TypeError(f"Cannot render {type(value).__name__} as a TOML value")
+
+
+def format_toml_multiline(value: str) -> str:
+    """
+    Render a string as a multi-line literal, kept byte for byte without escaping.
+
+    Falls back to a basic string when a literal cannot hold the text
+    (it contains ''' or control characters other than newline and tab, or
+    ends with a quote that would merge with the closing delimiter).
+
+    Args:
+        value: The string.
+
+    Returns:
+        The TOML representation.
+    """
+    has_control = any((ch < " " or ch == _DEL) and ch not in "\n\t" for ch in value)
+    if _LITERAL_DELIMITER in value or has_control or value.endswith("'"):
+        return format_toml_value(value)
+    # The newline right after the opening delimiter is trimmed by TOML parsers.
+    return f"{_LITERAL_DELIMITER}\n{value}{_LITERAL_DELIMITER}"
+
+
+class TomlTemplateRenderer:
+    """Renders a FileSchema as a commented TOML file filled with the defaults."""
+
+    def __init__(self, schema: FileSchema):
+        """
+        Args:
+            schema: Schema of the file to render.
+        """
+        self._schema = schema
+
+    def render(self) -> str:
+        """
+        Build the template text.
+
+        Returns:
+            A TOML document that parses back to the schema defaults.
+        """
+        lines: List[str] = self._comment(self._schema.header)
+        lines += self._comment("Generated by `llm-conductor init`. Absent keys take the defaults below; "
+                               "unknown keys are an error.")
+        for section in self._schema.sections:
+            lines.append("")
+            lines += self._comment(section.description)
+            lines.append(f"[{section.name}]")
+            for setting in section.settings:
+                lines += self._render_setting(setting)
+        return "\n".join(lines) + "\n"
+
+    def _render_setting(self, setting: Setting) -> List[str]:
+        """Comment line(s) and the key line of one setting."""
+        value = setting.init_value
+        if setting.multiline:
+            return self._comment(setting.description) + [
+                f"{setting.key} = {format_toml_multiline(value)}"]
+        rendered = format_toml_value(value)
+        # Comment shows the absent-key default; the line may use template_default.
+        default_note = format_toml_value(setting.default)
+        return self._comment(f"{setting.description} Default: {default_note}") + [
+            f"{setting.key} = {rendered}"]
+
+    @staticmethod
+    def _comment(text: str) -> List[str]:
+        """Turn text into '# '-prefixed lines."""
+        return [f"# {line}" if line else "#" for line in text.split("\n")]
+
+
+def create_config_files(conductor_path: str, llm_path: str,
+                        file_ops: Optional[FileOperationsProtocol] = None) -> None:
+    """
+    Write fresh conductor.toml and llm.toml templates.
+
+    Existing files are never overwritten: a configured profile may hold
+    hand-tuned prompts and keys.
+
+    Args:
+        conductor_path: Destination of conductor.toml.
+        llm_path: Destination of llm.toml.
+        file_ops: File operations for writing; default FileOperations.
+
+    Raises:
+        ConfigValidationError: If either file already exists.
+        FileOperationError: If a file cannot be written.
+    """
+    targets: Tuple[Tuple[str, FileSchema], ...] = ((conductor_path, CONDUCTOR_SCHEMA), (llm_path, LLM_SCHEMA))
+    existing = [path for path, _ in targets if os.path.exists(path)]
+    if existing:
+        raise ConfigValidationError(
+            f"Refusing to overwrite existing configuration: {', '.join(existing)}. "
+            f"Remove or rename the file(s) to generate fresh templates.")
+
+    writer = file_ops or FileOperations()
+    for path, schema in targets:
+        text = TomlTemplateRenderer(schema).render()
+        if not safe_write_file(path, text, f"{schema.file_name} template", writer):
+            raise FileOperationError(f"Failed to write {path}")
+        print(f"Created {path}")
+
+    print("For OpenAI-compatible APIs (OpenAI, DeepSeek, OpenRouter), edit llm.toml:")
+    print("  - set type = \"openai\" and url in [translation.api] / [verification.api]")
+    print("  - set key_env to the environment variable holding the key (or key to the key itself)")