codeplain 0.3.11.dev22__tar.gz → 0.3.12__tar.gz

{codeplain-0.3.11.dev22 → codeplain-0.3.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: codeplain
-Version: 0.3.11.dev22
+Version: 0.3.12
 Summary: Transform plain language specifications into working code
 License-File: LICENSE
 Classifier: Environment :: Console

{codeplain-0.3.11.dev22 → codeplain-0.3.12}/codeplain_REST_api.py

@@ -1,8 +1,11 @@
+import socket
 import time
 from typing import Optional
 
 import requests
+from requests.adapters import HTTPAdapter
 from requests.exceptions import ConnectionError, RequestException, Timeout
+from urllib3.connection import HTTPConnection
 
 import plain2code_exceptions
 from plain2code_console import RETRY_COLOR
@@ -15,6 +18,57 @@ RETRY_DELAY = 3
 # at exit, and requests waits indefinitely by default.
 REPORT_TIMEOUT_SECONDS = 5
 
+# A render call can leave the connection with nothing on it for several minutes while the
+# service works on the answer. Network equipment between the two, such as the address
+# translation a CI runner sits behind, drops a connection that quiet and tells neither end.
+# The answer then has nowhere to go and the client waits for it for as long as it is allowed
+# to run. These probes put a packet on the connection often enough to keep it, and they end
+# it when the other end has really gone.
+KEEPALIVE_IDLE_SECONDS = 60
+KEEPALIVE_INTERVAL_SECONDS = 30
+KEEPALIVE_FAILED_PROBES = 4
+
+
+def keepalive_socket_options():
+    """The socket options that keep a long, quiet request alive, for the platforms that have them."""
+    options = list(HTTPConnection.default_socket_options)
+    options.append((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1))
+    # The option that sets the idle time is named differently per platform, and Windows has
+    # none of the three. Whatever is missing is left out; the keepalive itself still applies.
+    for idle_option in ("TCP_KEEPIDLE", "TCP_KEEPALIVE"):
+        if hasattr(socket, idle_option):
+            options.append((socket.IPPROTO_TCP, getattr(socket, idle_option), KEEPALIVE_IDLE_SECONDS))
+            break
+    if hasattr(socket, "TCP_KEEPINTVL"):
+        options.append((socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, KEEPALIVE_INTERVAL_SECONDS))
+    if hasattr(socket, "TCP_KEEPCNT"):
+        options.append((socket.IPPROTO_TCP, socket.TCP_KEEPCNT, KEEPALIVE_FAILED_PROBES))
+    return options
+
+
+class KeepaliveAdapter(HTTPAdapter):
+    """Opens every connection with keepalive on, whether it goes direct or through a proxy."""
+
+    def init_poolmanager(self, *args, **kwargs):
+        kwargs["socket_options"] = keepalive_socket_options()
+        return super().init_poolmanager(*args, **kwargs)
+
+    def proxy_manager_for(self, proxy, **proxy_kwargs):
+        # A request that goes through a proxy is served from a separate pool, which the
+        # options above never reach. A render behind a proxy is as quiet as any other and
+        # needs the same keepalive.
+        proxy_kwargs["socket_options"] = keepalive_socket_options()
+        return super().proxy_manager_for(proxy, **proxy_kwargs)
+
+
+def new_session():
+    """A requests session whose connections carry keepalive."""
+    session = requests.Session()
+    session.mount("https://", KeepaliveAdapter())
+    session.mount("http://", KeepaliveAdapter())
+    return session
+
+
 RETRY_ERROR_CODES = [
     "LLMInternalError",
 ]
@@ -36,6 +90,7 @@ class CodeplainAPI:
     def __init__(self, api_key, console):
         self.api_key = api_key
         self.console = console
+        self.session = new_session()
 
     @property
     def api_url(self):
@@ -139,7 +194,7 @@ class CodeplainAPI:
 
         for attempt in range(num_retries + 1):
             try:
-                response = requests.post(endpoint_url, headers=headers, json=payload, timeout=timeout)
+                response = self.session.post(endpoint_url, headers=headers, json=payload, timeout=timeout)
 
                 try:
                     response_json = response.json()
@@ -253,38 +308,6 @@ class CodeplainAPI:
 
         return self.post_request(endpoint_url, headers, payload, run_state)
 
-    def fix_unittests_issue(
-        self,
-        frid,
-        plain_source_tree,
-        linked_resources,
-        existing_files_content,
-        module_name: str,
-        required_modules,
-        unittests_issue,
-        run_state: RunState,
-        conformance_tests_fixes: list[dict] | None = None,
-    ):
-        endpoint_url = f"{self.api_url}/fix_unittests_issue"
-        headers = {"X-API-Key": self.api_key, "Content-Type": "application/json"}
-
-        payload = {
-            "frid": frid,
-            "plain_source_tree": plain_source_tree,
-            "linked_resources": linked_resources,
-            "existing_files_content": existing_files_content,
-            "module_name": module_name,
-            "required_modules": required_modules,
-            "unittests_issue": unittests_issue,
-            "unittest_batch_id": run_state.unittest_batch_id,
-        }
-        # Implementation code changes made by the conformance tests fixer right before this unit tests run;
-        # only sent when unit tests are processed inside the conformance tests phase.
-        if conformance_tests_fixes is not None:
-            payload["conformance_tests_fixes"] = conformance_tests_fixes
-
-        return self.post_request(endpoint_url, headers, payload, run_state)
-
     def distill_conformance_test_memory(
         self,
         frid,
@@ -558,3 +581,29 @@ class CodeplainAPI:
         }
 
         return self.post_request(endpoint_url, headers, payload, run_state)
+
+    def agent_start(self, task_type: str, task_params: dict, frid: str, module_name: str, run_state: RunState):
+        """Start a server-side agent session; returns the first turn (tool calls or completion)."""
+        endpoint_url = f"{self.api_url}/agent/start"
+        headers = {"X-API-Key": self.api_key, "Content-Type": "application/json"}
+        payload = {
+            "task_type": task_type,
+            "task_params": task_params,
+            "frid": frid,
+            "module_name": module_name,
+        }
+        return self.post_request(endpoint_url, headers, payload, run_state)
+
+    def agent_continue(
+        self, session_id: str, tool_results: list[dict], frid: str, module_name: str, run_state: RunState
+    ):
+        """Feed tool results into an agent session and run its next turn."""
+        endpoint_url = f"{self.api_url}/agent/continue"
+        headers = {"X-API-Key": self.api_key, "Content-Type": "application/json"}
+        payload = {
+            "session_id": session_id,
+            "tool_results": tool_results,
+            "frid": frid,
+            "module_name": module_name,
+        }
+        return self.post_request(endpoint_url, headers, payload, run_state)

{codeplain-0.3.11.dev22 → codeplain-0.3.12}/docs/plain2code_cli.md

@@ -121,6 +121,8 @@ options:
   --headless            Run in headless mode: no TUI. Progress is written to
                         the log file and streamed to stdout as it happens; the
                         specification text itself stays in the log file only.
+                        Used automatically when stdin, stdout or stderr is not
+                        a terminal.
   --status              Display account status including user information, API
                         key label, and rendering credits. Does not render any
                         code.

{codeplain-0.3.11.dev22 → codeplain-0.3.12}/plain2code.py

@@ -161,6 +161,17 @@ def setup_logging(
     return logging.getLevelName(configured_log_level)
 
 
+def attached_to_terminal() -> bool:
+    """Whether a person at a terminal can see and drive the TUI.
+
+    The TUI reads keys from stdin and draws on stderr; the exit summary goes to
+    stdout. Started from a script or an agent, stdin is usually at end-of-file,
+    and Textual's input thread then reads EOF in a tight loop at 100% CPU while
+    a stopped render waits for an Enter nobody can press.
+    """
+    return all(stream is not None and stream.isatty() for stream in (sys.stdin, sys.stdout, sys.stderr))
+
+
 def _check_connection(codeplainAPI: codeplain_api.CodeplainAPI) -> Optional[str]:
     """Check API connectivity, validate API key and client version, and return the user email."""
     response = codeplainAPI.connection_check(system_config.client_version)
@@ -473,12 +484,21 @@ def main():  # noqa: C901
 
     run_state = RunState(spec_filename=args.filename, replay_with=args.replay_with)
 
+    # Without a terminal nobody can watch or drive the TUI, so render as --headless does:
+    # progress on stdout, and a non-zero exit status when the render stops.
+    no_terminal = not args.headless and not attached_to_terminal()
+    if no_terminal:
+        args.headless = True
+
     if args.headless:
         # Suppress Rich console output.
         console.quiet = True
 
     default_log_level = setup_logging(args, event_bus, run_state, args.log_to_file, args.log_file_name, args.headless)
 
+    if no_terminal:
+        console.info("No terminal attached, so rendering without the TUI, as with --headless.")
+
     initialize_telemetry()
 
     exc_info = None

{codeplain-0.3.11.dev22 → codeplain-0.3.12}/plain2code_arguments.py

@@ -428,7 +428,8 @@ def create_parser():
         action="store_true",
         default=False,
         help="Run in headless mode: no TUI. Progress is written to the log file and streamed "
-        "to stdout as it happens; the specification text itself stays in the log file only.",
+        "to stdout as it happens; the specification text itself stays in the log file only. "
+        "Used automatically when stdin, stdout or stderr is not a terminal.",
     )
 
     parser.add_argument(

codeplain-0.3.12/render_machine/actions/fix_unit_tests.py

@@ -0,0 +1,256 @@
+import os
+from http import HTTPStatus
+from typing import Any
+
+import requests
+
+import file_utils
+import plain_spec
+from plain2code_console import console
+from plain2code_exceptions import InternalClientError
+from render_machine import agent_tools
+from render_machine.actions.base_action import BaseAction
+from render_machine.render_context import RenderContext
+
+TASK_TYPE = "fix_unit_tests"
+SUBMIT_FIX_TOOL = "submit_fix"
+# Upper bound on LLM turns spent on one fix attempt; the server bounds the whole session.
+MAX_AGENT_TURNS_PER_ATTEMPT = 40
+# Answer to a submit_fix whose fix was accepted in an earlier unit-test loop of the conformance phase.
+ACCEPTED_THEN_CHANGED_MESSAGE = (
+    "Your fix was accepted: the unit tests passed. Afterwards the implementation code was changed to fix the "
+    "conformance tests{see_below}, and the unit tests now fail again. Files may have changed since you last read "
+    "them."
+)
+# Seeding the first turn: the build folder's file list and the files changed for the FRID.
+MAX_FILE_TREE_ENTRIES = 500
+MAX_RELEVANT_FILES_CHARS = 60_000
+
+
+class FixUnitTests(BaseAction):
+    """Fix failing unit tests with a server-side agent session that spans the fix attempts.
+
+    The first failure starts a session; the agent then drives read/grep/edit/run tool calls
+    (executed here) until it calls submit_fix. The state machine re-runs the unit tests and,
+    if they still fail, the next execution of this action answers that submit_fix call with
+    the new failure output inside the same session, so earlier attempts stay in context.
+
+    During the conformance phase the session also spans unit-test loops: after the conformance
+    tests fixer changes implementation code, the unit tests fail again and the same session is
+    continued, told what the conformance tests fixer changed (which it must preserve).
+    """
+
+    SUCCESSFUL_OUTCOME = "unit_tests_fix_generated"
+
+    def execute(self, render_context: RenderContext, previous_action_payload: Any | None):
+        if not previous_action_payload or not previous_action_payload.get("previous_unittests_issue"):
+            raise InternalClientError(
+                "Internal client error: Previous action payload does not contain previous unit tests issue."
+            )
+        unittests_issue = previous_action_payload["previous_unittests_issue"]
+        context = render_context.unit_tests_running_context
+        session = render_context.unit_tests_agent_session
+        api = render_context.codeplain_api
+        frid, module_name = render_context.frid_context.frid, render_context.module_name
+        changed_files_before = set(context.changed_files)
+        log_path = render_context.script_execution_history.latest_unit_test_output_path
+        if log_path:
+            agent_tools.register_log_path(log_path, render_context)
+        conformance_tests_fixes = self._get_conformance_tests_fixes(render_context)
+        new_conformance_tests_fixes = conformance_tests_fixes[session.conformance_fixes_handed_off :]
+        if new_conformance_tests_fixes:
+            console.info(
+                f"Unit tests are fixed while preserving {len(new_conformance_tests_fixes)} implementation code "
+                "change(s) made to fix the conformance tests."
+            )
+
+        response = None
+        if session.session_id is not None:
+            if context.agent_used_in_this_loop:
+                console.info(f"Continuing agent session {session.session_id} with the new unit tests failure.")
+                output = "The fix was applied, but the unit tests still fail."
+            else:
+                # The session's last fix was accepted in an earlier unit-test loop of this conformance
+                # phase; since then the conformance tests fixer changed the code and the tests fail again.
+                console.info(
+                    f"Continuing agent session {session.session_id}: the unit tests fail again after the "
+                    "implementation was changed to fix the conformance tests."
+                )
+                output = ACCEPTED_THEN_CHANGED_MESSAGE.format(
+                    see_below=" (see the Conformance Tests Fix below)" if new_conformance_tests_fixes else ""
+                )
+                # Files changed outside the session, so earlier read results are stale.
+                context.tool_result_cache.clear()
+            submit_result: dict = {
+                "call_id": session.pending_submit_call_id,
+                "output": output + agent_tools.full_log_pointer(log_path),
+                "test_output": unittests_issue,
+            }
+            if new_conformance_tests_fixes:
+                submit_result["conformance_tests_fixes"] = new_conformance_tests_fixes
+            tool_results = session.pending_tool_results + [submit_result]
+            session.pending_tool_results, session.pending_submit_call_id = [], None
+            response = self._continue_session(render_context, session.session_id, tool_results)
+        if response is None:
+            console.info("Starting an agent session to fix the unit tests.")
+            # Cached read results point at earlier turns, which a new session does not have.
+            context.tool_result_cache.clear()
+            response = api.agent_start(
+                TASK_TYPE,
+                self._build_task_params(render_context, unittests_issue, conformance_tests_fixes),
+                frid,
+                module_name,
+                render_context.run_state,
+            )
+            session.session_id = response["session_id"]
+        assert session.session_id is not None
+        session.conformance_fixes_handed_off = len(conformance_tests_fixes)
+        context.agent_used_in_this_loop = True
+
+        submitted = False
+        turns = 0
+        while response.get("status") == "tool_calls" and turns < MAX_AGENT_TURNS_PER_ATTEMPT:
+            turns += 1
+            calls = response["calls"]
+            submit_call = next((call for call in calls if call["name"] == SUBMIT_FIX_TOOL), None)
+            tool_results = agent_tools.execute_calls(
+                [call for call in calls if call is not submit_call], render_context
+            )
+            if submit_call is not None:
+                session.pending_submit_call_id = submit_call["id"]
+                session.pending_tool_results = tool_results
+                submitted = True
+                console.info(f"Agent submitted a fix: {submit_call['args'].get('changes_made', '')}")
+                break
+            response = api.agent_continue(session.session_id, tool_results, frid, module_name, render_context.run_state)
+
+        if not submitted:
+            # The session ended without a submission (finished in text, failed, or used up this
+            # attempt's turn budget) — a fresh session is started if the tests still fail.
+            status = response.get("status")
+            if status == "failed":
+                console.warning(f"Agent session failed: {response.get('error', 'unknown error')}")
+            elif status == "tool_calls":
+                console.warning(f"Agent used {MAX_AGENT_TURNS_PER_ATTEMPT} turns without submitting a fix.")
+            session.reset()
+
+        console.print_files(
+            "Files changed while fixing unit tests:",
+            render_context.build_folder,
+            {path: "" for path in sorted(context.changed_files - changed_files_before)},
+            style=console.OUTPUT_STYLE,
+        )
+        return self.SUCCESSFUL_OUTCOME, None
+
+    @staticmethod
+    def _get_conformance_tests_fixes(render_context: RenderContext) -> list[dict]:
+        """Implementation code changes the conformance tests fixer made during this conformance phase, oldest
+        first. Empty outside the conformance phase (the implementation and refactoring unit-test loops)."""
+        conformance_tests_running_context = getattr(render_context, "conformance_tests_running_context", None)
+        if conformance_tests_running_context is None:
+            return []
+        return list(getattr(conformance_tests_running_context, "implementation_code_fixes", None) or [])
+
+    @staticmethod
+    def _continue_session(render_context: RenderContext, session_id: str, tool_results: list[dict]) -> dict | None:
+        """Continue the session; None if the server no longer has it (expired), so a new one is started."""
+        try:
+            return render_context.codeplain_api.agent_continue(
+                session_id,
+                tool_results,
+                render_context.frid_context.frid,
+                render_context.module_name,
+                render_context.run_state,
+            )
+        except requests.exceptions.HTTPError as e:
+            if e.response is None or e.response.status_code != HTTPStatus.NOT_FOUND:
+                raise
+            console.warning(f"Agent session {session_id} has expired; starting a new one.")
+            render_context.unit_tests_agent_session.reset()
+            return None
+
+    @staticmethod
+    def _build_task_params(
+        render_context: RenderContext, unittests_issue: str, conformance_tests_fixes: list[dict]
+    ) -> dict:
+        frid = render_context.frid_context.frid
+        specifications, _ = plain_spec.get_specifications_for_frid(render_context.plain_source_tree, frid)
+        context = render_context.unit_tests_running_context
+        # The files the conformance tests fixes changed are always seeded, so the agent sees what to preserve.
+        conformance_fix_files = {name for fix in conformance_tests_fixes for name in (fix or {}).get("code_diff") or {}}
+        task_params = {
+            "definitions": "\n".join(specifications.get(plain_spec.DEFINITIONS, [])),
+            "non_functional_requirements": "\n".join(specifications.get(plain_spec.NON_FUNCTIONAL_REQUIREMENTS, [])),
+            "functional_requirements": FixUnitTests._functional_requirements_section(render_context, specifications),
+            "linked_resources": render_context.frid_context.linked_resources,
+            "build_folder": render_context.build_folder,
+            "module_name": render_context.module_name,
+            "unittests_script_content": FixUnitTests._read_script(render_context.unittests_script),
+            "unittests_issue": unittests_issue,
+            "unittests_log_path": render_context.script_execution_history.latest_unit_test_output_path,
+            "file_tree": FixUnitTests._file_tree(render_context.build_folder),
+            "relevant_files": FixUnitTests._relevant_files(
+                render_context.build_folder,
+                render_context.frid_context.changed_files | context.changed_files | conformance_fix_files,
+            ),
+        }
+        if conformance_tests_fixes:
+            task_params["conformance_tests_fixes"] = conformance_tests_fixes
+        session = render_context.unit_tests_agent_session
+        if session.previous_session_id:
+            task_params["previous_session_id"] = session.previous_session_id
+        return task_params
+
+    @staticmethod
+    def _file_tree(build_folder: str) -> str:
+        paths = []
+        for root, dirs, files in os.walk(build_folder):
+            dirs[:] = sorted(d for d in dirs if d not in agent_tools.GREP_EXCLUDED_DIRS and not d.startswith("."))
+            paths.extend(os.path.relpath(os.path.join(root, name), build_folder) for name in sorted(files))
+            if len(paths) > MAX_FILE_TREE_ENTRIES:
+                return "\n".join(paths[:MAX_FILE_TREE_ENTRIES]) + "\n... [more files not listed]"
+        return "\n".join(paths)
+
+    @staticmethod
+    def _relevant_files(build_folder: str, file_names: set[str]) -> dict[str, str]:
+        """Contents of the given build-relative files, smallest first, within MAX_RELEVANT_FILES_CHARS."""
+        contents = {}
+        for name in file_names:
+            full_path = os.path.join(build_folder, name)
+            if os.path.isfile(full_path):
+                with open(full_path, "r", encoding="utf-8", errors="replace") as f:
+                    contents[name] = f.read()
+        relevant, total = {}, 0
+        for name in sorted(contents, key=lambda n: (len(contents[n]), n)):
+            if total + len(contents[name]) > MAX_RELEVANT_FILES_CHARS:
+                break
+            relevant[name] = contents[name]
+            total += len(contents[name])
+        return dict(sorted(relevant.items()))
+
+    @staticmethod
+    def _functional_requirements_section(render_context: RenderContext, specifications: dict) -> str:
+        sections = []
+        for module_name, functionalities in render_context.get_required_modules_functionalities().items():
+            sections.append(
+                f"### Module: {module_name} (Already Implemented, for context)\n" + "\n".join(functionalities)
+            )
+        current = specifications.get(plain_spec.FUNCTIONAL_REQUIREMENTS, [])
+        if len(current) > 1:
+            sections.append(
+                f"### Module: {render_context.module_name} (Already Implemented, for context)\n"
+                + "\n".join(current[:-1])
+            )
+        if current:
+            sections.append(f"### Module: {render_context.module_name} (Currently Being Implemented)\n{current[-1]}")
+        return "\n\n".join(sections)
+
+    @staticmethod
+    def _read_script(script: str | None) -> str:
+        if not script:
+            return ""
+        try:
+            with open(file_utils.add_current_path_if_no_path(script), "r", encoding="utf-8") as f:
+                return f.read()
+        except OSError:
+            return ""

{codeplain-0.3.11.dev22 → codeplain-0.3.12}/render_machine/actions/run_unit_tests.py

@@ -16,6 +16,15 @@ class RunUnitTests(BaseAction):
     UNRECOVERABLE_ERROR_OUTCOME = "unrecoverable_error_occurred"
 
     def execute(self, render_context: RenderContext, _previous_action_payload: Any | None):
+        context = render_context.unit_tests_running_context
+        if context.verified_passing:
+            # The fixing agent's own run passed and no file changed since; running again is redundant.
+            context.verified_passing = False
+            console.info("Unit tests already passed in the fixing agent's last run; not running them again.")
+            render_context.script_execution_history.latest_unit_test_output_path = context.verified_passing_log_path
+            render_context.script_execution_history.should_update_script_outputs = True
+            return self.SUCCESSFUL_OUTCOME, None
+
         unittests_script = os.path.normpath(render_context.unittests_script)
 
         console.info(