codeplain 0.3.10.dev2__tar.gz → 0.3.10.dev4__tar.gz

codeplain-0.3.10.dev4/.github/dependabot.yml

@@ -0,0 +1,28 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 3
+    commit-message:
+      prefix: "dependabot"
+    labels:
+      - "dependencies"
+
+  # Keeps the SHA-pinned actions in publish-to-pypi.yml patched.
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 5
+    commit-message:
+      prefix: "ci"
+    labels:
+      - "dependencies"
+      - "github-actions"
+    groups:
+      actions-minor-and-patch:
+        update-types:
+          - "minor"
+          - "patch"

{codeplain-0.3.10.dev2 → codeplain-0.3.10.dev4}/.github/workflows/e2e.yml

@@ -11,19 +11,19 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: '3.12'
 
       - run: pip install pytest
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
 
       - name: Build E2E image (cached)
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5
         with:
           context: .
           file: tests/e2e/Dockerfile
@@ -39,7 +39,7 @@ jobs:
 
       - name: Upload generated outputs on failure
         if: failure()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: e2e-outputs-linux
           path: /tmp/pytest-of-runner/**/container_work*/**
@@ -48,9 +48,9 @@ jobs:
     runs-on: windows-latest
     timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: '3.12'
 
@@ -81,8 +81,10 @@ jobs:
 
       - name: Send failure notification to Slack
         if: ${{ env.SLACK_WEBHOOK_URL != '' }}
-        uses: slackapi/slack-github-action@v1.24.0
+        uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
         with:
+          webhook: ${{ env.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
           payload: |
             {
               "text": ${{ toJSON(format('[Codeplain Client] E2E tests FAILED on {0}. See <{1}/{2}/actions/runs/{3}|the run log>.', steps.platforms.outputs.failed, github.server_url, github.repository, github.run_id)) }}

{codeplain-0.3.10.dev2 → codeplain-0.3.10.dev4}/.github/workflows/lint-and-test.yml

@@ -12,13 +12,13 @@ jobs:
     name: Black Formatting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Cache pip
-        uses: actions/cache@v4
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
@@ -33,13 +33,13 @@ jobs:
     name: Import Sorting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Cache pip
-        uses: actions/cache@v4
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
@@ -54,13 +54,13 @@ jobs:
     name: Flake8 Linting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Cache pip
-        uses: actions/cache@v4
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
@@ -75,13 +75,13 @@ jobs:
     name: MyPy Type Checking
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Cache pip
-        uses: actions/cache@v4
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
@@ -96,9 +96,9 @@ jobs:
     name: Run Tests
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Configure git for tests
@@ -107,7 +107,7 @@ jobs:
           git config --global user.name "Test Runner"
           git config --global init.defaultBranch main
       - name: Cache pip
-        uses: actions/cache@v4
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
@@ -122,5 +122,7 @@ jobs:
           coverage run -m pytest tests/ -v
           coverage xml
           coverage report
-      - name: Upload coverage reports
-        uses: codecov/codecov-action@v3
+      # Disabled: v3's Node runtime is gone from the runners. Re-enable with a
+      # current major version, which renamed several inputs.
+      # - name: Upload coverage reports
+      #   uses: codecov/codecov-action@v3

{codeplain-0.3.10.dev2 → codeplain-0.3.10.dev4}/.github/workflows/lint-powershell.yml

@@ -16,7 +16,7 @@ jobs:
     name: Parse under Windows PowerShell 5.1
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       # The install scripts are fetched over HTTP and executed on stock Windows,
       # where powershell.exe (5.1) is the default shell. 5.1 reads BOM-less files
@@ -96,7 +96,7 @@ jobs:
     name: Parse under PowerShell 7
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Parse each script with 7
         shell: pwsh

{codeplain-0.3.10.dev2 → codeplain-0.3.10.dev4}/.github/workflows/nofity-slack-on-main-merge.yml

@@ -12,11 +12,11 @@ jobs:
     if: github.event.pull_request.merged == true
     steps:
       - name: Send notification to Slack
-        uses: slackapi/slack-github-action@v1.24.0
+        uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0
         with:
+          webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+          webhook-type: incoming-webhook
           payload: |
             {
               "text": ${{ toJSON(format('[Codeplain Client] PR <{0}|#{1}: {2}> was merged to main by `{3}`.', github.event.pull_request.html_url, github.event.pull_request.number, github.event.pull_request.title, github.actor)) }}
             }
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

{codeplain-0.3.10.dev2 → codeplain-0.3.10.dev4}/.github/workflows/publish-install-script.yml

@@ -18,7 +18,7 @@ jobs:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       - name: Upload install scripts to R2
         env:

{codeplain-0.3.10.dev2 → codeplain-0.3.10.dev4}/.github/workflows/publish-to-pypi.yml

@@ -19,7 +19,7 @@ jobs:
       SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           # hatch-vcs derives the version from git tags, so we need full history.
           fetch-depth: 0
@@ -46,7 +46,7 @@ jobs:
           ls -l dist/
 
       - name: Store the distribution packages
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: python-package-distributions
           path: dist/
@@ -81,13 +81,13 @@ jobs:
 
     steps:
       - name: Download distribution packages
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: python-package-distributions
           path: dist/
 
       - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
 
       - name: Notify Slack (success)
         if: ${{ success() && env.SLACK_WEBHOOK_URL != '' }}
@@ -115,3 +115,45 @@ jobs:
           curl -s -X POST "$SLACK_WEBHOOK_URL" \
             -H 'Content-type: application/json' \
             --data "{\"text\":\"*Codeplain Client*: Publishing \`${{ github.ref_name }}\` to PyPI FAILED :x:. See <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|the run log>.\"}"
+
+  # Release pipelines to be turned on only after the rest of the infrastructure
+  # is ready.
+  #
+  # # Runs only after the package is on PyPI. Kept as its own job so the publish job
+  # # keeps its minimal permissions and no checkout.
+  # linear-release:
+  #   name: Record release in Linear
+  #   needs: publish-to-pypi
+  #   runs-on: ubuntu-latest
+  #
+  #   permissions:
+  #     contents: read
+  #
+  #   env:
+  #     SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  #
+  #   steps:
+  #     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+  #       with:
+  #         # The scan needs full history.
+  #         fetch-depth: 0
+  #
+  #     # Non-fatal: the package is already published.
+  #     - name: Record release in Linear
+  #       id: linear
+  #       continue-on-error: true
+  #       uses: linear/linear-release-action@0a25abab892a91062ebf42260dbb2ce6277aa205 # v0.16.0
+  #       with:
+  #         access_key: ${{ secrets.LINEAR_ACCESS_KEY }}
+  #         version: ${{ github.ref_name }}
+  #         # Pin the CLI the action downloads, not just the action.
+  #         cli_version: v0.16.0
+  #
+  #     # continue-on-error keeps the job green, so this checks the step's own
+  #     # outcome — `failure()` would never fire.
+  #     - name: Notify Slack (release not recorded)
+  #       if: ${{ steps.linear.outcome == 'failure' && env.SLACK_WEBHOOK_URL != '' }}
+  #       run: |
+  #         curl -s -X POST "$SLACK_WEBHOOK_URL" \
+  #           -H 'Content-type: application/json' \
+  #           --data "{\"text\":\"*Codeplain Client*: \`${{ github.ref_name }}\` is on PyPI, but recording it in Linear FAILED :warning:. See <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|the run log>.\"}"

{codeplain-0.3.10.dev2 → codeplain-0.3.10.dev4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: codeplain
-Version: 0.3.10.dev2
+Version: 0.3.10.dev4
 Summary: Transform plain language specifications into working code
 License-File: LICENSE
 Classifier: Environment :: Console

{codeplain-0.3.10.dev2 → codeplain-0.3.10.dev4}/codeplain_REST_api.py

@@ -254,7 +254,7 @@ class CodeplainAPI:
 
         return self.post_request(endpoint_url, headers, payload, run_state)
 
-    def create_conformance_test_memory(
+    def distill_conformance_test_memory(
         self,
         frid,
         plain_source_tree,
@@ -263,15 +263,10 @@ class CodeplainAPI:
         memory_files_content,
         module_name,
         required_modules,
-        code_diff,
-        conformance_tests_files,
-        acceptance_tests,
-        conformance_tests_issue,
-        conformance_tests_folder_name,
-        previous_conformance_tests_issue_old,
+        fix_journals,
         run_state: RunState,
     ):
-        endpoint_url = f"{self.api_url}/create_conformance_test_memory"
+        endpoint_url = f"{self.api_url}/distill_conformance_test_memory"
         headers = {"X-API-Key": self.api_key, "Content-Type": "application/json"}
 
         payload = {
@@ -282,12 +277,7 @@ class CodeplainAPI:
             "memory_files_content": memory_files_content,
             "module_name": module_name,
             "required_modules": required_modules,
-            "code_diff": code_diff,
-            "conformance_tests_files": conformance_tests_files,
-            "acceptance_tests": acceptance_tests,
-            "conformance_tests_issue": conformance_tests_issue,
-            "conformance_tests_folder_name": conformance_tests_folder_name,
-            "previous_conformance_tests_issue": previous_conformance_tests_issue_old,
+            "fix_journals": fix_journals,
         }
 
         return self.post_request(endpoint_url, headers, payload, run_state)
@@ -381,6 +371,7 @@ class CodeplainAPI:
         conformance_tests_folder_name,
         current_testing_frid_high_level_implementation_plan: Optional[str],
         conflicting_requirements_count: int,
+        fix_attempts_journal: list,
         run_state: RunState,
     ):
         endpoint_url = f"{self.api_url}/fix_conformance_tests_issue"
@@ -403,6 +394,7 @@ class CodeplainAPI:
             "conformance_tests_folder_name": conformance_tests_folder_name,
             "current_testing_frid_high_level_implementation_plan": current_testing_frid_high_level_implementation_plan,
             "conflicting_requirements_count": conflicting_requirements_count,
+            "fix_attempts_journal": fix_attempts_journal,
         }
 
         if acceptance_tests is not None:

codeplain-0.3.10.dev4/conformance_fix_journal.py

@@ -0,0 +1,207 @@
+"""Deterministic journal of conformance tests fix attempts.
+
+While a functionality's conformance tests are failing, every fix attempt is recorded here: the
+issue that triggered it, what the fixer said it tried, which files it changed and whether the tests
+passed afterwards. A digest of the journal is sent back with every fix request so the fixer does
+not repeat attempts that already failed, and once the functionality's conformance tests all pass
+the journal is distilled into durable memories and deleted.
+
+Each entry carries the issue in two forms: the raw script output (kept untruncated, on disk only)
+and the prepared form the server built for the fix prompt (trimmed, and summarized or truncated
+when oversized) - the issue exactly the way the fixer saw it. The prepared form is what travels in
+the fix digest and the distillation payload; the raw form exists for humans debugging a render.
+An attempt's outcome text is not stored twice: the output produced after an attempt is the raw
+issue of the next attempt.
+
+Journals are keyed by the (module, frid) whose tests are being fixed - during the regression sweep
+that can be a previously implemented functionality, possibly from a required module.
+"""
+
+import hashlib
+import json
+import os
+import shutil
+
+from plain2code_console import console
+
+CONFORMANCE_TEST_JOURNAL_SUBFOLDER = "conformance_test_journal"
+
+# Cap for the issue excerpts included in payloads (the per-attempt issue in the fix digest, and the
+# initial issue in the distillation payload). The server-prepared issue is already bounded (LLM
+# summary or ~10k-char truncation), so this only trims the rare oversized case - it must not
+# routinely cut into summaries the server already sized for a prompt.
+MAX_ISSUE_EXCERPT_CHARS = 8000
+
+# The keys of a journal entry that make up the digest sent to the fixer and the distiller.
+DIGEST_KEYS = ["attempt", "hypothesis", "approach", "target", "files_changed", "duplicate_of", "result"]
+
+
+def _trim_issue(issue: str | None, max_chars: int) -> str | None:
+    if issue is None or len(issue) <= max_chars:
+        return issue
+    return f"(TRUNCATED - showing the last {max_chars} characters)\n" + issue[-max_chars:]
+
+
+def normalized_diff_hash(diff_files: dict[str, str] | None) -> str | None:
+    """Hashes a fix's diff so that essentially identical retries can be detected.
+
+    Whitespace-only differences are ignored - a fix that reproduces an earlier failed change with
+    different formatting is still the same attempt.
+    """
+    if not diff_files:
+        return None
+
+    normalized_parts = []
+    for file_name in sorted(diff_files):
+        content_lines = [line.strip() for line in (diff_files[file_name] or "").splitlines() if line.strip()]
+        normalized_parts.append(file_name + "\n" + "\n".join(content_lines))
+
+    return hashlib.sha256("\n".join(normalized_parts).encode("utf-8")).hexdigest()
+
+
+class FixAttemptJournal:
+    """File-backed journal, one JSONL file per (module, frid) whose conformance tests are fixed."""
+
+    def __init__(self, memory_folder: str):
+        self.journal_folder = os.path.join(memory_folder, CONFORMANCE_TEST_JOURNAL_SUBFOLDER)
+
+    def _journal_path(self, module_name: str, frid: str) -> str:
+        return os.path.join(self.journal_folder, f"{module_name}__{frid}.jsonl")
+
+    def _read_entries(self, module_name: str, frid: str) -> list[dict]:
+        journal_path = self._journal_path(module_name, frid)
+        if not os.path.exists(journal_path):
+            return []
+
+        entries = []
+        with open(journal_path, "r", encoding="utf-8") as journal_file:
+            for line in journal_file:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    entries.append(json.loads(line))
+                except json.JSONDecodeError:
+                    console.error(f"Skipping malformed journal entry in {journal_path}.")
+
+        return entries
+
+    def _write_entries(self, module_name: str, frid: str, entries: list[dict]):
+        os.makedirs(self.journal_folder, exist_ok=True)
+        with open(self._journal_path(module_name, frid), "w", encoding="utf-8") as journal_file:
+            for entry in entries:
+                journal_file.write(json.dumps(entry) + "\n")
+
+    def open_attempt(self, module_name: str, frid: str, attempt_no: int, issue_before: str):
+        """Starts a new journal entry for a fix attempt that is about to be made.
+
+        The raw issue is stored untruncated - it never travels in a payload, only its prepared form
+        (recorded later by record_fix) does.
+        """
+        entries = self._read_entries(module_name, frid)
+        entries.append({"attempt": attempt_no, "issue_before_raw": issue_before})
+        self._write_entries(module_name, frid, entries)
+
+    def record_fix(
+        self,
+        module_name: str,
+        frid: str,
+        fix_attempt_summary: dict | None,
+        files_changed: list[str],
+        target: str,
+        diff_files: dict[str, str] | None,
+        prepared_issue: str | None,
+    ):
+        """Completes the open attempt with what the fix actually did.
+
+        The prepared issue is the form the server built for the fix prompt - the issue exactly the
+        way the fixer saw it. The diff hash marks the attempt as a duplicate when an earlier attempt
+        in the same journal produced essentially the same change - the strongest signal that the
+        fixer is going in circles.
+        """
+        entries = self._read_entries(module_name, frid)
+        if not entries:
+            console.error("Cannot record a fix - no journal entry is open.")
+            return
+
+        entry = entries[-1]
+        if isinstance(fix_attempt_summary, dict):
+            entry["hypothesis"] = fix_attempt_summary.get("hypothesis")
+            entry["approach"] = fix_attempt_summary.get("approach")
+        if prepared_issue:
+            entry["issue_before"] = prepared_issue
+        entry["files_changed"] = sorted(files_changed)
+        entry["target"] = target
+
+        diff_hash = normalized_diff_hash(diff_files)
+        entry["diff_hash"] = diff_hash
+        if diff_hash is not None:
+            for earlier_entry in entries[:-1]:
+                if earlier_entry.get("diff_hash") == diff_hash:
+                    entry["duplicate_of"] = earlier_entry["attempt"]
+                    break
+
+        self._write_entries(module_name, frid, entries)
+
+    def record_result(self, module_name: str, frid: str, passed: bool):
+        """Records whether the conformance tests passed after the last fix attempt was applied.
+
+        A failed run's output is not stored here - it becomes the raw issue of the next attempt.
+        Does nothing when no attempt is pending: the first test run of a functionality has no fix
+        attempt behind it.
+        """
+        entries = self._read_entries(module_name, frid)
+        if not entries or "result" in entries[-1]:
+            return
+
+        entries[-1]["result"] = "conformance tests passed" if passed else "conformance tests still failed"
+        self._write_entries(module_name, frid, entries)
+
+    def build_digest(self, module_name: str, frid: str) -> list[dict]:
+        """Builds the compact attempt history sent with a fix request.
+
+        Only attempts whose fix has been recorded are included - the entry just opened for the
+        attempt being made (and any dangling entry an interrupted render left behind) carries no
+        information for the fixer. Each included attempt carries the prepared issue it addressed,
+        so the fixer can see how the issue evolved across attempts. Raw outputs never travel - the
+        current issue is sent with the request separately.
+        """
+        return [self._digest_entry(entry) for entry in self._read_entries(module_name, frid) if "target" in entry]
+
+    @staticmethod
+    def _digest_entry(entry: dict) -> dict:
+        digest_entry = {key: entry[key] for key in DIGEST_KEYS if key in entry}
+        if entry.get("issue_before"):
+            digest_entry["issue"] = _trim_issue(entry["issue_before"], MAX_ISSUE_EXCERPT_CHARS)
+        return digest_entry
+
+    def collect_all(self) -> list[dict]:
+        """Collects the digests of every journal, keyed by the tested module and frid, for distillation."""
+        if not os.path.exists(self.journal_folder):
+            return []
+
+        journals = []
+        for file_name in sorted(os.listdir(self.journal_folder)):
+            if not file_name.endswith(".jsonl") or "__" not in file_name:
+                continue
+            module_name, _, frid = file_name[: -len(".jsonl")].rpartition("__")
+            entries = [entry for entry in self._read_entries(module_name, frid) if "target" in entry]
+            if not entries:
+                continue
+            initial_issue = entries[0].get("issue_before") or _trim_issue(
+                entries[0].get("issue_before_raw"), MAX_ISSUE_EXCERPT_CHARS
+            )
+            journal = {
+                "module": module_name,
+                "frid": frid,
+                "initial_issue": initial_issue,
+                "attempts": [self._digest_entry(entry) for entry in entries],
+            }
+            journals.append(journal)
+
+        return journals
+
+    def clear_all(self):
+        """Deletes every journal. Called after the journals have been distilled into memories."""
+        if os.path.exists(self.journal_folder):
+            shutil.rmtree(self.journal_folder)

codeplain-0.3.10.dev4/memory_management.py

@@ -0,0 +1,52 @@
+import os
+
+import file_utils
+from conformance_fix_journal import FixAttemptJournal
+from plain2code_console import console
+
+CONFORMANCE_TEST_MEMORY_SUBFOLDER = "conformance_test_memory"
+
+
+class MemoryManager:
+    """Manages a module's memory store at <build>/<module>/.memory.
+
+    The distilled memories are inherited by the next module in the chain (copied over when its
+    render starts, like the code repo), so learnings travel across modules.
+    """
+
+    @staticmethod
+    def fetch_memory_files(memory_folder: str) -> tuple[list[str], dict[str, str]]:
+        """Fetch memory files from memory_folder/conformance_test_memory."""
+        memory_path = os.path.join(memory_folder, CONFORMANCE_TEST_MEMORY_SUBFOLDER)
+        if not os.path.exists(memory_path):
+            return [], {}
+        memory_files = file_utils.list_all_text_files(memory_path)
+        memory_files_content = file_utils.get_existing_files_content(memory_path, memory_files)
+        console.debug(f"Loaded {len(memory_files_content)} memory files.")
+        return memory_files, memory_files_content
+
+    def __init__(self, codeplain_api, memory_folder: str):
+        self.codeplain_api = codeplain_api
+        self.memory_folder = memory_folder
+        self.journal = FixAttemptJournal(memory_folder)
+
+    def store_memory_files(self, response_files: dict[str, str]):
+        """Stores distilled memory files, applying deletions (None content) to existing memories."""
+        memory_path = os.path.join(self.memory_folder, CONFORMANCE_TEST_MEMORY_SUBFOLDER)
+        memory_files, _ = MemoryManager.fetch_memory_files(self.memory_folder)
+        file_utils.store_response_files(memory_path, response_files, memory_files)
+
+    @staticmethod
+    def inherit_memories(source_memory_folder: str, destination_memory_folder: str):
+        """Copies the distilled memories of a previously rendered module into this module's store.
+
+        Only the distilled memories travel - fix attempt journals are transient state of one
+        module's render. A previous module without memories (e.g. shipped as an archive that
+        carries only code) is a no-op.
+        """
+        source_path = os.path.join(source_memory_folder, CONFORMANCE_TEST_MEMORY_SUBFOLDER)
+        if not os.path.isdir(source_path):
+            return
+        file_utils.copy_folder_content(
+            source_path, os.path.join(destination_memory_folder, CONFORMANCE_TEST_MEMORY_SUBFOLDER)
+        )