cellarbrain 0.2.2__tar.gz → 0.2.3__tar.gz

{cellarbrain-0.2.2/src/cellarbrain.egg-info → cellarbrain-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cellarbrain
-Version: 0.2.2
+Version: 0.2.3
 Summary: AI sommelier for your wine cellar — ETL pipeline, DuckDB query layer, Markdown dossiers, and MCP server for wine cellar CSV exports
 Author-email: Urban Busslinger <urbanb@me.com>
 License-Expression: MIT

{cellarbrain-0.2.2 → cellarbrain-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cellarbrain"
-version = "0.2.2"
+version = "0.2.3"
 description = "AI sommelier for your wine cellar — ETL pipeline, DuckDB query layer, Markdown dossiers, and MCP server for wine cellar CSV exports"
 requires-python = ">=3.11"
 license = "MIT"

{cellarbrain-0.2.2 → cellarbrain-0.2.3}/src/cellarbrain/cli.py

@@ -1349,6 +1349,9 @@ def _cmd_ingest(args: argparse.Namespace, settings: Settings) -> None:
 
     if args.once:
         count = poll_once(config, settings, dry_run=args.dry_run)
+        if count < 0:
+            print(f"Failed {-count} batch(es) (ETL error — messages left unprocessed).")
+            sys.exit(1)
         print(f"Processed {count} batch(es).")
         sys.exit(0)
 

{cellarbrain-0.2.2 → cellarbrain-0.2.3}/src/cellarbrain/email_poll/__init__.py

@@ -58,7 +58,8 @@ def poll_once(
 ) -> int:
     """Execute a single poll cycle.
 
-    Returns the number of batches successfully processed (0 or more).
+    Returns the number of batches successfully processed (0 or more),
+    or a negative number indicating how many batches failed ETL.
     """
     from .credentials import resolve_credentials
     from .etl_runner import run_etl
@@ -111,6 +112,7 @@ def poll_once(
             return 0
 
         processed = 0
+        failed = 0
         for batch in batches:
             logger.info(
                 "Batch detected — %s",
@@ -143,9 +145,15 @@ def poll_once(
                 expected_files=config.expected_files,
             )
             if exit_code != 0:
-                logger.error("ETL failed (exit %d)", exit_code)
+                logger.error(
+                    "ETL failed (exit %d) — leaving messages unprocessed (UIDs: %s)",
+                    exit_code,
+                    list(batch.uids),
+                )
+                failed += 1
+                continue
 
-            # Mark as processed (regardless of ETL outcome)
+            # Mark as processed only on successful ETL
             batch_uids = list(batch.uids)
             if config.processed_action == "move":
                 client.move_messages(batch_uids, config.processed_folder)
@@ -161,6 +169,8 @@ def poll_once(
 
             processed += 1
 
+    if failed:
+        return -failed
     return processed
 
 
@@ -195,9 +205,17 @@ class IngestDaemon:
         while True:
             try:
                 count = poll_once(self.config, self.settings, dry_run=dry_run)
-                if count > 0:
+                if count < 0:
+                    logger.error("ETL failed for %d batch(es) — will retry next cycle", -count)
+                    self._current_interval = min(
+                        self._current_interval * 2,
+                        self._max_interval,
+                    )
+                elif count > 0:
                     logger.info("Processed %d batch(es)", count)
-                self._current_interval = self._base_interval
+                    self._current_interval = self._base_interval
+                else:
+                    self._current_interval = self._base_interval
             except ValueError:
                 # Credential / config errors — fatal, stop daemon
                 raise

{cellarbrain-0.2.2 → cellarbrain-0.2.3}/src/cellarbrain/email_poll/etl_runner.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import logging
+import os
 import subprocess
 import sys
 from pathlib import Path
@@ -58,11 +59,13 @@ def run_etl(
 
     logger.info("Running ETL: %s", " ".join(cmd))
     try:
+        env = {**os.environ, "PYTHONUTF8": "1", "PYTHONIOENCODING": "utf-8"}
         result = subprocess.run(
             cmd,
             capture_output=True,
             text=True,
             timeout=_ETL_TIMEOUT,
+            env=env,
         )
         output = result.stdout + result.stderr
         if result.returncode == 0:

{cellarbrain-0.2.2 → cellarbrain-0.2.3}/src/cellarbrain/email_poll/imap.py

@@ -103,14 +103,16 @@ class ImapClient:
             return []
 
         results: list[tuple[EmailMessage, bytes]] = []
-        raw_responses = self._client.fetch(uids, ["RFC822", "INTERNALDATE"])
+        raw_responses = self._client.fetch(uids, ["BODY.PEEK[]", "INTERNALDATE"])
 
         for uid, data in raw_responses.items():
             internal_date = data.get(b"INTERNALDATE")
             if internal_date is None:
                 internal_date = datetime.now(UTC)
 
-            rfc822 = data.get(b"RFC822", b"")
+            rfc822 = data.get(b"BODY[]") or data.get(b"RFC822", b"")
+            if not rfc822:
+                continue
             msg = email.message_from_bytes(rfc822, policy=email.policy.default)
 
             attachments = _extract_attachments(msg)

{cellarbrain-0.2.2 → cellarbrain-0.2.3/src/cellarbrain.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cellarbrain
-Version: 0.2.2
+Version: 0.2.3
 Summary: AI sommelier for your wine cellar — ETL pipeline, DuckDB query layer, Markdown dossiers, and MCP server for wine cellar CSV exports
 Author-email: Urban Busslinger <urbanb@me.com>
 License-Expression: MIT

{cellarbrain-0.2.2 → cellarbrain-0.2.3}/tests/test_email_poll.py

@@ -315,6 +315,26 @@ class TestRunEtl:
         assert "--config" in cmd
         assert str(config) in cmd
 
+    def test_utf8_env_forced(self, tmp_path):
+        """ETL subprocess gets PYTHONUTF8=1 and PYTHONIOENCODING=utf-8 (#002)."""
+        from cellarbrain.email_poll.etl_runner import run_etl
+
+        raw_dir = tmp_path / "raw"
+        raw_dir.mkdir()
+        output_dir = tmp_path / "output"
+        output_dir.mkdir()
+
+        mock_result = MagicMock()
+        mock_result.returncode = 0
+        mock_result.stdout = ""
+        mock_result.stderr = ""
+
+        with patch("cellarbrain.email_poll.etl_runner.subprocess.run", return_value=mock_result) as mock_run:
+            run_etl(raw_dir, output_dir)
+        env = mock_run.call_args[1]["env"]
+        assert env["PYTHONUTF8"] == "1"
+        assert env["PYTHONIOENCODING"] == "utf-8"
+
 
 # ---------------------------------------------------------------------------
 # TestStateFile
@@ -341,3 +361,203 @@ class TestStateFile:
         loaded = _load_state(tmp_path)
         assert loaded["processed_uids"] == [1, 2, 3]
         assert loaded["last_batch"] == "260428-1435"
+
+
+# ---------------------------------------------------------------------------
+# TestImapFetchMessages
+# ---------------------------------------------------------------------------
+
+
+def _make_mime_message(filename: str, payload: bytes) -> bytes:
+    """Build a minimal MIME message with one attachment."""
+    import email.mime.application
+    import email.mime.multipart
+
+    msg = email.mime.multipart.MIMEMultipart()
+    msg["Subject"] = "[VinoCell] CSV file"
+    att = email.mime.application.MIMEApplication(payload, Name=filename)
+    att.add_header("Content-Disposition", "attachment", filename=filename)
+    msg.attach(att)
+    return msg.as_bytes()
+
+
+class TestImapFetchMessages:
+    """Regression tests for iCloud IMAP BODY[] key issue (#001)."""
+
+    def test_body_bracket_key(self):
+        """iCloud returns body under b'BODY[]' — should still work."""
+        from cellarbrain.email_poll.imap import ImapClient
+
+        mime_bytes = _make_mime_message("export-wines.csv", b"wine,data")
+
+        mock_client = MagicMock()
+        mock_client.fetch.return_value = {
+            48: {
+                b"BODY[]": mime_bytes,
+                b"INTERNALDATE": datetime(2026, 5, 1, 10, 0),
+            }
+        }
+
+        imap = ImapClient.__new__(ImapClient)
+        imap._client = mock_client
+
+        results = imap.fetch_messages([48], ["export-wines.csv"])
+        assert len(results) == 1
+        em, data = results[0]
+        assert em.uid == 48
+        assert em.filename == "export-wines.csv"
+        assert data == b"wine,data"
+        mock_client.fetch.assert_called_once_with([48], ["BODY.PEEK[]", "INTERNALDATE"])
+
+    def test_rfc822_key_fallback(self):
+        """Servers returning b'RFC822' should still work (fallback)."""
+        from cellarbrain.email_poll.imap import ImapClient
+
+        mime_bytes = _make_mime_message("export-wines.csv", b"wine,data")
+
+        mock_client = MagicMock()
+        mock_client.fetch.return_value = {
+            99: {
+                b"RFC822": mime_bytes,
+                b"INTERNALDATE": datetime(2026, 5, 1, 10, 0),
+            }
+        }
+
+        imap = ImapClient.__new__(ImapClient)
+        imap._client = mock_client
+
+        results = imap.fetch_messages([99], ["export-wines.csv"])
+        assert len(results) == 1
+        em, data = results[0]
+        assert em.uid == 99
+        assert em.filename == "export-wines.csv"
+
+    def test_empty_body_skipped(self):
+        """Messages with no body data should be skipped."""
+        from cellarbrain.email_poll.imap import ImapClient
+
+        mock_client = MagicMock()
+        mock_client.fetch.return_value = {
+            50: {
+                b"INTERNALDATE": datetime(2026, 5, 1, 10, 0),
+            }
+        }
+
+        imap = ImapClient.__new__(ImapClient)
+        imap._client = mock_client
+
+        results = imap.fetch_messages([50], ["export-wines.csv"])
+        assert len(results) == 0
+
+
+# ---------------------------------------------------------------------------
+# TestPollOnceEtlFailure
+# ---------------------------------------------------------------------------
+
+
+class TestPollOnceEtlFailure:
+    """Verify poll_once does NOT mark messages processed on ETL failure (#003)."""
+
+    def _make_settings(self, tmp_path):
+        """Build minimal Settings and IngestConfig for testing."""
+        from cellarbrain.settings import IngestConfig
+
+        raw_dir = tmp_path / "raw"
+        raw_dir.mkdir()
+        output_dir = tmp_path / "output"
+        output_dir.mkdir()
+
+        config = IngestConfig()
+
+        settings = MagicMock()
+        settings.paths.raw_dir = str(raw_dir)
+        settings.paths.data_dir = str(output_dir)
+        settings.config_source = None
+
+        return config, settings, raw_dir
+
+    def test_etl_failure_leaves_messages_unprocessed(self, tmp_path):
+        """On ETL failure, messages should NOT be marked as read."""
+        from cellarbrain.email_poll import poll_once
+        from cellarbrain.email_poll.grouping import Batch
+
+        config, settings, raw_dir = self._make_settings(tmp_path)
+
+        t0 = datetime(2026, 5, 1, 10, 0)
+
+        mock_imap = MagicMock()
+        mock_imap.search_unseen.return_value = [48, 49, 50]
+        mock_imap.fetch_messages.return_value = [
+            (EmailMessage(uid=48, date=t0, filename="export-wines.csv", size=100), b"wines"),
+            (EmailMessage(uid=49, date=t0, filename="export-bottles-stored.csv", size=100), b"bottles"),
+            (EmailMessage(uid=50, date=t0, filename="export-bottles-gone.csv", size=100), b"gone"),
+        ]
+        mock_imap.__enter__ = MagicMock(return_value=mock_imap)
+        mock_imap.__exit__ = MagicMock(return_value=False)
+
+        batch = Batch(
+            messages=[
+                EmailMessage(uid=48, date=t0, filename="export-wines.csv", size=100),
+                EmailMessage(uid=49, date=t0, filename="export-bottles-stored.csv", size=100),
+                EmailMessage(uid=50, date=t0, filename="export-bottles-gone.csv", size=100),
+            ]
+        )
+
+        with (
+            patch("cellarbrain.email_poll.imap.ImapClient", return_value=mock_imap),
+            patch("cellarbrain.email_poll.poll_once.__module__", "cellarbrain.email_poll"),
+            patch("cellarbrain.email_poll.credentials.resolve_credentials", return_value=("u", "p")),
+            patch("cellarbrain.email_poll.grouping.group_messages", return_value=[batch]),
+            patch("cellarbrain.email_poll.placement.place_batch", return_value=raw_dir / "260501-1000"),
+            patch("cellarbrain.email_poll.etl_runner.run_etl", return_value=(1, "Error: bad data")),
+        ):
+            result = poll_once(config, settings)
+
+        # ETL failed → should NOT mark messages
+        mock_imap.mark_seen.assert_not_called()
+        mock_imap.move_messages.assert_not_called()
+        # Return value should be negative (indicating failure)
+        assert result == -1
+
+    def test_etl_success_marks_messages(self, tmp_path):
+        """On ETL success, messages should be marked as read."""
+        from cellarbrain.email_poll import poll_once
+        from cellarbrain.email_poll.grouping import Batch
+
+        config, settings, raw_dir = self._make_settings(tmp_path)
+
+        t0 = datetime(2026, 5, 1, 10, 0)
+
+        mock_imap = MagicMock()
+        mock_imap.search_unseen.return_value = [48, 49, 50]
+        mock_imap.fetch_messages.return_value = [
+            (EmailMessage(uid=48, date=t0, filename="export-wines.csv", size=100), b"wines"),
+            (EmailMessage(uid=49, date=t0, filename="export-bottles-stored.csv", size=100), b"bottles"),
+            (EmailMessage(uid=50, date=t0, filename="export-bottles-gone.csv", size=100), b"gone"),
+        ]
+        mock_imap.__enter__ = MagicMock(return_value=mock_imap)
+        mock_imap.__exit__ = MagicMock(return_value=False)
+
+        batch = Batch(
+            messages=[
+                EmailMessage(uid=48, date=t0, filename="export-wines.csv", size=100),
+                EmailMessage(uid=49, date=t0, filename="export-bottles-stored.csv", size=100),
+                EmailMessage(uid=50, date=t0, filename="export-bottles-gone.csv", size=100),
+            ]
+        )
+
+        snapshot_dir = raw_dir / "260501-1000"
+        snapshot_dir.mkdir(parents=True)
+
+        with (
+            patch("cellarbrain.email_poll.imap.ImapClient", return_value=mock_imap),
+            patch("cellarbrain.email_poll.credentials.resolve_credentials", return_value=("u", "p")),
+            patch("cellarbrain.email_poll.grouping.group_messages", return_value=[batch]),
+            patch("cellarbrain.email_poll.placement.place_batch", return_value=snapshot_dir),
+            patch("cellarbrain.email_poll.etl_runner.run_etl", return_value=(0, "OK")),
+        ):
+            result = poll_once(config, settings)
+
+        # ETL succeeded → messages should be marked
+        mock_imap.mark_seen.assert_called_once_with([48, 49, 50])
+        assert result == 1

{cellarbrain-0.2.2 → cellarbrain-0.2.3}/tests/test_price.py

@@ -2,7 +2,7 @@
 
 from __future__ import annotations
 
-from datetime import date, datetime
+from datetime import date, datetime, timedelta
 from decimal import Decimal
 
 import pytest
@@ -414,7 +414,7 @@ class TestWishlistAlerts:
             "price": Decimal("50.00"),
             "currency": "CHF",
             "in_stock": True,
-            "observed_at": datetime(2026, 4, 7, 12, 0),
+            "observed_at": datetime.now() - timedelta(days=5),
             "observation_source": "agent",
         }
         log_price(price_dir, obs)
@@ -432,7 +432,7 @@ class TestWishlistAlerts:
             "price": Decimal("100.00"),
             "currency": "CHF",
             "in_stock": True,
-            "observed_at": datetime(2026, 3, 1, 12, 0),
+            "observed_at": datetime.now() - timedelta(days=20),
             "observation_source": "agent",
         }
         log_price(price_dir, old_obs)
@@ -444,7 +444,7 @@ class TestWishlistAlerts:
             "price": Decimal("80.00"),
             "currency": "CHF",
             "in_stock": True,
-            "observed_at": datetime(2026, 4, 5, 12, 0),
+            "observed_at": datetime.now() - timedelta(days=5),
             "observation_source": "agent",
         }
         log_price(price_dir, new_obs)
@@ -461,7 +461,7 @@ class TestWishlistAlerts:
             "price": Decimal("100.00"),
             "currency": "CHF",
             "in_stock": True,
-            "observed_at": datetime(2026, 3, 1, 12, 0),
+            "observed_at": datetime.now() - timedelta(days=20),
             "observation_source": "agent",
         }
         log_price(price_dir, old_obs)
@@ -473,7 +473,7 @@ class TestWishlistAlerts:
             "price": Decimal("95.00"),
             "currency": "CHF",
             "in_stock": True,
-            "observed_at": datetime(2026, 4, 5, 12, 0),
+            "observed_at": datetime.now() - timedelta(days=5),
             "observation_source": "agent",
         }
         log_price(price_dir, new_obs)
@@ -489,7 +489,7 @@ class TestWishlistAlerts:
             "price": Decimal("60.00"),
             "currency": "CHF",
             "in_stock": False,
-            "observed_at": datetime(2026, 3, 1, 12, 0),
+            "observed_at": datetime.now() - timedelta(days=20),
             "observation_source": "agent",
         }
         log_price(price_dir, out_obs)
@@ -501,7 +501,7 @@ class TestWishlistAlerts:
             "price": Decimal("60.00"),
             "currency": "CHF",
             "in_stock": True,
-            "observed_at": datetime(2026, 4, 5, 12, 0),
+            "observed_at": datetime.now() - timedelta(days=5),
             "observation_source": "agent",
         }
         log_price(price_dir, in_obs)
@@ -518,7 +518,7 @@ class TestWishlistAlerts:
             "price": Decimal("35.00"),
             "currency": "CHF",
             "in_stock": True,
-            "observed_at": datetime(2026, 4, 7, 12, 0),
+            "observed_at": datetime.now() - timedelta(days=5),
             "observation_source": "agent",
             "notes": "En primeur release",
         }