sqlite-export-for-ynab 2.9.2__tar.gz → 2.9.4__tar.gz

{sqlite_export_for_ynab-2.9.2/sqlite_export_for_ynab.egg-info → sqlite_export_for_ynab-2.9.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlite_export_for_ynab
-Version: 2.9.2
+Version: 2.9.4
 Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
 Author-email: Max R <mxr@users.noreply.github.com>
 License-Expression: MIT
@@ -16,7 +16,7 @@ License-File: LICENSE
 Requires-Dist: aiohttp>=3
 Requires-Dist: aiopathlib
 Requires-Dist: aiosqlite
-Requires-Dist: asyncio-for-ynab~=1.84.0
+Requires-Dist: asyncio-for-ynab~=1.85.0
 Requires-Dist: fasteners
 Requires-Dist: rich>=10
 Requires-Dist: tenacity

{sqlite_export_for_ynab-2.9.2 → sqlite_export_for_ynab-2.9.4}/pyproject.toml

@@ -22,7 +22,7 @@ dependencies = [
   "aiohttp>=3",
   "aiopathlib",
   "aiosqlite",
-  "asyncio-for-ynab~=1.84.0",
+  "asyncio-for-ynab~=1.85.0",
   "fasteners",
   "rich>=10",
   "tenacity",
@@ -34,7 +34,7 @@ license-files = ["LICENSE"]
 name = "sqlite_export_for_ynab"
 readme = "README.md"
 requires-python = ">=3.12"
-version = "2.9.2"
+version = "2.9.4"
 
 [project.scripts]
 sqlite-export-for-ynab = "sqlite_export_for_ynab._main:main"
@@ -131,3 +131,6 @@ commands = [
   ["coverage", "report"],
 ]
 dependency_groups = ["dev"]
+
+[tool.ty.rules]
+all = "error"

{sqlite_export_for_ynab-2.9.2 → sqlite_export_for_ynab-2.9.4}/sqlite_export_for_ynab/_main.py

@@ -7,6 +7,8 @@ from contextlib import asynccontextmanager
 from contextlib import contextmanager
 from dataclasses import dataclass
 from dataclasses import fields
+from datetime import date
+from datetime import timedelta
 from importlib import resources
 from importlib.metadata import version
 from itertools import batched
@@ -14,6 +16,7 @@ from pathlib import Path
 from typing import Any
 from typing import Literal
 from typing import overload
+from typing import override
 from typing import TYPE_CHECKING
 
 import aiosqlite
@@ -34,6 +37,8 @@ from asyncio_for_ynab import ScheduledTransactionDetail
 from asyncio_for_ynab import ScheduledTransactionsApi
 from asyncio_for_ynab import TransactionDetail
 from asyncio_for_ynab import TransactionsApi
+from asyncio_for_ynab import TransactionsResponse
+from asyncio_for_ynab import TransactionsResponseData
 from rich.progress import BarColumn
 from rich.progress import Progress
 from rich.progress import TaskID
@@ -67,6 +72,7 @@ except ImportError:  # pragma: no cover
             self.separator = separator
             super().__init__(table_column=table_column)
 
+        @override
         def render(self, task: Task) -> Text:
             """Show completed/total."""
             completed = int(task.completed)
@@ -217,8 +223,11 @@ async def _context(
     progress = Progress(*_PROGRESS_COLUMNS, disable=quiet)
     lock_path = db.parent / f"{db.name}.lock"
     lock = fasteners.InterProcessLock(lock_path)
+    # ApiClient's constructor builds an SSL context (loads certs from disk),
+    # so build it off the event loop thread.
+    api_client = await asyncio.to_thread(ApiClient, configuration)
     async with (
-        ApiClient(configuration) as api_client,
+        api_client,
         aiosqlite.connect(db) as con,
     ):
         con.row_factory = aiosqlite.Row
@@ -290,9 +299,7 @@ async def sync(
                 "Plan Data", total=len(plans) * len(_ENDPOINTS)
             )
 
-            all_data = await _get_all_ynab(
-                context, [str(plan.id) for plan in plans], lkos, task
-            )
+            all_data = await _get_all_ynab(context, plans, lkos, task)
 
         _print("Done", quiet=quiet)
 
@@ -630,24 +637,30 @@ async def _get_plan_summaries(api_client: ApiClient) -> list[PlanSummary]:
 
 
 async def _get_all_ynab(
-    context: _Context, plan_ids: list[str], lkos: dict[str, int], task_id: TaskID
+    context: _Context, plans: list[PlanSummary], lkos: dict[str, int], task_id: TaskID
 ) -> dict[str, _YnabPlanData]:
     return dict(
         await asyncio.gather(
-            *(_get_plan_data(context, plan_id, lkos, task_id) for plan_id in plan_ids)
+            *(_get_plan_data(context, plan, lkos, task_id) for plan in plans)
         )
     )
 
 
 async def _get_plan_data(
-    context: _Context, plan_id: str, lkos: dict[str, int], task_id: TaskID
+    context: _Context, plan: PlanSummary, lkos: dict[str, int], task_id: TaskID
 ) -> tuple[str, _YnabPlanData]:
+    plan_id = str(plan.id)
+    assert plan.first_month is not None
     py = _ProgressYnab(context, plan_id, lkos, task_id)
     accounts, categories, payees, transactions, scheduled = await asyncio.gather(
         py.get(AccountsApi(context.api_client).get_accounts),
         py.get(CategoriesApi(context.api_client).get_categories),
         py.get(PayeesApi(context.api_client).get_payees),
-        py.get(TransactionsApi(context.api_client).get_transactions),
+        py.get(
+            ChunkedTransactionsApi(
+                context.api_client, plan.first_month
+            ).get_transactions
+        ),
         py.get(ScheduledTransactionsApi(context.api_client).get_scheduled_transactions),
     )
     return (
@@ -681,6 +694,70 @@ class _ProgressYnab:
             self.context.progress.update(self.task_id, advance=1)
 
 
+@dataclass(slots=True, frozen=True)
+class ChunkedTransactionsApi:
+    api_client: ApiClient
+    first_month: date
+
+    async def get_transactions(
+        self, *, plan_id: str, last_knowledge_of_server: int | None
+    ) -> TransactionsResponse:
+        transactions_api = TransactionsApi(self.api_client)
+        if last_knowledge_of_server is not None:
+            return await transactions_api.get_transactions(
+                plan_id=plan_id,
+                last_knowledge_of_server=last_knowledge_of_server,
+            )
+        return await self._get(transactions_api, plan_id)
+
+    async def _get(
+        self, transactions_api: TransactionsApi, plan_id: str
+    ) -> TransactionsResponse:
+        today = date.today()
+        responses = await asyncio.gather(
+            *(
+                self._chunk(transactions_api, plan_id, sd, ud)
+                for sd, ud in _quarterly(self.first_month, today)
+            )
+        )
+        transactions = [t for r in responses for t in r.data.transactions]
+        ids = [t.id for t in transactions]
+        assert len(set(ids)) == len(ids)  # paranoid check for no overlap across chunks
+        return TransactionsResponse(
+            data=TransactionsResponseData(
+                transactions=transactions,
+                server_knowledge=max(r.data.server_knowledge for r in responses),
+            )
+        )
+
+    @retry(stop=stop_after_attempt(3))
+    async def _chunk(
+        self,
+        transactions_api: TransactionsApi,
+        plan_id: str,
+        since_date: date,
+        until_date: date,
+    ) -> TransactionsResponse:
+        return await transactions_api.get_transactions(
+            plan_id=plan_id,
+            since_date=since_date,
+            until_date=until_date,
+        )
+
+
+def _add_months(d: date, months: int) -> date:
+    month_index = d.month - 1 + months
+    return date(d.year + month_index // 12, month_index % 12 + 1, 1)
+
+
+def _quarterly(first_month: date, today: date) -> Iterator[tuple[date, date]]:
+    since_date = first_month
+    while since_date <= today:
+        until_date = min(_add_months(since_date, 3) - timedelta(days=1), today)
+        yield since_date, until_date
+        since_date = until_date + timedelta(days=1)
+
+
 def main(
     argv: Sequence[str] | None = None, *, token_override: str | None = None
 ) -> int:

{sqlite_export_for_ynab-2.9.2 → sqlite_export_for_ynab-2.9.4/sqlite_export_for_ynab.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlite_export_for_ynab
-Version: 2.9.2
+Version: 2.9.4
 Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
 Author-email: Max R <mxr@users.noreply.github.com>
 License-Expression: MIT
@@ -16,7 +16,7 @@ License-File: LICENSE
 Requires-Dist: aiohttp>=3
 Requires-Dist: aiopathlib
 Requires-Dist: aiosqlite
-Requires-Dist: asyncio-for-ynab~=1.84.0
+Requires-Dist: asyncio-for-ynab~=1.85.0
 Requires-Dist: fasteners
 Requires-Dist: rich>=10
 Requires-Dist: tenacity

{sqlite_export_for_ynab-2.9.2 → sqlite_export_for_ynab-2.9.4}/sqlite_export_for_ynab.egg-info/requires.txt

@@ -1,7 +1,7 @@
 aiohttp>=3
 aiopathlib
 aiosqlite
-asyncio-for-ynab~=1.84.0
+asyncio-for-ynab~=1.85.0
 fasteners
 rich>=10
 tenacity

{sqlite_export_for_ynab-2.9.2 → sqlite_export_for_ynab-2.9.4}/testing/fixtures.py

@@ -37,6 +37,7 @@ PLANS: list[PlanSummary] = [
     PlanSummary(
         id=UUID(PLAN_ID_1),
         name="Plan 1",
+        first_month=date.fromisoformat("2024-01-01"),
         currency_format=CurrencyFormat(
             currency_symbol="$",
             decimal_digits=2,
@@ -51,6 +52,7 @@ PLANS: list[PlanSummary] = [
     PlanSummary(
         id=UUID(PLAN_ID_2),
         name="Plan 2",
+        first_month=date.fromisoformat("2024-01-01"),
         currency_format=CurrencyFormat(
             currency_symbol="$",
             decimal_digits=2,
@@ -67,6 +69,11 @@ PLANS: list[PlanSummary] = [
 SERVER_KNOWLEDGE_1 = 107667
 SERVER_KNOWLEDGE_2 = 107668
 
+TRANSACTION_CHUNKS = [
+    (date(2021, 1, 1), date(2021, 3, 31)),
+    (date(2021, 4, 1), date(2021, 6, 30)),
+]
+
 LKOS = {
     PLAN_ID_1: SERVER_KNOWLEDGE_1,
     PLAN_ID_2: SERVER_KNOWLEDGE_2,

{sqlite_export_for_ynab-2.9.2 → sqlite_export_for_ynab-2.9.4}/tests/_main_test.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import tomllib
+from datetime import date
 from pathlib import Path
 from unittest.mock import AsyncMock
 from unittest.mock import Mock
@@ -20,8 +21,10 @@ from sqlite_export_for_ynab._main import _ENV_TOKEN
 from sqlite_export_for_ynab._main import _get_plan_summaries
 from sqlite_export_for_ynab._main import _PACKAGE
 from sqlite_export_for_ynab._main import _PROGRESS_COLUMNS
+from sqlite_export_for_ynab._main import _quarterly
 from sqlite_export_for_ynab._main import async_main
 from sqlite_export_for_ynab._main import asyncio_for_ynab
+from sqlite_export_for_ynab._main import ChunkedTransactionsApi
 from sqlite_export_for_ynab._main import contents
 from sqlite_export_for_ynab._main import get_last_knowledge_of_server
 from sqlite_export_for_ynab._main import get_relations
@@ -71,9 +74,11 @@ from testing.fixtures import SCHEDULED_TRANSACTION_ID_3
 from testing.fixtures import SCHEDULED_TRANSACTIONS
 from testing.fixtures import scheduled_transactions_response
 from testing.fixtures import SERVER_KNOWLEDGE_1
+from testing.fixtures import SERVER_KNOWLEDGE_2
 from testing.fixtures import SUBTRANSACTION_ID_1
 from testing.fixtures import SUBTRANSACTION_ID_2
 from testing.fixtures import TOKEN
+from testing.fixtures import TRANSACTION_CHUNKS
 from testing.fixtures import TRANSACTION_ID_1
 from testing.fixtures import TRANSACTION_ID_2
 from testing.fixtures import TRANSACTION_ID_3
@@ -106,6 +111,87 @@ async def context(tmp_path):
             yield _Context(progress, con, lock, Mock(spec=asyncio_for_ynab.ApiClient))
 
 
+@patch("sqlite_export_for_ynab._main.TransactionsApi.get_transactions")
+@pytest.mark.asyncio
+async def test_chunked_transactions_api_uses_last_knowledge_of_server_when_present(
+    get_transactions, context
+):
+    get_transactions.return_value = transactions_response(
+        TRANSACTIONS, SERVER_KNOWLEDGE_1
+    )
+    chunked_transactions_api = ChunkedTransactionsApi(
+        context.api_client, date(2020, 1, 1)
+    )
+
+    response = await chunked_transactions_api.get_transactions(
+        plan_id=PLAN_ID_1, last_knowledge_of_server=LKOS[PLAN_ID_1]
+    )
+
+    get_transactions.assert_awaited_once_with(
+        plan_id=PLAN_ID_1, last_knowledge_of_server=LKOS[PLAN_ID_1]
+    )
+    assert response.data.transactions == TRANSACTIONS
+
+
+@pytest.mark.parametrize(
+    ("first_month", "today", "expected_chunks"),
+    (
+        pytest.param(
+            date(2021, 4, 1),
+            date(2022, 1, 15),
+            [
+                (date(2021, 4, 1), date(2021, 6, 30)),
+                (date(2021, 7, 1), date(2021, 9, 30)),
+                (date(2021, 10, 1), date(2021, 12, 31)),
+                (date(2022, 1, 1), date(2022, 1, 15)),
+            ],
+            id="spans-multiple-quarters-with-partial-last-chunk",
+        ),
+        pytest.param(
+            date(2024, 1, 1),
+            date(2024, 1, 1),
+            [(date(2024, 1, 1), date(2024, 1, 1))],
+            id="single-day-range",
+        ),
+    ),
+)
+def test_quarterly(first_month, today, expected_chunks):
+    assert list(_quarterly(first_month, today)) == expected_chunks
+
+
+def _chunk_merge_test_side_effect(*, plan_id, since_date, until_date):
+    assert plan_id == PLAN_ID_1
+    assert (since_date, until_date) in TRANSACTION_CHUNKS
+    old_transaction, new_transaction, _ = TRANSACTIONS
+    if since_date == TRANSACTION_CHUNKS[0][0]:
+        return transactions_response([old_transaction], SERVER_KNOWLEDGE_1)
+    return transactions_response([new_transaction], SERVER_KNOWLEDGE_2)
+
+
+@patch("sqlite_export_for_ynab._main._quarterly")
+@patch("sqlite_export_for_ynab._main.TransactionsApi.get_transactions")
+@pytest.mark.asyncio
+async def test_chunked_transactions_api_merges_chunk_responses(
+    get_transactions, quarterly_chunks, context
+):
+    get_transactions.side_effect = _chunk_merge_test_side_effect
+    quarterly_chunks.return_value = iter(TRANSACTION_CHUNKS)
+    chunked_transactions_api = ChunkedTransactionsApi(
+        context.api_client, date(2021, 1, 1)
+    )
+
+    response = await chunked_transactions_api.get_transactions(
+        plan_id=PLAN_ID_1, last_knowledge_of_server=None
+    )
+
+    old_transaction, new_transaction, _ = TRANSACTIONS
+    assert {t.id for t in response.data.transactions} == {
+        old_transaction.id,
+        new_transaction.id,
+    }
+    assert response.data.server_knowledge == SERVER_KNOWLEDGE_2
+
+
 @pytest.mark.parametrize(
     ("xdg_data_home", "expected_prefix"),
     (
@@ -1109,7 +1195,12 @@ async def test_sync_no_data_quiet(tmp_path, capsys):
 )
 @patch(
     "sqlite_export_for_ynab._main.TransactionsApi.get_transactions",
-    new=AsyncMock(return_value=transactions_response(TRANSACTIONS, SERVER_KNOWLEDGE_1)),
+    new=AsyncMock(
+        side_effect=lambda *, plan_id, since_date, until_date: transactions_response(
+            [t for t in TRANSACTIONS if since_date <= t.var_date <= until_date],
+            SERVER_KNOWLEDGE_1,
+        )
+    ),
 )
 @patch(
     "sqlite_export_for_ynab._main.ScheduledTransactionsApi.get_scheduled_transactions",