sqlite-export-for-ynab 2.3.0__tar.gz → 2.5.0__tar.gz

{sqlite_export_for_ynab-2.3.0/sqlite_export_for_ynab.egg-info → sqlite_export_for_ynab-2.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlite_export_for_ynab
-Version: 2.3.0
+Version: 2.5.0
 Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
 Home-page: https://github.com/mxr/sqlite-export-for-ynab
 Author: Max R
@@ -15,7 +15,9 @@ Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp>=3
-Requires-Dist: tldm
+Requires-Dist: aiopathlib
+Requires-Dist: aiosqlite
+Requires-Dist: rich
 Dynamic: license-file
 
 # sqlite-export-for-ynab

{sqlite_export_for_ynab-2.3.0 → sqlite_export_for_ynab-2.5.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = sqlite_export_for_ynab
-version = 2.3.0
+version = 2.5.0
 description = SQLite Export for YNAB - Export YNAB Data to SQLite
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -20,7 +20,9 @@ keywords = ynab, sqlite, sql, budget, plan, cli
 packages = find:
 install_requires = 
 	aiohttp>=3
-	tldm
+	aiopathlib
+	aiosqlite
+	rich
 python_requires = >=3.12
 
 [options.entry_points]

{sqlite_export_for_ynab-2.3.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab/_main.py

@@ -4,10 +4,12 @@ import argparse
 import asyncio
 import json
 import os
-import sqlite3
+from contextlib import asynccontextmanager
+from contextlib import contextmanager
 from dataclasses import dataclass
 from importlib import resources
 from importlib.metadata import version
+from itertools import batched
 from pathlib import Path
 from typing import Any
 from typing import ClassVar
@@ -20,13 +22,19 @@ from urllib.parse import urljoin
 from urllib.parse import urlunparse
 
 import aiohttp
-from tldm import tldm
+import aiosqlite
+from aiopathlib import AsyncPath
+from rich.progress import BarColumn
+from rich.progress import MofNCompleteColumn
+from rich.progress import Progress
+from rich.progress import TaskID
+from rich.progress import TextColumn
+from rich.progress import TimeElapsedColumn
 
 from sqlite_export_for_ynab import ddl
 
 if TYPE_CHECKING:
-    from collections.abc import Awaitable, Sequence
-    from typing import Never
+    from collections.abc import AsyncIterator, Awaitable, Iterator, Sequence
 
 
 _EntryTable = (
@@ -40,6 +48,14 @@ _EntryTable = (
     | Literal["scheduled_transactions"]
     | Literal["scheduled_subtransactions"]
 )
+_Endpoint = (
+    Literal["accounts"]
+    | Literal["categories"]
+    | Literal["payees"]
+    | Literal["transactions"]
+    | Literal["scheduled_transactions"]
+)
+_ENDPOINTS = tuple(lit.__args__[0] for lit in _Endpoint.__args__)
 _ALL_RELATIONS = frozenset(
     ("plans", "flat_transactions", "scheduled_flat_transactions")
     + tuple(lit.__args__[0] for lit in _EntryTable.__args__)
@@ -49,6 +65,15 @@ _ENV_TOKEN = "YNAB_PERSONAL_ACCESS_TOKEN"
 
 _PACKAGE = "sqlite-export-for-ynab"
 
+_BATCH_SIZE = 100
+
+_PROGRESS_COLUMNS = (
+    TextColumn("[progress.description]{task.description}"),
+    BarColumn(),
+    MofNCompleteColumn(),
+    TimeElapsedColumn(),
+)
+
 
 def resolve_token(token_override: str | None = None) -> str:
     token = token_override or os.environ.get(_ENV_TOKEN)
@@ -113,67 +138,92 @@ def _print(message: str, *, quiet: bool) -> None:
         print(message)
 
 
+@dataclass
+class _Context:
+    session: aiohttp.ClientSession
+    progress: Progress
+    con: aiosqlite.Connection
+
+
+@asynccontextmanager
+async def _context(db: Path, *, quiet: bool) -> AsyncIterator[_Context]:
+    progress = Progress(*_PROGRESS_COLUMNS, disable=quiet)
+    async with aiohttp.ClientSession() as session, aiosqlite.connect(db) as con:
+        con.row_factory = aiosqlite.Row
+        yield _Context(session, progress, con)
+
+
+@contextmanager
+def _progress(context: _Context) -> Iterator[None]:
+    context.progress.start()
+    try:
+        yield
+    finally:
+        context.progress.stop()
+        for task_id in context.progress.task_ids:
+            context.progress.remove_task(task_id)
+
+
 async def sync(
     token: str, db: Path, full_refresh: bool, *, quiet: bool = False
 ) -> None:
-    async with aiohttp.ClientSession() as session:
-        plans = (await YnabClient(token, session)("plans"))["plans"]
+    await AsyncPath(db).parent.mkdir(parents=True, exist_ok=True)
 
-    plan_ids = [plan["id"] for plan in plans]
+    async with _context(db, quiet=quiet) as context:
+        plans = (await YnabClient(token, context.session)("plans"))["plans"]
 
-    if not db.exists():
-        db.parent.mkdir(parents=True, exist_ok=True)
-
-    with sqlite3.connect(db) as con:
-        con.row_factory = sqlite3.Row
-        cur = con.cursor()
+        plan_ids = [plan["id"] for plan in plans]
 
         if full_refresh:
             _print("Dropping relations...", quiet=quiet)
-            cur.executescript(contents("drop-relations.sql"))
-            con.commit()
+            async with context.con.cursor() as cur:
+                await cur.executescript(await contents("drop-relations.sql"))
+            await context.con.commit()
             _print("Done", quiet=quiet)
 
-        relations = get_relations(cur)
-        if relations != _ALL_RELATIONS:
-            _print("Recreating relations...", quiet=quiet)
-            cur.executescript(contents("create-relations.sql"))
-            con.commit()
-            _print("Done", quiet=quiet)
+        async with context.con.cursor() as cur:
+            relations = await get_relations(cur)
+            if relations != _ALL_RELATIONS:
+                _print("Recreating relations...", quiet=quiet)
+                await cur.executescript(await contents("create-relations.sql"))
+                await context.con.commit()
+                _print("Done", quiet=quiet)
 
         _print("Fetching plan data...", quiet=quiet)
-        lkos = get_last_knowledge_of_server(cur)
-        async with aiohttp.ClientSession() as session:
-            with tldm(desc="Plan Data", total=len(plans) * 5, disable=quiet) as pbar:
-                yc = ProgressYnabClient(YnabClient(token, session), pbar)
-
-                account_jobs = jobs(yc, "accounts", plan_ids, lkos)
-                cat_jobs = jobs(yc, "categories", plan_ids, lkos)
-                payee_jobs = jobs(yc, "payees", plan_ids, lkos)
-                txn_jobs = jobs(yc, "transactions", plan_ids, lkos)
-                sched_txn_jobs = jobs(yc, "scheduled_transactions", plan_ids, lkos)
-
-                data = await asyncio.gather(
-                    *account_jobs, *cat_jobs, *payee_jobs, *txn_jobs, *sched_txn_jobs
-                )
+        async with context.con.cursor() as cur:
+            lkos = await get_last_knowledge_of_server(cur)
+        with _progress(context):
+            yc = ProgressYnabClient(
+                YnabClient(token, context.session),
+                context,
+                context.progress.add_task(
+                    "Plan Data", total=len(plans) * len(_ENDPOINTS)
+                ),
+            )
 
-            la = len(account_jobs)
-            lc = len(cat_jobs)
-            lp = len(payee_jobs)
-            lt = len(txn_jobs)
-
-            all_account_data = data[:la]
-            all_cat_data = data[la : la + lc]
-            all_payee_data = data[la + lc : la + lc + lp]
-            all_txn_data = data[la + lc + lp : la + lc + lp + lt]
-            all_sched_txn_data = data[la + lc + lp + lt :]
-
-            new_lkos = {
-                plan_id: transaction_data["server_knowledge"]
-                for plan_id, transaction_data in zip(
-                    plan_ids, all_txn_data, strict=True
+            endpoint_data = dict(
+                zip(
+                    _ENDPOINTS,
+                    await asyncio.gather(
+                        *(
+                            asyncio.gather(*jobs(yc, endpoint, plan_ids, lkos))
+                            for endpoint in _ENDPOINTS
+                        )
+                    ),
+                    strict=True,
                 )
-            }
+            )
+
+        all_account_data = endpoint_data["accounts"]
+        all_cat_data = endpoint_data["categories"]
+        all_payee_data = endpoint_data["payees"]
+        all_txn_data = endpoint_data["transactions"]
+        all_sched_txn_data = endpoint_data["scheduled_transactions"]
+
+        new_lkos = {
+            plan_id: transaction_data["server_knowledge"]
+            for plan_id, transaction_data in zip(plan_ids, all_txn_data, strict=True)
+        }
         _print("Done", quiet=quiet)
 
         if (
@@ -186,95 +236,129 @@ async def sync(
             _print("No new data fetched", quiet=quiet)
         else:
             _print("Inserting plan data...", quiet=quiet)
-            insert_plans(cur, plans, new_lkos)
-            for plan_id, account_data in zip(plan_ids, all_account_data, strict=True):
-                insert_accounts(cur, plan_id, account_data["accounts"], quiet=quiet)
-            for plan_id, cat_data in zip(plan_ids, all_cat_data, strict=True):
-                insert_category_groups(
-                    cur, plan_id, cat_data["category_groups"], quiet=quiet
-                )
-            for plan_id, payee_data in zip(plan_ids, all_payee_data, strict=True):
-                insert_payees(cur, plan_id, payee_data["payees"], quiet=quiet)
-            for plan_id, txn_data in zip(plan_ids, all_txn_data, strict=True):
-                insert_transactions(cur, plan_id, txn_data["transactions"], quiet=quiet)
-            for plan_id, sched_txn_data in zip(
-                plan_ids, all_sched_txn_data, strict=True
-            ):
-                insert_scheduled_transactions(
-                    cur, plan_id, sched_txn_data["scheduled_transactions"], quiet=quiet
+            with _progress(context):
+                await insert_plan_data(
+                    context,
+                    plans,
+                    plan_ids,
+                    all_account_data,
+                    all_cat_data,
+                    all_payee_data,
+                    all_txn_data,
+                    all_sched_txn_data,
+                    new_lkos,
                 )
+                await context.con.commit()
             _print("Done", quiet=quiet)
 
 
-def contents(filename: str) -> str:
-    return (resources.files(ddl) / filename).read_text()
+async def contents(filename: str) -> str:
+    return await AsyncPath(resources.files(ddl) / filename).read_text()
 
 
-def get_relations(cur: sqlite3.Cursor) -> set[str]:
-    return {
-        t["name"]
-        for t in cur.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' OR type='view'"
-        ).fetchall()
-    }
+async def get_relations(cur: aiosqlite.Cursor) -> set[str]:
+    await cur.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' OR type='view'"
+    )
+    return {t["name"] for t in await cur.fetchall()}
 
 
-def get_last_knowledge_of_server(cur: sqlite3.Cursor) -> dict[str, int]:
-    return {
-        r["id"]: r["last_knowledge_of_server"]
-        for r in cur.execute(
-            "SELECT id, last_knowledge_of_server FROM plans",
-        ).fetchall()
-    }
+async def get_last_knowledge_of_server(cur: aiosqlite.Cursor) -> dict[str, int]:
+    await cur.execute(
+        "SELECT id, last_knowledge_of_server FROM plans",
+    )
+    return {r["id"]: r["last_knowledge_of_server"] for r in await cur.fetchall()}
 
 
-def insert_plans(
-    cur: sqlite3.Cursor, plans: list[dict[str, Any]], lkos: dict[str, int]
+async def insert_plan_data(
+    context: _Context,
+    plans: list[dict[str, Any]],
+    plan_ids: list[str],
+    all_account_data: list[dict[str, Any]],
+    all_cat_data: list[dict[str, Any]],
+    all_payee_data: list[dict[str, Any]],
+    all_txn_data: list[dict[str, Any]],
+    all_sched_txn_data: list[dict[str, Any]],
+    new_lkos: dict[str, int],
 ) -> None:
-    cur.executemany(
-        """
-        INSERT OR REPLACE INTO plans (
-            id
-            , name
-            , currency_format_currency_symbol
-            , currency_format_decimal_digits
-            , currency_format_decimal_separator
-            , currency_format_display_symbol
-            , currency_format_group_separator
-            , currency_format_iso_code
-            , currency_format_symbol_first
-            , last_knowledge_of_server
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        """,
-        (
-            (
-                plan_id := plan["id"],
-                plan["name"],
-                plan["currency_format"]["currency_symbol"],
-                plan["currency_format"]["decimal_digits"],
-                plan["currency_format"]["decimal_separator"],
-                plan["currency_format"]["display_symbol"],
-                plan["currency_format"]["group_separator"],
-                plan["currency_format"]["iso_code"],
-                plan["currency_format"]["symbol_first"],
-                lkos[plan_id],
+    await insert_plans(context, plans, new_lkos)
+    await asyncio.gather(
+        *(
+            insert_accounts(context, plan_id, account_data["accounts"])
+            for plan_id, account_data in zip(plan_ids, all_account_data, strict=True)
+        ),
+        *(
+            insert_category_groups(context, plan_id, cat_data["category_groups"])
+            for plan_id, cat_data in zip(plan_ids, all_cat_data, strict=True)
+        ),
+        *(
+            insert_payees(context, plan_id, payee_data["payees"])
+            for plan_id, payee_data in zip(plan_ids, all_payee_data, strict=True)
+        ),
+    )
+    await asyncio.gather(
+        *(
+            insert_transactions(context, plan_id, txn_data["transactions"])
+            for plan_id, txn_data in zip(plan_ids, all_txn_data, strict=True)
+        ),
+        *(
+            insert_scheduled_transactions(
+                context, plan_id, sched_txn_data["scheduled_transactions"]
+            )
+            for plan_id, sched_txn_data in zip(
+                plan_ids, all_sched_txn_data, strict=True
             )
-            for plan in plans
         ),
     )
 
 
+async def insert_plans(
+    context: _Context, plans: list[dict[str, Any]], lkos: dict[str, int]
+) -> None:
+    async with context.con.cursor() as cur:
+        for plan_batch in batched(plans, _BATCH_SIZE):
+            await cur.executemany(
+                """
+                INSERT OR REPLACE INTO plans (
+                    id
+                    , name
+                    , currency_format_currency_symbol
+                    , currency_format_decimal_digits
+                    , currency_format_decimal_separator
+                    , currency_format_display_symbol
+                    , currency_format_group_separator
+                    , currency_format_iso_code
+                    , currency_format_symbol_first
+                    , last_knowledge_of_server
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    (
+                        plan_id := plan["id"],
+                        plan["name"],
+                        plan["currency_format"]["currency_symbol"],
+                        plan["currency_format"]["decimal_digits"],
+                        plan["currency_format"]["decimal_separator"],
+                        plan["currency_format"]["display_symbol"],
+                        plan["currency_format"]["group_separator"],
+                        plan["currency_format"]["iso_code"],
+                        plan["currency_format"]["symbol_first"],
+                        lkos[plan_id],
+                    )
+                    for plan in plan_batch
+                ),
+            )
+
+
 _LOAN_ACCOUNT_PERIODIC_VALUES = frozenset(
     ("debt_escrow_amounts", "debt_interest_rates", "debt_minimum_payments")
 )
 
 
-def insert_accounts(
-    cur: sqlite3.Cursor,
+async def insert_accounts(
+    context: _Context,
     plan_id: str,
     accounts: list[dict[str, Any]],
-    *,
-    quiet: bool = False,
 ) -> None:
     # YNAB's LoanAccountPeriodValues are untyped dicts so we need to turn them into a more standard sub-entry view
     updated_accounts = [
@@ -294,147 +378,127 @@ def insert_accounts(
         for account in accounts
     ]
 
-    return insert_nested_entries(
-        cur,
+    await insert_nested_entries(
+        context,
         plan_id,
         updated_accounts,
         "Accounts",
         "accounts",
         "account_periodic_values",
         "account_periodic_values",
-        quiet=quiet,
     )
 
 
-def insert_category_groups(
-    cur: sqlite3.Cursor,
+async def insert_category_groups(
+    context: _Context,
     plan_id: str,
     category_groups: list[dict[str, Any]],
-    *,
-    quiet: bool = False,
 ) -> None:
-    return insert_nested_entries(
-        cur,
+    await insert_nested_entries(
+        context,
         plan_id,
         category_groups,
         "Categories",
         "category_groups",
         "categories",
         "categories",
-        quiet=quiet,
     )
 
 
-def insert_payees(
-    cur: sqlite3.Cursor,
+async def insert_payees(
+    context: _Context,
     plan_id: str,
     payees: list[dict[str, Any]],
-    *,
-    quiet: bool = False,
 ) -> None:
     if not payees:
         return
 
-    for payee in tldm(payees, desc="Payees", disable=quiet):
-        insert_entry(cur, "payees", plan_id, payee)
+    task_id = context.progress.add_task("Payees", total=len(payees))
+    await insert_entries(context, "payees", plan_id, payees, task_id)
 
 
-def insert_transactions(
-    cur: sqlite3.Cursor,
+async def insert_transactions(
+    context: _Context,
     plan_id: str,
     transactions: list[dict[str, Any]],
-    *,
-    quiet: bool = False,
 ) -> None:
-    return insert_nested_entries(
-        cur,
+    await insert_nested_entries(
+        context,
         plan_id,
         transactions,
         "Transactions",
         "transactions",
         "subtransactions",
         "subtransactions",
-        quiet=quiet,
     )
 
 
-def insert_scheduled_transactions(
-    cur: sqlite3.Cursor,
+async def insert_scheduled_transactions(
+    context: _Context,
     plan_id: str,
     scheduled_transactions: list[dict[str, Any]],
-    *,
-    quiet: bool = False,
 ) -> None:
-    return insert_nested_entries(
-        cur,
+    await insert_nested_entries(
+        context,
         plan_id,
         scheduled_transactions,
         "Scheduled Transactions",
         "scheduled_transactions",
         "subtransactions",
         "scheduled_subtransactions",
-        quiet=quiet,
     )
 
 
 @overload
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    context: _Context,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: Literal["Accounts"],
     entries_name: Literal["accounts"],
     subentries_name: Literal["account_periodic_values"],
     subentries_table_name: Literal["account_periodic_values"],
-    *,
-    quiet: bool = False,
 ) -> None: ...
 
 
 @overload
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    context: _Context,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: Literal["Categories"],
     entries_name: Literal["category_groups"],
     subentries_name: Literal["categories"],
     subentries_table_name: Literal["categories"],
-    *,
-    quiet: bool = False,
 ) -> None: ...
 
 
 @overload
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    context: _Context,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: Literal["Transactions"],
     entries_name: Literal["transactions"],
     subentries_name: Literal["subtransactions"],
     subentries_table_name: Literal["subtransactions"],
-    *,
-    quiet: bool = False,
 ) -> None: ...
 
 
 @overload
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    context: _Context,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: Literal["Scheduled Transactions"],
     entries_name: Literal["scheduled_transactions"],
     subentries_name: Literal["subtransactions"],
     subentries_table_name: Literal["scheduled_subtransactions"],
-    *,
-    quiet: bool = False,
 ) -> None: ...
 
 
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    context: _Context,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: (
@@ -460,55 +524,55 @@ def insert_nested_entries(
         | Literal["subtransactions"]
         | Literal["scheduled_subtransactions"]
     ),
-    *,
-    quiet: bool = False,
 ) -> None:
     if not entries:
         return
 
-    with tldm(
-        total=sum(1 + len(e[subentries_name]) for e in entries),
-        desc=desc,
-        disable=quiet,
-    ) as pbar:
-        for entry in entries:
-            insert_entry(
-                cur,
-                entries_name,
-                plan_id,
-                {k: v for k, v in entry.items() if k != subentries_name},
-            )
-            pbar.update()
-
-            for subentry in entry[subentries_name]:
-                insert_entry(cur, subentries_table_name, plan_id, subentry)
-                pbar.update()
+    task_id = context.progress.add_task(
+        desc, total=sum(1 + len(e[subentries_name]) for e in entries)
+    )
+    await insert_entries(
+        context,
+        entries_name,
+        plan_id,
+        [{k: v for k, v in entry.items() if k != subentries_name} for entry in entries],
+        task_id,
+    )
+    await insert_entries(
+        context,
+        subentries_table_name,
+        plan_id,
+        [subentry for entry in entries for subentry in entry[subentries_name]],
+        task_id,
+    )
 
 
-def insert_entry(
-    cur: sqlite3.Cursor,
+async def insert_entries(
+    context: _Context,
     table: _EntryTable,
     plan_id: str,
-    entry: dict[str, Any],
+    entries: list[dict[str, Any]],
+    task_id: TaskID,
 ) -> None:
-    ekeys, evalues = zip(*entry.items(), strict=True)
-    keys, values = ekeys + ("plan_id",), evalues + (plan_id,)
+    if not entries:
+        return
 
-    cur.execute(
-        f"INSERT OR REPLACE INTO {table} ({', '.join(keys)}) VALUES ({', '.join('?' * len(values))})",
-        values,
-    )
+    entry_keys = tuple(entries[0])
+    sql = f"INSERT OR REPLACE INTO {table} ({', '.join(entry_keys + ('plan_id',))}) VALUES ({', '.join('?' * (len(entry_keys) + 1))})"
+
+    async with context.con.cursor() as cur:
+        for entry_batch in batched(entries, _BATCH_SIZE):
+            values_batch = [
+                tuple(entry[key] for key in entry_keys) + (plan_id,)
+                for entry in entry_batch
+            ]
+            await cur.executemany(sql, values_batch)
+            context.progress.update(task_id, advance=len(values_batch))
 
 
 def jobs(
     yc: SupportsYnabClient,
-    endpoint: (
-        Literal["accounts"]
-        | Literal["categories"]
-        | Literal["payees"]
-        | Literal["transactions"]
-        | Literal["scheduled_transactions"]
-    ),
+    endpoint: _Endpoint,
     plan_ids: list[str],
     lkos: dict[str, int],
 ) -> list[Awaitable[dict[str, Any]]]:
@@ -527,7 +591,8 @@ class SupportsYnabClient(Protocol):
 @dataclass
 class ProgressYnabClient:
     yc: YnabClient
-    pbar: tldm[Never]
+    context: _Context
+    task_id: TaskID
 
     async def __call__(
         self, path: str, last_knowledge_of_server: int | None = None
@@ -535,7 +600,7 @@ class ProgressYnabClient:
         try:
             return await self.yc(path, last_knowledge_of_server)
         finally:
-            self.pbar.update()
+            self.context.progress.update(self.task_id, advance=1)
 
 
 @dataclass

{sqlite_export_for_ynab-2.3.0 → sqlite_export_for_ynab-2.5.0/sqlite_export_for_ynab.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlite_export_for_ynab
-Version: 2.3.0
+Version: 2.5.0
 Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
 Home-page: https://github.com/mxr/sqlite-export-for-ynab
 Author: Max R
@@ -15,7 +15,9 @@ Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp>=3
-Requires-Dist: tldm
+Requires-Dist: aiopathlib
+Requires-Dist: aiosqlite
+Requires-Dist: rich
 Dynamic: license-file
 
 # sqlite-export-for-ynab

sqlite_export_for_ynab-2.5.0/sqlite_export_for_ynab.egg-info/requires.txt

@@ -0,0 +1,4 @@
+aiohttp>=3
+aiopathlib
+aiosqlite
+rich

{sqlite_export_for_ynab-2.3.0 → sqlite_export_for_ynab-2.5.0}/testing/fixtures.py

@@ -1,14 +1,15 @@
 from __future__ import annotations
 
 import re
-import sqlite3
 from typing import Any
+from typing import TYPE_CHECKING
 from uuid import uuid4
 
 import pytest
 from aioresponses import aioresponses
 
-from sqlite_export_for_ynab._main import contents
+if TYPE_CHECKING:
+    import aiosqlite
 
 PLAN_ID_1 = str(uuid4())
 PLAN_ID_2 = str(uuid4())
@@ -144,6 +145,15 @@ CATEGORY_GROUPS: list[dict[str, Any]] = [
                 "activity_currency": 1.0,
                 "budgeted_formatted": "$10.25",
                 "budgeted_currency": 10.25,
+                "goal_target_formatted": None,
+                "goal_target_currency": None,
+                "goal_under_funded_formatted": None,
+                "goal_under_funded_currency": None,
+                "goal_overall_funded_formatted": None,
+                "goal_overall_funded_currency": None,
+                "goal_overall_left_formatted": None,
+                "goal_overall_left_currency": None,
+                "goal_target_date": None,
             },
         ],
     },
@@ -162,6 +172,15 @@ CATEGORY_GROUPS: list[dict[str, Any]] = [
                 "activity_currency": 7.5,
                 "budgeted_formatted": "$15.00",
                 "budgeted_currency": 15.0,
+                "goal_target_formatted": None,
+                "goal_target_currency": None,
+                "goal_under_funded_formatted": None,
+                "goal_under_funded_currency": None,
+                "goal_overall_funded_formatted": None,
+                "goal_overall_funded_currency": None,
+                "goal_overall_left_formatted": None,
+                "goal_overall_left_currency": None,
+                "goal_target_date": None,
             },
             {
                 "id": CATEGORY_ID_4,
@@ -174,6 +193,15 @@ CATEGORY_GROUPS: list[dict[str, Any]] = [
                 "activity_currency": 19.0,
                 "budgeted_formatted": "$20.00",
                 "budgeted_currency": 20.0,
+                "goal_target_formatted": None,
+                "goal_target_currency": None,
+                "goal_under_funded_formatted": None,
+                "goal_under_funded_currency": None,
+                "goal_overall_funded_formatted": None,
+                "goal_overall_funded_currency": None,
+                "goal_overall_left_formatted": None,
+                "goal_overall_left_currency": None,
+                "goal_target_date": None,
             },
         ],
     },
@@ -325,22 +353,13 @@ SCHEDULED_TRANSACTIONS: list[dict[str, Any]] = [
 ]
 
 
-@pytest.fixture
-def cur():
-    with sqlite3.connect(":memory:") as con:
-        con.row_factory = sqlite3.Row
-        cursor = con.cursor()
-        cursor.executescript(contents("create-relations.sql"))
-        yield cursor
-
-
 @pytest.fixture
 def mock_aioresponses():
     with aioresponses() as m:
         yield m
 
 
-def strip_nones(d: sqlite3.Row) -> dict[str, Any]:
+def strip_nones(d: aiosqlite.Row) -> dict[str, Any]:
     return {k: v for k, v in dict(d).items() if v is not None}
 
 

{sqlite_export_for_ynab-2.3.0 → sqlite_export_for_ynab-2.5.0}/tests/_main_test.py

@@ -2,20 +2,23 @@ from __future__ import annotations
 
 import asyncio
 import json
-import sqlite3
 from configparser import ConfigParser
 from pathlib import Path
 from unittest.mock import patch
 
 import aiohttp
+import aiosqlite
 import pytest
+import pytest_asyncio
 from aiohttp.http_exceptions import HttpProcessingError
-from tldm import tldm
+from rich.progress import Progress
 
 from sqlite_export_for_ynab import default_db_path
 from sqlite_export_for_ynab._main import _ALL_RELATIONS
+from sqlite_export_for_ynab._main import _Context
 from sqlite_export_for_ynab._main import _ENV_TOKEN
 from sqlite_export_for_ynab._main import _PACKAGE
+from sqlite_export_for_ynab._main import _PROGRESS_COLUMNS
 from sqlite_export_for_ynab._main import contents
 from sqlite_export_for_ynab._main import get_last_knowledge_of_server
 from sqlite_export_for_ynab._main import get_relations
@@ -49,7 +52,6 @@ from testing.fixtures import CATEGORY_NAME_1
 from testing.fixtures import CATEGORY_NAME_2
 from testing.fixtures import CATEGORY_NAME_3
 from testing.fixtures import CATEGORY_NAME_4
-from testing.fixtures import cur
 from testing.fixtures import EXAMPLE_ENDPOINT_RE
 from testing.fixtures import LKOS
 from testing.fixtures import mock_aioresponses
@@ -80,6 +82,24 @@ from testing.fixtures import TRANSACTIONS
 from testing.fixtures import TRANSACTIONS_ENDPOINT_RE
 
 
+async def fetchall(con, query):
+    async with con.cursor() as cur:
+        await cur.execute(query)
+        return await cur.fetchall()
+
+
+@pytest_asyncio.fixture
+async def context():
+    with Progress(*_PROGRESS_COLUMNS, disable=True) as progress:
+        async with (
+            aiohttp.ClientSession(loop=asyncio.get_event_loop()) as session,
+            aiosqlite.connect(":memory:") as con,
+        ):
+            con.row_factory = aiosqlite.Row
+            await con.executescript(await contents("create-relations.sql"))
+            yield _Context(session, progress, con)
+
+
 @pytest.mark.parametrize(
     ("xdg_data_home", "expected_prefix"),
     (
@@ -92,22 +112,26 @@ def test_default_db_path(monkeypatch, xdg_data_home, expected_prefix):
     assert default_db_path() == expected_prefix / "sqlite-export-for-ynab" / "db.sqlite"
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_get_relations(cur):
-    assert get_relations(cur) == _ALL_RELATIONS
+@pytest.mark.asyncio
+async def test_get_relations(context):
+    async with context.con.cursor() as cur:
+        assert await get_relations(cur) == _ALL_RELATIONS
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_get_last_knowledge_of_server(cur):
-    insert_plans(cur, PLANS, LKOS)
-    assert get_last_knowledge_of_server(cur) == LKOS
+@pytest.mark.asyncio
+async def test_get_last_knowledge_of_server(context):
+    await insert_plans(context, PLANS, LKOS)
+    async with context.con.cursor() as cur:
+        assert await get_last_knowledge_of_server(cur) == LKOS
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_plans(cur):
-    insert_plans(cur, PLANS, LKOS)
-    cur.execute("SELECT * FROM plans ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.asyncio
+async def test_insert_plans(context):
+    await insert_plans(context, PLANS, LKOS)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(context.con, "SELECT * FROM plans ORDER BY name")
+    ] == [
         {
             "id": PLAN_ID_1,
             "name": PLANS[0]["name"],
@@ -135,15 +159,17 @@ def test_insert_plans(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_accounts(cur):
-    insert_accounts(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM accounts").fetchall()
-    assert not cur.execute("SELECT * FROM account_periodic_values").fetchall()
-
-    insert_accounts(cur, PLAN_ID_1, ACCOUNTS)
-    cur.execute("SELECT * FROM accounts ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.asyncio
+async def test_insert_accounts(context):
+    await insert_accounts(context, PLAN_ID_1, [])
+    assert not await fetchall(context.con, "SELECT * FROM accounts")
+    assert not await fetchall(context.con, "SELECT * FROM account_periodic_values")
+
+    await insert_accounts(context, PLAN_ID_1, ACCOUNTS)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(context.con, "SELECT * FROM accounts ORDER BY name")
+    ] == [
         {
             "id": ACCOUNT_ID_1,
             "plan_id": PLAN_ID_1,
@@ -170,8 +196,12 @@ def test_insert_accounts(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM account_periodic_values ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            context.con, "SELECT * FROM account_periodic_values ORDER BY name"
+        )
+    ] == [
         {
             "account_id": ACCOUNT_ID_1,
             "plan_id": PLAN_ID_1,
@@ -189,15 +219,19 @@ def test_insert_accounts(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_category_groups(cur):
-    insert_category_groups(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM category_groups").fetchall()
-    assert not cur.execute("SELECT * FROM categories").fetchall()
-
-    insert_category_groups(cur, PLAN_ID_1, CATEGORY_GROUPS)
-    cur.execute("SELECT * FROM category_groups ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.asyncio
+async def test_insert_category_groups(context):
+    await insert_category_groups(context, PLAN_ID_1, [])
+    assert not await fetchall(context.con, "SELECT * FROM category_groups")
+    assert not await fetchall(context.con, "SELECT * FROM categories")
+
+    await insert_category_groups(context, PLAN_ID_1, CATEGORY_GROUPS)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            context.con, "SELECT * FROM category_groups ORDER BY name"
+        )
+    ] == [
         {
             "id": CATEGORY_GROUP_ID_1,
             "name": CATEGORY_GROUP_NAME_1,
@@ -210,8 +244,10 @@ def test_insert_category_groups(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM categories ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(context.con, "SELECT * FROM categories ORDER BY name")
+    ] == [
         {
             "id": CATEGORY_ID_1,
             "category_group_id": CATEGORY_GROUP_ID_1,
@@ -276,14 +312,41 @@ def test_insert_category_groups(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_payees(cur):
-    insert_payees(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM payees").fetchall()
+@pytest.mark.asyncio
+async def test_insert_category_group_without_categories(context):
+    category_group = {
+        "id": CATEGORY_GROUP_ID_1,
+        "name": CATEGORY_GROUP_NAME_1,
+        "categories": [],
+    }
+
+    await insert_category_groups(context, PLAN_ID_1, [category_group])
+
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            context.con, "SELECT * FROM category_groups ORDER BY name"
+        )
+    ] == [
+        {
+            "id": CATEGORY_GROUP_ID_1,
+            "name": CATEGORY_GROUP_NAME_1,
+            "plan_id": PLAN_ID_1,
+        },
+    ]
+    assert not await fetchall(context.con, "SELECT * FROM categories")
+
 
-    insert_payees(cur, PLAN_ID_1, PAYEES)
-    cur.execute("SELECT * FROM payees ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.asyncio
+async def test_insert_payees(context):
+    await insert_payees(context, PLAN_ID_1, [])
+    assert not await fetchall(context.con, "SELECT * FROM payees")
+
+    await insert_payees(context, PLAN_ID_1, PAYEES)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(context.con, "SELECT * FROM payees ORDER BY name")
+    ] == [
         {
             "id": PAYEE_ID_1,
             "plan_id": PLAN_ID_1,
@@ -297,16 +360,18 @@ def test_insert_payees(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_transactions(cur):
-    insert_transactions(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM transactions").fetchall()
-    assert not cur.execute("SELECT * FROM subtransactions").fetchall()
-
-    insert_category_groups(cur, PLAN_ID_1, CATEGORY_GROUPS)
-    insert_transactions(cur, PLAN_ID_1, TRANSACTIONS)
-    cur.execute("SELECT * FROM transactions ORDER BY date")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.asyncio
+async def test_insert_transactions(context):
+    await insert_transactions(context, PLAN_ID_1, [])
+    assert not await fetchall(context.con, "SELECT * FROM transactions")
+    assert not await fetchall(context.con, "SELECT * FROM subtransactions")
+
+    await insert_category_groups(context, PLAN_ID_1, CATEGORY_GROUPS)
+    await insert_transactions(context, PLAN_ID_1, TRANSACTIONS)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(context.con, "SELECT * FROM transactions ORDER BY date")
+    ] == [
         {
             "id": TRANSACTION_ID_1,
             "plan_id": PLAN_ID_1,
@@ -345,8 +410,12 @@ def test_insert_transactions(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM subtransactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            context.con, "SELECT * FROM subtransactions ORDER BY amount"
+        )
+    ] == [
         {
             "id": SUBTRANSACTION_ID_1,
             "transaction_id": TRANSACTION_ID_1,
@@ -371,8 +440,12 @@ def test_insert_transactions(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM flat_transactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            context.con, "SELECT * FROM flat_transactions ORDER BY amount"
+        )
+    ] == [
         {
             "transaction_id": TRANSACTION_ID_1,
             "subtransaction_id": SUBTRANSACTION_ID_1,
@@ -404,16 +477,20 @@ def test_insert_transactions(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_scheduled_transactions(cur):
-    insert_scheduled_transactions(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM scheduled_transactions").fetchall()
-    assert not cur.execute("SELECT * FROM scheduled_subtransactions").fetchall()
-
-    insert_category_groups(cur, PLAN_ID_1, CATEGORY_GROUPS)
-    insert_scheduled_transactions(cur, PLAN_ID_1, SCHEDULED_TRANSACTIONS)
-    cur.execute("SELECT * FROM scheduled_transactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.asyncio
+async def test_insert_scheduled_transactions(context):
+    await insert_scheduled_transactions(context, PLAN_ID_1, [])
+    assert not await fetchall(context.con, "SELECT * FROM scheduled_transactions")
+    assert not await fetchall(context.con, "SELECT * FROM scheduled_subtransactions")
+
+    await insert_category_groups(context, PLAN_ID_1, CATEGORY_GROUPS)
+    await insert_scheduled_transactions(context, PLAN_ID_1, SCHEDULED_TRANSACTIONS)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            context.con, "SELECT * FROM scheduled_transactions ORDER BY amount"
+        )
+    ] == [
         {
             "id": SCHEDULED_TRANSACTION_ID_1,
             "plan_id": PLAN_ID_1,
@@ -449,8 +526,12 @@ def test_insert_scheduled_transactions(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM scheduled_subtransactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            context.con, "SELECT * FROM scheduled_subtransactions ORDER BY amount"
+        )
+    ] == [
         {
             "id": SCHEDULED_SUBTRANSACTION_ID_1,
             "scheduled_transaction_id": SCHEDULED_TRANSACTION_ID_1,
@@ -475,8 +556,12 @@ def test_insert_scheduled_transactions(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM scheduled_flat_transactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            context.con, "SELECT * FROM scheduled_flat_transactions ORDER BY amount"
+        )
+    ] == [
         {
             "transaction_id": SCHEDULED_TRANSACTION_ID_3,
             "plan_id": PLAN_ID_1,
@@ -523,14 +608,13 @@ def test_insert_scheduled_transactions(cur):
 
 @pytest.mark.asyncio
 @pytest.mark.usefixtures(mock_aioresponses.__name__)
-async def test_progress_ynab_client_ok(mock_aioresponses):
+async def test_progress_ynab_client_ok(context, mock_aioresponses):
     expected = {"example": [{"id": 1, "value": 2}, {"id": 3, "value": 4}]}
     mock_aioresponses.get(EXAMPLE_ENDPOINT_RE, body=json.dumps({"data": expected}))
 
-    with tldm(disable=True) as pbar:
-        async with aiohttp.ClientSession(loop=asyncio.get_event_loop()) as session:
-            pyc = ProgressYnabClient(YnabClient(TOKEN, session), pbar)
-            entries = await pyc("example")
+    task_id = context.progress.add_task("Example", total=1)
+    pyc = ProgressYnabClient(YnabClient(TOKEN, context.session), context, task_id)
+    entries = await pyc("example")
 
     assert entries == expected
 
@@ -650,8 +734,8 @@ async def test_sync_no_data(tmp_path, mock_aioresponses):
 
     # create the db and tables to exercise all code branches
     db = tmp_path / "db.sqlite"
-    with sqlite3.connect(db) as con:
-        con.executescript(contents("create-relations.sql"))
+    async with aiosqlite.connect(db) as con:
+        await con.executescript(await contents("create-relations.sql"))
 
     await sync(TOKEN, db, False)
 
@@ -694,8 +778,8 @@ async def test_sync_no_data_quiet(tmp_path, mock_aioresponses, capsys):
     )
 
     db = tmp_path / "db.sqlite"
-    with sqlite3.connect(db) as con:
-        con.executescript(contents("create-relations.sql"))
+    async with aiosqlite.connect(db) as con:
+        await con.executescript(await contents("create-relations.sql"))
 
     await sync(TOKEN, db, False, quiet=True)
 

sqlite_export_for_ynab-2.3.0/sqlite_export_for_ynab.egg-info/requires.txt

@@ -1,2 +0,0 @@
-aiohttp>=3
-tldm