sqlite-export-for-ynab 2.2.0__tar.gz → 2.4.0__tar.gz

{sqlite_export_for_ynab-2.2.0/sqlite_export_for_ynab.egg-info → sqlite_export_for_ynab-2.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlite_export_for_ynab
-Version: 2.2.0
+Version: 2.4.0
 Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
 Home-page: https://github.com/mxr/sqlite-export-for-ynab
 Author: Max R
@@ -15,6 +15,7 @@ Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp>=3
+Requires-Dist: aiosqlite
 Requires-Dist: tldm
 Dynamic: license-file
 
@@ -82,7 +83,7 @@ The relations are defined in [create-relations.sql](sqlite_export_for_ynab/ddl/c
 
 1. Some objects are pulled out into their own tables so they can be more cleanly modeled in SQLite (ex: subtransactions, loan account periodic values).
 1. Foreign keys are added as needed (ex: plan ID, transaction ID) so data across plans remains separate.
-1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted transactions/subtransactions and project payee/category fields to make querying more ergonomic.
+1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted/unapproved transactions/subtransactions and project payee/category fields to make querying more ergonomic.
 
 ## Querying
 
@@ -136,6 +137,7 @@ WITH used_payees AS (
     FROM transactions
     WHERE
         TRUE
+        AND approved
         AND payee_id IS NOT NULL
         AND NOT deleted
     UNION

{sqlite_export_for_ynab-2.2.0 → sqlite_export_for_ynab-2.4.0}/README.md

@@ -62,7 +62,7 @@ The relations are defined in [create-relations.sql](sqlite_export_for_ynab/ddl/c
 
 1. Some objects are pulled out into their own tables so they can be more cleanly modeled in SQLite (ex: subtransactions, loan account periodic values).
 1. Foreign keys are added as needed (ex: plan ID, transaction ID) so data across plans remains separate.
-1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted transactions/subtransactions and project payee/category fields to make querying more ergonomic.
+1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted/unapproved transactions/subtransactions and project payee/category fields to make querying more ergonomic.
 
 ## Querying
 
@@ -116,6 +116,7 @@ WITH used_payees AS (
     FROM transactions
     WHERE
         TRUE
+        AND approved
         AND payee_id IS NOT NULL
         AND NOT deleted
     UNION

{sqlite_export_for_ynab-2.2.0 → sqlite_export_for_ynab-2.4.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = sqlite_export_for_ynab
-version = 2.2.0
+version = 2.4.0
 description = SQLite Export for YNAB - Export YNAB Data to SQLite
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -20,6 +20,7 @@ keywords = ynab, sqlite, sql, budget, plan, cli
 packages = find:
 install_requires = 
 	aiohttp>=3
+	aiosqlite
 	tldm
 python_requires = >=3.12
 

{sqlite_export_for_ynab-2.2.0 → sqlite_export_for_ynab-2.4.0}/sqlite_export_for_ynab/_main.py

@@ -4,10 +4,10 @@ import argparse
 import asyncio
 import json
 import os
-import sqlite3
 from dataclasses import dataclass
 from importlib import resources
 from importlib.metadata import version
+from itertools import batched
 from pathlib import Path
 from typing import Any
 from typing import ClassVar
@@ -20,6 +20,7 @@ from urllib.parse import urljoin
 from urllib.parse import urlunparse
 
 import aiohttp
+import aiosqlite
 from tldm import tldm
 
 from sqlite_export_for_ynab import ddl
@@ -40,6 +41,14 @@ _EntryTable = (
     | Literal["scheduled_transactions"]
     | Literal["scheduled_subtransactions"]
 )
+_Endpoint = (
+    Literal["accounts"]
+    | Literal["categories"]
+    | Literal["payees"]
+    | Literal["transactions"]
+    | Literal["scheduled_transactions"]
+)
+_ENDPOINTS = tuple(lit.__args__[0] for lit in _Endpoint.__args__)
 _ALL_RELATIONS = frozenset(
     ("plans", "flat_transactions", "scheduled_flat_transactions")
     + tuple(lit.__args__[0] for lit in _EntryTable.__args__)
@@ -49,6 +58,8 @@ _ENV_TOKEN = "YNAB_PERSONAL_ACCESS_TOKEN"
 
 _PACKAGE = "sqlite-export-for-ynab"
 
+_BATCH_SIZE = 100
+
 
 def resolve_token(token_override: str | None = None) -> str:
     token = token_override or os.environ.get(_ENV_TOKEN)
@@ -124,49 +135,51 @@ async def sync(
     if not db.exists():
         db.parent.mkdir(parents=True, exist_ok=True)
 
-    with sqlite3.connect(db) as con:
-        con.row_factory = sqlite3.Row
-        cur = con.cursor()
+    async with aiosqlite.connect(db) as con:
+        con.row_factory = aiosqlite.Row
 
         if full_refresh:
             _print("Dropping relations...", quiet=quiet)
-            cur.executescript(contents("drop-relations.sql"))
-            con.commit()
+            async with con.cursor() as cur:
+                await cur.executescript(contents("drop-relations.sql"))
+            await con.commit()
             _print("Done", quiet=quiet)
 
-        relations = get_relations(cur)
-        if relations != _ALL_RELATIONS:
-            _print("Recreating relations...", quiet=quiet)
-            cur.executescript(contents("create-relations.sql"))
-            con.commit()
-            _print("Done", quiet=quiet)
+        async with con.cursor() as cur:
+            relations = await get_relations(cur)
+            if relations != _ALL_RELATIONS:
+                _print("Recreating relations...", quiet=quiet)
+                await cur.executescript(contents("create-relations.sql"))
+                await con.commit()
+                _print("Done", quiet=quiet)
 
         _print("Fetching plan data...", quiet=quiet)
-        lkos = get_last_knowledge_of_server(cur)
+        async with con.cursor() as cur:
+            lkos = await get_last_knowledge_of_server(cur)
         async with aiohttp.ClientSession() as session:
-            with tldm(desc="Plan Data", total=len(plans) * 5, disable=quiet) as pbar:
+            with tldm(
+                desc="Plan Data", total=len(plans) * len(_ENDPOINTS), disable=quiet
+            ) as pbar:
                 yc = ProgressYnabClient(YnabClient(token, session), pbar)
 
-                account_jobs = jobs(yc, "accounts", plan_ids, lkos)
-                cat_jobs = jobs(yc, "categories", plan_ids, lkos)
-                payee_jobs = jobs(yc, "payees", plan_ids, lkos)
-                txn_jobs = jobs(yc, "transactions", plan_ids, lkos)
-                sched_txn_jobs = jobs(yc, "scheduled_transactions", plan_ids, lkos)
-
-                data = await asyncio.gather(
-                    *account_jobs, *cat_jobs, *payee_jobs, *txn_jobs, *sched_txn_jobs
+                endpoint_data = dict(
+                    zip(
+                        _ENDPOINTS,
+                        await asyncio.gather(
+                            *(
+                                asyncio.gather(*jobs(yc, endpoint, plan_ids, lkos))
+                                for endpoint in _ENDPOINTS
+                            )
+                        ),
+                        strict=True,
+                    )
                 )
 
-            la = len(account_jobs)
-            lc = len(cat_jobs)
-            lp = len(payee_jobs)
-            lt = len(txn_jobs)
-
-            all_account_data = data[:la]
-            all_cat_data = data[la : la + lc]
-            all_payee_data = data[la + lc : la + lc + lp]
-            all_txn_data = data[la + lc + lp : la + lc + lp + lt]
-            all_sched_txn_data = data[la + lc + lp + lt :]
+            all_account_data = endpoint_data["accounts"]
+            all_cat_data = endpoint_data["categories"]
+            all_payee_data = endpoint_data["payees"]
+            all_txn_data = endpoint_data["transactions"]
+            all_sched_txn_data = endpoint_data["scheduled_transactions"]
 
             new_lkos = {
                 plan_id: transaction_data["server_knowledge"]
@@ -186,23 +199,58 @@ async def sync(
             _print("No new data fetched", quiet=quiet)
         else:
             _print("Inserting plan data...", quiet=quiet)
-            insert_plans(cur, plans, new_lkos)
-            for plan_id, account_data in zip(plan_ids, all_account_data, strict=True):
-                insert_accounts(cur, plan_id, account_data["accounts"], quiet=quiet)
-            for plan_id, cat_data in zip(plan_ids, all_cat_data, strict=True):
-                insert_category_groups(
-                    cur, plan_id, cat_data["category_groups"], quiet=quiet
-                )
-            for plan_id, payee_data in zip(plan_ids, all_payee_data, strict=True):
-                insert_payees(cur, plan_id, payee_data["payees"], quiet=quiet)
-            for plan_id, txn_data in zip(plan_ids, all_txn_data, strict=True):
-                insert_transactions(cur, plan_id, txn_data["transactions"], quiet=quiet)
-            for plan_id, sched_txn_data in zip(
-                plan_ids, all_sched_txn_data, strict=True
-            ):
-                insert_scheduled_transactions(
-                    cur, plan_id, sched_txn_data["scheduled_transactions"], quiet=quiet
-                )
+            await insert_plans(con, plans, new_lkos)
+            await asyncio.gather(
+                *(
+                    insert_accounts(
+                        con,
+                        plan_id,
+                        account_data["accounts"],
+                        quiet=quiet,
+                    )
+                    for plan_id, account_data in zip(
+                        plan_ids, all_account_data, strict=True
+                    )
+                ),
+                *(
+                    insert_category_groups(
+                        con,
+                        plan_id,
+                        cat_data["category_groups"],
+                        quiet=quiet,
+                    )
+                    for plan_id, cat_data in zip(plan_ids, all_cat_data, strict=True)
+                ),
+                *(
+                    insert_payees(con, plan_id, payee_data["payees"], quiet=quiet)
+                    for plan_id, payee_data in zip(
+                        plan_ids, all_payee_data, strict=True
+                    )
+                ),
+            )
+            await asyncio.gather(
+                *(
+                    insert_transactions(
+                        con,
+                        plan_id,
+                        txn_data["transactions"],
+                        quiet=quiet,
+                    )
+                    for plan_id, txn_data in zip(plan_ids, all_txn_data, strict=True)
+                ),
+                *(
+                    insert_scheduled_transactions(
+                        con,
+                        plan_id,
+                        sched_txn_data["scheduled_transactions"],
+                        quiet=quiet,
+                    )
+                    for plan_id, sched_txn_data in zip(
+                        plan_ids, all_sched_txn_data, strict=True
+                    )
+                ),
+            )
+            await con.commit()
             _print("Done", quiet=quiet)
 
 
@@ -210,58 +258,56 @@ def contents(filename: str) -> str:
     return (resources.files(ddl) / filename).read_text()
 
 
-def get_relations(cur: sqlite3.Cursor) -> set[str]:
-    return {
-        t["name"]
-        for t in cur.execute(
-            "SELECT name FROM sqlite_master WHERE type='table' OR type='view'"
-        ).fetchall()
-    }
+async def get_relations(cur: aiosqlite.Cursor) -> set[str]:
+    await cur.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' OR type='view'"
+    )
+    return {t["name"] for t in await cur.fetchall()}
 
 
-def get_last_knowledge_of_server(cur: sqlite3.Cursor) -> dict[str, int]:
-    return {
-        r["id"]: r["last_knowledge_of_server"]
-        for r in cur.execute(
-            "SELECT id, last_knowledge_of_server FROM plans",
-        ).fetchall()
-    }
+async def get_last_knowledge_of_server(cur: aiosqlite.Cursor) -> dict[str, int]:
+    await cur.execute(
+        "SELECT id, last_knowledge_of_server FROM plans",
+    )
+    return {r["id"]: r["last_knowledge_of_server"] for r in await cur.fetchall()}
 
 
-def insert_plans(
-    cur: sqlite3.Cursor, plans: list[dict[str, Any]], lkos: dict[str, int]
+async def insert_plans(
+    con: aiosqlite.Connection, plans: list[dict[str, Any]], lkos: dict[str, int]
 ) -> None:
-    cur.executemany(
-        """
-        INSERT OR REPLACE INTO plans (
-            id
-            , name
-            , currency_format_currency_symbol
-            , currency_format_decimal_digits
-            , currency_format_decimal_separator
-            , currency_format_display_symbol
-            , currency_format_group_separator
-            , currency_format_iso_code
-            , currency_format_symbol_first
-            , last_knowledge_of_server
-        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        """,
-        (
-            (
-                plan_id := plan["id"],
-                plan["name"],
-                plan["currency_format"]["currency_symbol"],
-                plan["currency_format"]["decimal_digits"],
-                plan["currency_format"]["decimal_separator"],
-                plan["currency_format"]["display_symbol"],
-                plan["currency_format"]["group_separator"],
-                plan["currency_format"]["iso_code"],
-                plan["currency_format"]["symbol_first"],
-                lkos[plan_id],
+    async with con.cursor() as cur:
+        for plan_batch in batched(plans, _BATCH_SIZE):
+            await cur.executemany(
+                """
+                INSERT OR REPLACE INTO plans (
+                    id
+                    , name
+                    , currency_format_currency_symbol
+                    , currency_format_decimal_digits
+                    , currency_format_decimal_separator
+                    , currency_format_display_symbol
+                    , currency_format_group_separator
+                    , currency_format_iso_code
+                    , currency_format_symbol_first
+                    , last_knowledge_of_server
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    (
+                        plan_id := plan["id"],
+                        plan["name"],
+                        plan["currency_format"]["currency_symbol"],
+                        plan["currency_format"]["decimal_digits"],
+                        plan["currency_format"]["decimal_separator"],
+                        plan["currency_format"]["display_symbol"],
+                        plan["currency_format"]["group_separator"],
+                        plan["currency_format"]["iso_code"],
+                        plan["currency_format"]["symbol_first"],
+                        lkos[plan_id],
+                    )
+                    for plan in plan_batch
+                ),
             )
-            for plan in plans
-        ),
-    )
 
 
 _LOAN_ACCOUNT_PERIODIC_VALUES = frozenset(
@@ -269,8 +315,8 @@ _LOAN_ACCOUNT_PERIODIC_VALUES = frozenset(
 )
 
 
-def insert_accounts(
-    cur: sqlite3.Cursor,
+async def insert_accounts(
+    con: aiosqlite.Connection,
     plan_id: str,
     accounts: list[dict[str, Any]],
     *,
@@ -294,8 +340,8 @@ def insert_accounts(
         for account in accounts
     ]
 
-    return insert_nested_entries(
-        cur,
+    await insert_nested_entries(
+        con,
         plan_id,
         updated_accounts,
         "Accounts",
@@ -306,15 +352,15 @@ def insert_accounts(
     )
 
 
-def insert_category_groups(
-    cur: sqlite3.Cursor,
+async def insert_category_groups(
+    con: aiosqlite.Connection,
     plan_id: str,
     category_groups: list[dict[str, Any]],
     *,
     quiet: bool = False,
 ) -> None:
-    return insert_nested_entries(
-        cur,
+    await insert_nested_entries(
+        con,
         plan_id,
         category_groups,
         "Categories",
@@ -325,8 +371,8 @@ def insert_category_groups(
     )
 
 
-def insert_payees(
-    cur: sqlite3.Cursor,
+async def insert_payees(
+    con: aiosqlite.Connection,
     plan_id: str,
     payees: list[dict[str, Any]],
     *,
@@ -335,19 +381,19 @@ def insert_payees(
     if not payees:
         return
 
-    for payee in tldm(payees, desc="Payees", disable=quiet):
-        insert_entry(cur, "payees", plan_id, payee)
+    with tldm(total=len(payees), desc="Payees", disable=quiet) as pbar:
+        await insert_entries(con, "payees", plan_id, payees, pbar)
 
 
-def insert_transactions(
-    cur: sqlite3.Cursor,
+async def insert_transactions(
+    con: aiosqlite.Connection,
     plan_id: str,
     transactions: list[dict[str, Any]],
     *,
     quiet: bool = False,
 ) -> None:
-    return insert_nested_entries(
-        cur,
+    await insert_nested_entries(
+        con,
         plan_id,
         transactions,
         "Transactions",
@@ -358,15 +404,15 @@ def insert_transactions(
     )
 
 
-def insert_scheduled_transactions(
-    cur: sqlite3.Cursor,
+async def insert_scheduled_transactions(
+    con: aiosqlite.Connection,
     plan_id: str,
     scheduled_transactions: list[dict[str, Any]],
     *,
     quiet: bool = False,
 ) -> None:
-    return insert_nested_entries(
-        cur,
+    await insert_nested_entries(
+        con,
         plan_id,
         scheduled_transactions,
         "Scheduled Transactions",
@@ -378,8 +424,8 @@ def insert_scheduled_transactions(
 
 
 @overload
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    con: aiosqlite.Connection,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: Literal["Accounts"],
@@ -392,8 +438,8 @@ def insert_nested_entries(
 
 
 @overload
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    con: aiosqlite.Connection,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: Literal["Categories"],
@@ -406,8 +452,8 @@ def insert_nested_entries(
 
 
 @overload
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    con: aiosqlite.Connection,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: Literal["Transactions"],
@@ -420,8 +466,8 @@ def insert_nested_entries(
 
 
 @overload
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    con: aiosqlite.Connection,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: Literal["Scheduled Transactions"],
@@ -433,8 +479,8 @@ def insert_nested_entries(
 ) -> None: ...
 
 
-def insert_nested_entries(
-    cur: sqlite3.Cursor,
+async def insert_nested_entries(
+    con: aiosqlite.Connection,
     plan_id: str,
     entries: list[dict[str, Any]],
     desc: (
@@ -471,44 +517,51 @@ def insert_nested_entries(
         desc=desc,
         disable=quiet,
     ) as pbar:
-        for entry in entries:
-            insert_entry(
-                cur,
-                entries_name,
-                plan_id,
-                {k: v for k, v in entry.items() if k != subentries_name},
-            )
-            pbar.update()
-
-            for subentry in entry[subentries_name]:
-                insert_entry(cur, subentries_table_name, plan_id, subentry)
-                pbar.update()
+        await insert_entries(
+            con,
+            entries_name,
+            plan_id,
+            [
+                {k: v for k, v in entry.items() if k != subentries_name}
+                for entry in entries
+            ],
+            pbar,
+        )
+        await insert_entries(
+            con,
+            subentries_table_name,
+            plan_id,
+            [subentry for entry in entries for subentry in entry[subentries_name]],
+            pbar,
+        )
 
 
-def insert_entry(
-    cur: sqlite3.Cursor,
+async def insert_entries(
+    con: aiosqlite.Connection,
     table: _EntryTable,
     plan_id: str,
-    entry: dict[str, Any],
+    entries: list[dict[str, Any]],
+    pbar: tldm[Never],
 ) -> None:
-    ekeys, evalues = zip(*entry.items(), strict=True)
-    keys, values = ekeys + ("plan_id",), evalues + (plan_id,)
+    if not entries:
+        return
 
-    cur.execute(
-        f"INSERT OR REPLACE INTO {table} ({', '.join(keys)}) VALUES ({', '.join('?' * len(values))})",
-        values,
-    )
+    entry_keys = tuple(entries[0])
+    sql = f"INSERT OR REPLACE INTO {table} ({', '.join(entry_keys + ('plan_id',))}) VALUES ({', '.join('?' * (len(entry_keys) + 1))})"
+
+    async with con.cursor() as cur:
+        for entry_batch in batched(entries, _BATCH_SIZE):
+            values_batch = [
+                tuple(entry[key] for key in entry_keys) + (plan_id,)
+                for entry in entry_batch
+            ]
+            await cur.executemany(sql, values_batch)
+            pbar.update(len(values_batch))
 
 
 def jobs(
     yc: SupportsYnabClient,
-    endpoint: (
-        Literal["accounts"]
-        | Literal["categories"]
-        | Literal["payees"]
-        | Literal["transactions"]
-        | Literal["scheduled_transactions"]
-    ),
+    endpoint: _Endpoint,
     plan_ids: list[str],
     lkos: dict[str, int],
 ) -> list[Awaitable[dict[str, Any]]]:

{sqlite_export_for_ynab-2.2.0 → sqlite_export_for_ynab-2.4.0}/sqlite_export_for_ynab/ddl/create-relations.sql

@@ -181,7 +181,6 @@ SELECT
     , t.plan_id
     , t.account_id
     , t.account_name
-    , t.approved
     , t.cleared
     , t."date"
     , t.debt_transaction_type
@@ -227,6 +226,7 @@ INNER JOIN categories AS c
     )
 WHERE
     TRUE
+    AND t.approved
     AND NOT COALESCE(st.deleted, t.deleted)
 ;
 

{sqlite_export_for_ynab-2.2.0 → sqlite_export_for_ynab-2.4.0/sqlite_export_for_ynab.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlite_export_for_ynab
-Version: 2.2.0
+Version: 2.4.0
 Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
 Home-page: https://github.com/mxr/sqlite-export-for-ynab
 Author: Max R
@@ -15,6 +15,7 @@ Requires-Python: >=3.12
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: aiohttp>=3
+Requires-Dist: aiosqlite
 Requires-Dist: tldm
 Dynamic: license-file
 
@@ -82,7 +83,7 @@ The relations are defined in [create-relations.sql](sqlite_export_for_ynab/ddl/c
 
 1. Some objects are pulled out into their own tables so they can be more cleanly modeled in SQLite (ex: subtransactions, loan account periodic values).
 1. Foreign keys are added as needed (ex: plan ID, transaction ID) so data across plans remains separate.
-1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted transactions/subtransactions and project payee/category fields to make querying more ergonomic.
+1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted/unapproved transactions/subtransactions and project payee/category fields to make querying more ergonomic.
 
 ## Querying
 
@@ -136,6 +137,7 @@ WITH used_payees AS (
     FROM transactions
     WHERE
         TRUE
+        AND approved
         AND payee_id IS NOT NULL
         AND NOT deleted
     UNION

{sqlite_export_for_ynab-2.2.0 → sqlite_export_for_ynab-2.4.0}/sqlite_export_for_ynab.egg-info/requires.txt

@@ -1,2 +1,3 @@
 aiohttp>=3
+aiosqlite
 tldm

{sqlite_export_for_ynab-2.2.0 → sqlite_export_for_ynab-2.4.0}/testing/fixtures.py

@@ -1,11 +1,12 @@
 from __future__ import annotations
 
 import re
-import sqlite3
 from typing import Any
 from uuid import uuid4
 
+import aiosqlite
 import pytest
+import pytest_asyncio
 from aioresponses import aioresponses
 
 from sqlite_export_for_ynab._main import contents
@@ -144,6 +145,15 @@ CATEGORY_GROUPS: list[dict[str, Any]] = [
                 "activity_currency": 1.0,
                 "budgeted_formatted": "$10.25",
                 "budgeted_currency": 10.25,
+                "goal_target_formatted": None,
+                "goal_target_currency": None,
+                "goal_under_funded_formatted": None,
+                "goal_under_funded_currency": None,
+                "goal_overall_funded_formatted": None,
+                "goal_overall_funded_currency": None,
+                "goal_overall_left_formatted": None,
+                "goal_overall_left_currency": None,
+                "goal_target_date": None,
             },
         ],
     },
@@ -162,6 +172,15 @@ CATEGORY_GROUPS: list[dict[str, Any]] = [
                 "activity_currency": 7.5,
                 "budgeted_formatted": "$15.00",
                 "budgeted_currency": 15.0,
+                "goal_target_formatted": None,
+                "goal_target_currency": None,
+                "goal_under_funded_formatted": None,
+                "goal_under_funded_currency": None,
+                "goal_overall_funded_formatted": None,
+                "goal_overall_funded_currency": None,
+                "goal_overall_left_formatted": None,
+                "goal_overall_left_currency": None,
+                "goal_target_date": None,
             },
             {
                 "id": CATEGORY_ID_4,
@@ -174,6 +193,15 @@ CATEGORY_GROUPS: list[dict[str, Any]] = [
                 "activity_currency": 19.0,
                 "budgeted_formatted": "$20.00",
                 "budgeted_currency": 20.0,
+                "goal_target_formatted": None,
+                "goal_target_currency": None,
+                "goal_under_funded_formatted": None,
+                "goal_under_funded_currency": None,
+                "goal_overall_funded_formatted": None,
+                "goal_overall_funded_currency": None,
+                "goal_overall_left_formatted": None,
+                "goal_overall_left_currency": None,
+                "goal_target_date": None,
             },
         ],
     },
@@ -207,6 +235,7 @@ TRANSACTIONS: list[dict[str, Any]] = [
         "amount": -10000,
         "amount_formatted": "$10.00",
         "amount_currency": 10.0,
+        "approved": True,
         "category_id": CATEGORY_ID_3,
         "category_name": CATEGORY_NAME_3,
         "deleted": False,
@@ -239,6 +268,7 @@ TRANSACTIONS: list[dict[str, Any]] = [
         "amount": -15000,
         "amount_formatted": "$15.00",
         "amount_currency": 15.0,
+        "approved": True,
         "category_id": CATEGORY_ID_2,
         "category_name": CATEGORY_NAME_2,
         "deleted": True,
@@ -250,6 +280,7 @@ TRANSACTIONS: list[dict[str, Any]] = [
         "amount": -19000,
         "amount_formatted": "$19.00",
         "amount_currency": 19.0,
+        "approved": False,
         "category_id": CATEGORY_ID_4,
         "category_name": CATEGORY_NAME_4,
         "deleted": False,
@@ -322,12 +353,17 @@ SCHEDULED_TRANSACTIONS: list[dict[str, Any]] = [
 ]
 
 
-@pytest.fixture
-def cur():
-    with sqlite3.connect(":memory:") as con:
-        con.row_factory = sqlite3.Row
-        cursor = con.cursor()
-        cursor.executescript(contents("create-relations.sql"))
+@pytest_asyncio.fixture
+async def con():
+    async with aiosqlite.connect(":memory:") as con:
+        con.row_factory = aiosqlite.Row
+        await con.executescript(contents("create-relations.sql"))
+        yield con
+
+
+@pytest_asyncio.fixture
+async def cur(con):
+    async with con.cursor() as cursor:
         yield cursor
 
 
@@ -337,7 +373,7 @@ def mock_aioresponses():
         yield m
 
 
-def strip_nones(d: sqlite3.Row) -> dict[str, Any]:
+def strip_nones(d: aiosqlite.Row) -> dict[str, Any]:
     return {k: v for k, v in dict(d).items() if v is not None}
 
 

{sqlite_export_for_ynab-2.2.0 → sqlite_export_for_ynab-2.4.0}/tests/_main_test.py

@@ -2,12 +2,12 @@ from __future__ import annotations
 
 import asyncio
 import json
-import sqlite3
 from configparser import ConfigParser
 from pathlib import Path
 from unittest.mock import patch
 
 import aiohttp
+import aiosqlite
 import pytest
 from aiohttp.http_exceptions import HttpProcessingError
 from tldm import tldm
@@ -49,6 +49,7 @@ from testing.fixtures import CATEGORY_NAME_1
 from testing.fixtures import CATEGORY_NAME_2
 from testing.fixtures import CATEGORY_NAME_3
 from testing.fixtures import CATEGORY_NAME_4
+from testing.fixtures import con
 from testing.fixtures import cur
 from testing.fixtures import EXAMPLE_ENDPOINT_RE
 from testing.fixtures import LKOS
@@ -80,6 +81,12 @@ from testing.fixtures import TRANSACTIONS
 from testing.fixtures import TRANSACTIONS_ENDPOINT_RE
 
 
+async def fetchall(con, query):
+    async with con.cursor() as cur:
+        await cur.execute(query)
+        return await cur.fetchall()
+
+
 @pytest.mark.parametrize(
     ("xdg_data_home", "expected_prefix"),
     (
@@ -93,21 +100,26 @@ def test_default_db_path(monkeypatch, xdg_data_home, expected_prefix):
 
 
 @pytest.mark.usefixtures(cur.__name__)
-def test_get_relations(cur):
-    assert get_relations(cur) == _ALL_RELATIONS
+@pytest.mark.asyncio
+async def test_get_relations(cur):
+    assert await get_relations(cur) == _ALL_RELATIONS
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_get_last_knowledge_of_server(cur):
-    insert_plans(cur, PLANS, LKOS)
-    assert get_last_knowledge_of_server(cur) == LKOS
+@pytest.mark.usefixtures(con.__name__)
+@pytest.mark.asyncio
+async def test_get_last_knowledge_of_server(con):
+    await insert_plans(con, PLANS, LKOS)
+    async with con.cursor() as cur:
+        assert await get_last_knowledge_of_server(cur) == LKOS
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_plans(cur):
-    insert_plans(cur, PLANS, LKOS)
-    cur.execute("SELECT * FROM plans ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.usefixtures(con.__name__)
+@pytest.mark.asyncio
+async def test_insert_plans(con):
+    await insert_plans(con, PLANS, LKOS)
+    assert [
+        strip_nones(d) for d in await fetchall(con, "SELECT * FROM plans ORDER BY name")
+    ] == [
         {
             "id": PLAN_ID_1,
             "name": PLANS[0]["name"],
@@ -135,15 +147,18 @@ def test_insert_plans(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_accounts(cur):
-    insert_accounts(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM accounts").fetchall()
-    assert not cur.execute("SELECT * FROM account_periodic_values").fetchall()
-
-    insert_accounts(cur, PLAN_ID_1, ACCOUNTS)
-    cur.execute("SELECT * FROM accounts ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.usefixtures(con.__name__)
+@pytest.mark.asyncio
+async def test_insert_accounts(con):
+    await insert_accounts(con, PLAN_ID_1, [])
+    assert not await fetchall(con, "SELECT * FROM accounts")
+    assert not await fetchall(con, "SELECT * FROM account_periodic_values")
+
+    await insert_accounts(con, PLAN_ID_1, ACCOUNTS)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(con, "SELECT * FROM accounts ORDER BY name")
+    ] == [
         {
             "id": ACCOUNT_ID_1,
             "plan_id": PLAN_ID_1,
@@ -170,8 +185,12 @@ def test_insert_accounts(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM account_periodic_values ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            con, "SELECT * FROM account_periodic_values ORDER BY name"
+        )
+    ] == [
         {
             "account_id": ACCOUNT_ID_1,
             "plan_id": PLAN_ID_1,
@@ -189,15 +208,18 @@ def test_insert_accounts(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_category_groups(cur):
-    insert_category_groups(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM category_groups").fetchall()
-    assert not cur.execute("SELECT * FROM categories").fetchall()
-
-    insert_category_groups(cur, PLAN_ID_1, CATEGORY_GROUPS)
-    cur.execute("SELECT * FROM category_groups ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.usefixtures(con.__name__)
+@pytest.mark.asyncio
+async def test_insert_category_groups(con):
+    await insert_category_groups(con, PLAN_ID_1, [])
+    assert not await fetchall(con, "SELECT * FROM category_groups")
+    assert not await fetchall(con, "SELECT * FROM categories")
+
+    await insert_category_groups(con, PLAN_ID_1, CATEGORY_GROUPS)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(con, "SELECT * FROM category_groups ORDER BY name")
+    ] == [
         {
             "id": CATEGORY_GROUP_ID_1,
             "name": CATEGORY_GROUP_NAME_1,
@@ -210,8 +232,10 @@ def test_insert_category_groups(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM categories ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(con, "SELECT * FROM categories ORDER BY name")
+    ] == [
         {
             "id": CATEGORY_ID_1,
             "category_group_id": CATEGORY_GROUP_ID_1,
@@ -276,14 +300,41 @@ def test_insert_category_groups(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_payees(cur):
-    insert_payees(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM payees").fetchall()
+@pytest.mark.usefixtures(con.__name__)
+@pytest.mark.asyncio
+async def test_insert_category_group_without_categories(con):
+    category_group = {
+        "id": CATEGORY_GROUP_ID_1,
+        "name": CATEGORY_GROUP_NAME_1,
+        "categories": [],
+    }
+
+    await insert_category_groups(con, PLAN_ID_1, [category_group])
+
+    assert [
+        strip_nones(d)
+        for d in await fetchall(con, "SELECT * FROM category_groups ORDER BY name")
+    ] == [
+        {
+            "id": CATEGORY_GROUP_ID_1,
+            "name": CATEGORY_GROUP_NAME_1,
+            "plan_id": PLAN_ID_1,
+        },
+    ]
+    assert not await fetchall(con, "SELECT * FROM categories")
 
-    insert_payees(cur, PLAN_ID_1, PAYEES)
-    cur.execute("SELECT * FROM payees ORDER BY name")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+
+@pytest.mark.usefixtures(con.__name__)
+@pytest.mark.asyncio
+async def test_insert_payees(con):
+    await insert_payees(con, PLAN_ID_1, [])
+    assert not await fetchall(con, "SELECT * FROM payees")
+
+    await insert_payees(con, PLAN_ID_1, PAYEES)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(con, "SELECT * FROM payees ORDER BY name")
+    ] == [
         {
             "id": PAYEE_ID_1,
             "plan_id": PLAN_ID_1,
@@ -297,16 +348,19 @@ def test_insert_payees(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_transactions(cur):
-    insert_transactions(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM transactions").fetchall()
-    assert not cur.execute("SELECT * FROM subtransactions").fetchall()
-
-    insert_category_groups(cur, PLAN_ID_1, CATEGORY_GROUPS)
-    insert_transactions(cur, PLAN_ID_1, TRANSACTIONS)
-    cur.execute("SELECT * FROM transactions ORDER BY date")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.usefixtures(con.__name__)
+@pytest.mark.asyncio
+async def test_insert_transactions(con):
+    await insert_transactions(con, PLAN_ID_1, [])
+    assert not await fetchall(con, "SELECT * FROM transactions")
+    assert not await fetchall(con, "SELECT * FROM subtransactions")
+
+    await insert_category_groups(con, PLAN_ID_1, CATEGORY_GROUPS)
+    await insert_transactions(con, PLAN_ID_1, TRANSACTIONS)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(con, "SELECT * FROM transactions ORDER BY date")
+    ] == [
         {
             "id": TRANSACTION_ID_1,
             "plan_id": PLAN_ID_1,
@@ -314,6 +368,7 @@ def test_insert_transactions(cur):
             "amount": -10000,
             "amount_formatted": "$10.00",
             "amount_currency": 10.0,
+            "approved": 1,
             "category_id": CATEGORY_ID_3,
             "category_name": CATEGORY_NAME_3,
             "deleted": False,
@@ -325,6 +380,7 @@ def test_insert_transactions(cur):
             "amount": -15000,
             "amount_formatted": "$15.00",
             "amount_currency": 15.0,
+            "approved": 1,
             "category_id": CATEGORY_ID_2,
             "category_name": CATEGORY_NAME_2,
             "deleted": True,
@@ -336,14 +392,17 @@ def test_insert_transactions(cur):
             "amount": -19000,
             "amount_formatted": "$19.00",
             "amount_currency": 19.0,
+            "approved": 0,
             "category_id": CATEGORY_ID_4,
             "category_name": CATEGORY_NAME_4,
             "deleted": False,
         },
     ]
 
-    cur.execute("SELECT * FROM subtransactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(con, "SELECT * FROM subtransactions ORDER BY amount")
+    ] == [
         {
             "id": SUBTRANSACTION_ID_1,
             "transaction_id": TRANSACTION_ID_1,
@@ -368,21 +427,10 @@ def test_insert_transactions(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM flat_transactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
-        {
-            "transaction_id": TRANSACTION_ID_3,
-            "plan_id": PLAN_ID_1,
-            "date": "2024-03-01",
-            "id": TRANSACTION_ID_3,
-            "amount": -19000,
-            "amount_formatted": "$19.00",
-            "amount_currency": 19.0,
-            "category_id": CATEGORY_ID_4,
-            "category_name": CATEGORY_NAME_4,
-            "category_group_id": CATEGORY_GROUP_ID_2,
-            "category_group_name": CATEGORY_GROUP_NAME_2,
-        },
+    assert [
+        strip_nones(d)
+        for d in await fetchall(con, "SELECT * FROM flat_transactions ORDER BY amount")
+    ] == [
         {
             "transaction_id": TRANSACTION_ID_1,
             "subtransaction_id": SUBTRANSACTION_ID_1,
@@ -414,16 +462,21 @@ def test_insert_transactions(cur):
     ]
 
 
-@pytest.mark.usefixtures(cur.__name__)
-def test_insert_scheduled_transactions(cur):
-    insert_scheduled_transactions(cur, PLAN_ID_1, [])
-    assert not cur.execute("SELECT * FROM scheduled_transactions").fetchall()
-    assert not cur.execute("SELECT * FROM scheduled_subtransactions").fetchall()
-
-    insert_category_groups(cur, PLAN_ID_1, CATEGORY_GROUPS)
-    insert_scheduled_transactions(cur, PLAN_ID_1, SCHEDULED_TRANSACTIONS)
-    cur.execute("SELECT * FROM scheduled_transactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+@pytest.mark.usefixtures(con.__name__)
+@pytest.mark.asyncio
+async def test_insert_scheduled_transactions(con):
+    await insert_scheduled_transactions(con, PLAN_ID_1, [])
+    assert not await fetchall(con, "SELECT * FROM scheduled_transactions")
+    assert not await fetchall(con, "SELECT * FROM scheduled_subtransactions")
+
+    await insert_category_groups(con, PLAN_ID_1, CATEGORY_GROUPS)
+    await insert_scheduled_transactions(con, PLAN_ID_1, SCHEDULED_TRANSACTIONS)
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            con, "SELECT * FROM scheduled_transactions ORDER BY amount"
+        )
+    ] == [
         {
             "id": SCHEDULED_TRANSACTION_ID_1,
             "plan_id": PLAN_ID_1,
@@ -459,8 +512,12 @@ def test_insert_scheduled_transactions(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM scheduled_subtransactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            con, "SELECT * FROM scheduled_subtransactions ORDER BY amount"
+        )
+    ] == [
         {
             "id": SCHEDULED_SUBTRANSACTION_ID_1,
             "scheduled_transaction_id": SCHEDULED_TRANSACTION_ID_1,
@@ -485,8 +542,12 @@ def test_insert_scheduled_transactions(cur):
         },
     ]
 
-    cur.execute("SELECT * FROM scheduled_flat_transactions ORDER BY amount")
-    assert [strip_nones(d) for d in cur.fetchall()] == [
+    assert [
+        strip_nones(d)
+        for d in await fetchall(
+            con, "SELECT * FROM scheduled_flat_transactions ORDER BY amount"
+        )
+    ] == [
         {
             "transaction_id": SCHEDULED_TRANSACTION_ID_3,
             "plan_id": PLAN_ID_1,
@@ -660,8 +721,8 @@ async def test_sync_no_data(tmp_path, mock_aioresponses):
 
     # create the db and tables to exercise all code branches
     db = tmp_path / "db.sqlite"
-    with sqlite3.connect(db) as con:
-        con.executescript(contents("create-relations.sql"))
+    async with aiosqlite.connect(db) as con:
+        await con.executescript(contents("create-relations.sql"))
 
     await sync(TOKEN, db, False)
 
@@ -704,8 +765,8 @@ async def test_sync_no_data_quiet(tmp_path, mock_aioresponses, capsys):
     )
 
     db = tmp_path / "db.sqlite"
-    with sqlite3.connect(db) as con:
-        con.executescript(contents("create-relations.sql"))
+    async with aiosqlite.connect(db) as con:
+        await con.executescript(contents("create-relations.sql"))
 
     await sync(TOKEN, db, False, quiet=True)