sqlite-export-for-ynab 2.1.0__tar.gz → 2.2.0__tar.gz

{sqlite_export_for_ynab-2.1.0/sqlite_export_for_ynab.egg-info → sqlite_export_for_ynab-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlite_export_for_ynab
-Version: 2.1.0
+Version: 2.2.0
 Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
 Home-page: https://github.com/mxr/sqlite-export-for-ynab
 Author: Max R
@@ -51,6 +51,7 @@ $ sqlite-export-for-ynab
 ```
 
 Running it again will pull only data that changed since the last pull (this is done with [Delta Requests](https://api.ynab.com/#deltas)). If you want to wipe the DB and pull all data again use the `--full-refresh` flag.
+Pass `--quiet` to suppress all CLI output, including progress bars.
 
 <a id="db-path"></a>You can specify the DB path with the following options
 1. The `--db` flag.
@@ -345,44 +346,80 @@ To compare assigned category values to a given account's balance:
 --   -cmd ".parameter set @account_name_like %Savings%" \
 --   -cmd ".parameter set @include_category_groups 'Home,Food'" \
 --   < query.sql
+CREATE TEMP TABLE excess_query_results AS
 WITH params AS (
     SELECT
         TRIM(COALESCE(@account_name_like, '')) AS account_name_like
+        , TRIM(COALESCE(@plan_id, '')) AS plan_id
         , TRIM(COALESCE(@include_category_groups, ''))
             AS include_category_groups
         , TRIM(COALESCE(@exclude_category_groups, ''))
             AS exclude_category_groups
 )
 
-, validation AS (
-    SELECT 'Set @account_name_like' AS error
-    FROM params AS p
-    WHERE p.account_name_like = ''
+, scoped_plans AS (
+    SELECT
+        p.id
+        , p.name
+    FROM plans AS p
+    CROSS JOIN params AS prm
+    WHERE prm.plan_id = '' OR p.id = prm.plan_id
+)
+
+, split_include_category_groups (value, rest) AS (
+    SELECT
+        ''
+        , prm.include_category_groups || ','
+    FROM params AS prm
     UNION ALL
     SELECT
-        'Set only one of @include_category_groups'
-        || ' or @exclude_category_groups' AS error
-    FROM params AS p
-    WHERE p.include_category_groups != '' AND p.exclude_category_groups != ''
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_include_category_groups
+    WHERE rest != ''
 )
 
-SELECT v.error AS error_message
-FROM validation AS v
-WHERE v.error IS NOT NULL
-;
+, include_category_groups AS (
+    SELECT value AS name
+    FROM split_include_category_groups
+    WHERE value != ''
+)
 
-WITH params AS (
+, split_exclude_category_groups (value, rest) AS (
     SELECT
-        TRIM(COALESCE(@account_name_like, '')) AS account_name_like
-        , TRIM(COALESCE(@include_category_groups, ''))
-            AS include_category_groups
-        , TRIM(COALESCE(@exclude_category_groups, ''))
-            AS exclude_category_groups
+        ''
+        , prm.exclude_category_groups || ','
+    FROM params AS prm
+    UNION ALL
+    SELECT
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_exclude_category_groups
+    WHERE rest != ''
+)
+
+, exclude_category_groups AS (
+    SELECT value AS name
+    FROM split_exclude_category_groups
+    WHERE value != ''
+)
+
+, matching_accounts AS (
+    SELECT
+        sp.id AS plan_id
+        , sp.name AS plan_name
+        , COUNT(*) AS matches
+    FROM scoped_plans AS sp
+    INNER JOIN accounts AS a ON sp.id = a.plan_id
+    CROSS JOIN params AS prm
+    WHERE NOT a.deleted AND a.name LIKE prm.account_name_like
+    GROUP BY sp.id, sp.name
 )
 
 , validation AS (
     SELECT
         p.account_name_like
+        , p.plan_id
         , p.include_category_groups
         , p.exclude_category_groups
     FROM params AS p
@@ -398,11 +435,62 @@ WITH params AS (
         || ' or @exclude_category_groups' AS error
     FROM validation AS v
     WHERE v.include_category_groups != '' AND v.exclude_category_groups != ''
+    UNION ALL
+    SELECT 'No plan matched @plan_id' AS error
+    FROM validation AS v
+    WHERE
+        v.plan_id != '' AND NOT EXISTS (
+            SELECT 1
+            FROM scoped_plans
+        )
+    UNION ALL
+    SELECT 'No account names matched @account_name_like' AS error
+    FROM validation AS v
+    WHERE
+        v.account_name_like != '' AND NOT EXISTS (
+            SELECT 1
+            FROM matching_accounts
+        )
+    UNION ALL
+    SELECT
+        'Matched more than 1 account in plan: '
+        || ma.plan_name AS error
+    FROM matching_accounts AS ma
+    WHERE ma.matches > 1
+    UNION ALL
+    SELECT
+        'Unknown include category group in plan '
+        || sp.name
+        || ': '
+        || icg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN include_category_groups AS icg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(icg.name)
+    WHERE cg.id IS NULL
+    UNION ALL
+    SELECT
+        'Unknown exclude category group in plan '
+        || sp.name
+        || ': '
+        || ecg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN exclude_category_groups AS ecg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(ecg.name)
+    WHERE cg.id IS NULL
 )
 
 , valid_params AS (
     SELECT
         v.account_name_like
+        , v.plan_id
         , v.include_category_groups
         , v.exclude_category_groups
     FROM validation AS v
@@ -425,7 +513,7 @@ WITH params AS (
         TRUE
         AND NOT a.deleted
         AND a.name LIKE v.account_name_like
-        AND (COALESCE(@plan_id, '') = '' OR p.id = @plan_id)
+        AND (v.plan_id = '' OR p.id = v.plan_id)
 )
 
 , category_totals AS (
@@ -440,36 +528,61 @@ WITH params AS (
         AND c.category_group_name != 'Internal Master Category'
         AND (
             v.include_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.include_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR EXISTS (
+                SELECT 1
+                FROM include_category_groups AS icg
+                WHERE LOWER(icg.name) = LOWER(c.category_group_name)
             )
-            > 0
         )
         AND (
             v.exclude_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.exclude_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR NOT EXISTS (
+                SELECT 1
+                FROM exclude_category_groups AS ecg
+                WHERE LOWER(ecg.name) = LOWER(c.category_group_name)
             )
-            = 0
         )
-        AND (COALESCE(@plan_id, '') = '' OR c.plan_id = @plan_id)
+        AND (v.plan_id = '' OR c.plan_id = v.plan_id)
     GROUP BY c.plan_id
 )
 
 SELECT
-    ma.plan_name AS "plan"
+    ve.error AS error_message
+    , NULL AS "plan"
+    , NULL AS account
+    , NULL AS total
+    , NULL AS excess
+FROM validation_errors AS ve
+
+UNION ALL
+
+SELECT
+    NULL AS error_message
+    , ma.plan_name AS "plan"
     , ma.account_name AS account
     , PRINTF('%.2f', COALESCE(ct.total, 0)) AS total
-    , PRINTF('%.2f', COALESCE(ct.total, 0) - ma.account_amount) AS excess
+    , PRINTF('%.2f', ma.account_amount - COALESCE(ct.total, 0)) AS excess
 FROM matched_accounts AS ma
 LEFT JOIN category_totals AS ct ON ma.plan_id = ct.plan_id
-ORDER BY "plan", account
+;
+
+SELECT error_message
+FROM excess_query_results
+WHERE error_message IS NOT NULL
+;
+
+SELECT
+    eqr."plan"
+    , eqr.account
+    , eqr.total
+    , eqr.excess
+FROM excess_query_results AS eqr
+WHERE
+    NOT EXISTS (
+        SELECT 1
+        FROM excess_query_results AS eqr_errors
+        WHERE eqr_errors.error_message IS NOT NULL
+    )
 ;
 ```
 

{sqlite_export_for_ynab-2.1.0 → sqlite_export_for_ynab-2.2.0}/README.md

@@ -31,6 +31,7 @@ $ sqlite-export-for-ynab
 ```
 
 Running it again will pull only data that changed since the last pull (this is done with [Delta Requests](https://api.ynab.com/#deltas)). If you want to wipe the DB and pull all data again use the `--full-refresh` flag.
+Pass `--quiet` to suppress all CLI output, including progress bars.
 
 <a id="db-path"></a>You can specify the DB path with the following options
 1. The `--db` flag.
@@ -325,44 +326,80 @@ To compare assigned category values to a given account's balance:
 --   -cmd ".parameter set @account_name_like %Savings%" \
 --   -cmd ".parameter set @include_category_groups 'Home,Food'" \
 --   < query.sql
+CREATE TEMP TABLE excess_query_results AS
 WITH params AS (
     SELECT
         TRIM(COALESCE(@account_name_like, '')) AS account_name_like
+        , TRIM(COALESCE(@plan_id, '')) AS plan_id
         , TRIM(COALESCE(@include_category_groups, ''))
             AS include_category_groups
         , TRIM(COALESCE(@exclude_category_groups, ''))
             AS exclude_category_groups
 )
 
-, validation AS (
-    SELECT 'Set @account_name_like' AS error
-    FROM params AS p
-    WHERE p.account_name_like = ''
+, scoped_plans AS (
+    SELECT
+        p.id
+        , p.name
+    FROM plans AS p
+    CROSS JOIN params AS prm
+    WHERE prm.plan_id = '' OR p.id = prm.plan_id
+)
+
+, split_include_category_groups (value, rest) AS (
+    SELECT
+        ''
+        , prm.include_category_groups || ','
+    FROM params AS prm
     UNION ALL
     SELECT
-        'Set only one of @include_category_groups'
-        || ' or @exclude_category_groups' AS error
-    FROM params AS p
-    WHERE p.include_category_groups != '' AND p.exclude_category_groups != ''
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_include_category_groups
+    WHERE rest != ''
 )
 
-SELECT v.error AS error_message
-FROM validation AS v
-WHERE v.error IS NOT NULL
-;
+, include_category_groups AS (
+    SELECT value AS name
+    FROM split_include_category_groups
+    WHERE value != ''
+)
 
-WITH params AS (
+, split_exclude_category_groups (value, rest) AS (
     SELECT
-        TRIM(COALESCE(@account_name_like, '')) AS account_name_like
-        , TRIM(COALESCE(@include_category_groups, ''))
-            AS include_category_groups
-        , TRIM(COALESCE(@exclude_category_groups, ''))
-            AS exclude_category_groups
+        ''
+        , prm.exclude_category_groups || ','
+    FROM params AS prm
+    UNION ALL
+    SELECT
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_exclude_category_groups
+    WHERE rest != ''
+)
+
+, exclude_category_groups AS (
+    SELECT value AS name
+    FROM split_exclude_category_groups
+    WHERE value != ''
+)
+
+, matching_accounts AS (
+    SELECT
+        sp.id AS plan_id
+        , sp.name AS plan_name
+        , COUNT(*) AS matches
+    FROM scoped_plans AS sp
+    INNER JOIN accounts AS a ON sp.id = a.plan_id
+    CROSS JOIN params AS prm
+    WHERE NOT a.deleted AND a.name LIKE prm.account_name_like
+    GROUP BY sp.id, sp.name
 )
 
 , validation AS (
     SELECT
         p.account_name_like
+        , p.plan_id
         , p.include_category_groups
         , p.exclude_category_groups
     FROM params AS p
@@ -378,11 +415,62 @@ WITH params AS (
         || ' or @exclude_category_groups' AS error
     FROM validation AS v
     WHERE v.include_category_groups != '' AND v.exclude_category_groups != ''
+    UNION ALL
+    SELECT 'No plan matched @plan_id' AS error
+    FROM validation AS v
+    WHERE
+        v.plan_id != '' AND NOT EXISTS (
+            SELECT 1
+            FROM scoped_plans
+        )
+    UNION ALL
+    SELECT 'No account names matched @account_name_like' AS error
+    FROM validation AS v
+    WHERE
+        v.account_name_like != '' AND NOT EXISTS (
+            SELECT 1
+            FROM matching_accounts
+        )
+    UNION ALL
+    SELECT
+        'Matched more than 1 account in plan: '
+        || ma.plan_name AS error
+    FROM matching_accounts AS ma
+    WHERE ma.matches > 1
+    UNION ALL
+    SELECT
+        'Unknown include category group in plan '
+        || sp.name
+        || ': '
+        || icg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN include_category_groups AS icg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(icg.name)
+    WHERE cg.id IS NULL
+    UNION ALL
+    SELECT
+        'Unknown exclude category group in plan '
+        || sp.name
+        || ': '
+        || ecg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN exclude_category_groups AS ecg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(ecg.name)
+    WHERE cg.id IS NULL
 )
 
 , valid_params AS (
     SELECT
         v.account_name_like
+        , v.plan_id
         , v.include_category_groups
         , v.exclude_category_groups
     FROM validation AS v
@@ -405,7 +493,7 @@ WITH params AS (
         TRUE
         AND NOT a.deleted
         AND a.name LIKE v.account_name_like
-        AND (COALESCE(@plan_id, '') = '' OR p.id = @plan_id)
+        AND (v.plan_id = '' OR p.id = v.plan_id)
 )
 
 , category_totals AS (
@@ -420,36 +508,61 @@ WITH params AS (
         AND c.category_group_name != 'Internal Master Category'
         AND (
             v.include_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.include_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR EXISTS (
+                SELECT 1
+                FROM include_category_groups AS icg
+                WHERE LOWER(icg.name) = LOWER(c.category_group_name)
             )
-            > 0
         )
         AND (
             v.exclude_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.exclude_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR NOT EXISTS (
+                SELECT 1
+                FROM exclude_category_groups AS ecg
+                WHERE LOWER(ecg.name) = LOWER(c.category_group_name)
             )
-            = 0
         )
-        AND (COALESCE(@plan_id, '') = '' OR c.plan_id = @plan_id)
+        AND (v.plan_id = '' OR c.plan_id = v.plan_id)
     GROUP BY c.plan_id
 )
 
 SELECT
-    ma.plan_name AS "plan"
+    ve.error AS error_message
+    , NULL AS "plan"
+    , NULL AS account
+    , NULL AS total
+    , NULL AS excess
+FROM validation_errors AS ve
+
+UNION ALL
+
+SELECT
+    NULL AS error_message
+    , ma.plan_name AS "plan"
     , ma.account_name AS account
     , PRINTF('%.2f', COALESCE(ct.total, 0)) AS total
-    , PRINTF('%.2f', COALESCE(ct.total, 0) - ma.account_amount) AS excess
+    , PRINTF('%.2f', ma.account_amount - COALESCE(ct.total, 0)) AS excess
 FROM matched_accounts AS ma
 LEFT JOIN category_totals AS ct ON ma.plan_id = ct.plan_id
-ORDER BY "plan", account
+;
+
+SELECT error_message
+FROM excess_query_results
+WHERE error_message IS NOT NULL
+;
+
+SELECT
+    eqr."plan"
+    , eqr.account
+    , eqr.total
+    , eqr.excess
+FROM excess_query_results AS eqr
+WHERE
+    NOT EXISTS (
+        SELECT 1
+        FROM excess_query_results AS eqr_errors
+        WHERE eqr_errors.error_message IS NOT NULL
+    )
 ;
 ```
 

{sqlite_export_for_ynab-2.1.0 → sqlite_export_for_ynab-2.2.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = sqlite_export_for_ynab
-version = 2.1.0
+version = 2.2.0
 description = SQLite Export for YNAB - Export YNAB Data to SQLite
 long_description = file: README.md
 long_description_content_type = text/markdown

{sqlite_export_for_ynab-2.1.0 → sqlite_export_for_ynab-2.2.0}/sqlite_export_for_ynab/_main.py

@@ -50,7 +50,19 @@ _ENV_TOKEN = "YNAB_PERSONAL_ACCESS_TOKEN"
 _PACKAGE = "sqlite-export-for-ynab"
 
 
-async def async_main(argv: Sequence[str] | None = None) -> int:
+def resolve_token(token_override: str | None = None) -> str:
+    token = token_override or os.environ.get(_ENV_TOKEN)
+    if token:
+        return token
+
+    raise ValueError(
+        f"Must set YNAB access token as {_ENV_TOKEN!r} environment variable or pass token_override directly. See https://api.ynab.com/#personal-access-tokens"
+    )
+
+
+async def async_main(
+    argv: Sequence[str] | None = None, *, token_override: str | None = None
+) -> int:
     parser = argparse.ArgumentParser(prog=_PACKAGE)
     parser.add_argument(
         "--db",
@@ -66,20 +78,20 @@ async def async_main(argv: Sequence[str] | None = None) -> int:
     parser.add_argument(
         "--version", action="version", version=f"%(prog)s {version(_PACKAGE)}"
     )
+    parser.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Suppress all CLI output, including progress bars.",
+    )
 
     args = parser.parse_args(argv)
     db: Path = args.db
     full_refresh: bool = args.full_refresh
+    quiet: bool = args.quiet
 
-    token = os.environ.get(_ENV_TOKEN)
-    if not token:
-        raise ValueError(
-            f"Must set YNAB access token as {_ENV_TOKEN!r} "
-            "environment variable. See "
-            "https://api.ynab.com/#personal-access-tokens"
-        )
+    token = resolve_token(token_override)
 
-    await sync(token, db, full_refresh)
+    await sync(token, db, full_refresh, quiet=quiet)
 
     return 0
 
@@ -96,7 +108,14 @@ def default_db_path() -> Path:
     )
 
 
-async def sync(token: str, db: Path, full_refresh: bool) -> None:
+def _print(message: str, *, quiet: bool) -> None:
+    if not quiet:
+        print(message)
+
+
+async def sync(
+    token: str, db: Path, full_refresh: bool, *, quiet: bool = False
+) -> None:
     async with aiohttp.ClientSession() as session:
         plans = (await YnabClient(token, session)("plans"))["plans"]
 
@@ -106,26 +125,26 @@ async def sync(token: str, db: Path, full_refresh: bool) -> None:
         db.parent.mkdir(parents=True, exist_ok=True)
 
     with sqlite3.connect(db) as con:
-        con.row_factory = _row_factory
+        con.row_factory = sqlite3.Row
         cur = con.cursor()
 
         if full_refresh:
-            print("Dropping relations...")
+            _print("Dropping relations...", quiet=quiet)
             cur.executescript(contents("drop-relations.sql"))
             con.commit()
-            print("Done")
+            _print("Done", quiet=quiet)
 
         relations = get_relations(cur)
         if relations != _ALL_RELATIONS:
-            print("Recreating relations...")
+            _print("Recreating relations...", quiet=quiet)
             cur.executescript(contents("create-relations.sql"))
             con.commit()
-            print("Done")
+            _print("Done", quiet=quiet)
 
-        print("Fetching plan data...")
+        _print("Fetching plan data...", quiet=quiet)
         lkos = get_last_knowledge_of_server(cur)
         async with aiohttp.ClientSession() as session:
-            with tldm(desc="Plan Data", total=len(plans) * 5) as pbar:
+            with tldm(desc="Plan Data", total=len(plans) * 5, disable=quiet) as pbar:
                 yc = ProgressYnabClient(YnabClient(token, session), pbar)
 
                 account_jobs = jobs(yc, "accounts", plan_ids, lkos)
@@ -155,7 +174,7 @@ async def sync(token: str, db: Path, full_refresh: bool) -> None:
                     plan_ids, all_txn_data, strict=True
                 )
             }
-        print("Done")
+        _print("Done", quiet=quiet)
 
         if (
             not any(t["accounts"] for t in all_account_data)
@@ -164,29 +183,27 @@ async def sync(token: str, db: Path, full_refresh: bool) -> None:
             and not any(t["transactions"] for t in all_txn_data)
             and not any(s["scheduled_transactions"] for s in all_sched_txn_data)
         ):
-            print("No new data fetched")
+            _print("No new data fetched", quiet=quiet)
         else:
-            print("Inserting plan data...")
+            _print("Inserting plan data...", quiet=quiet)
             insert_plans(cur, plans, new_lkos)
             for plan_id, account_data in zip(plan_ids, all_account_data, strict=True):
-                insert_accounts(cur, plan_id, account_data["accounts"])
+                insert_accounts(cur, plan_id, account_data["accounts"], quiet=quiet)
             for plan_id, cat_data in zip(plan_ids, all_cat_data, strict=True):
-                insert_category_groups(cur, plan_id, cat_data["category_groups"])
+                insert_category_groups(
+                    cur, plan_id, cat_data["category_groups"], quiet=quiet
+                )
             for plan_id, payee_data in zip(plan_ids, all_payee_data, strict=True):
-                insert_payees(cur, plan_id, payee_data["payees"])
+                insert_payees(cur, plan_id, payee_data["payees"], quiet=quiet)
             for plan_id, txn_data in zip(plan_ids, all_txn_data, strict=True):
-                insert_transactions(cur, plan_id, txn_data["transactions"])
+                insert_transactions(cur, plan_id, txn_data["transactions"], quiet=quiet)
             for plan_id, sched_txn_data in zip(
                 plan_ids, all_sched_txn_data, strict=True
             ):
                 insert_scheduled_transactions(
-                    cur, plan_id, sched_txn_data["scheduled_transactions"]
+                    cur, plan_id, sched_txn_data["scheduled_transactions"], quiet=quiet
                 )
-            print("Done")
-
-
-def _row_factory(c: sqlite3.Cursor, row: tuple[Any, ...]) -> dict[str, Any]:
-    return {d[0]: r for d, r in zip(c.description, row, strict=True)}
+            _print("Done", quiet=quiet)
 
 
 def contents(filename: str) -> str:
@@ -253,7 +270,11 @@ _LOAN_ACCOUNT_PERIODIC_VALUES = frozenset(
 
 
 def insert_accounts(
-    cur: sqlite3.Cursor, plan_id: str, accounts: list[dict[str, Any]]
+    cur: sqlite3.Cursor,
+    plan_id: str,
+    accounts: list[dict[str, Any]],
+    *,
+    quiet: bool = False,
 ) -> None:
     # YNAB's LoanAccountPeriodValues are untyped dicts so we need to turn them into a more standard sub-entry view
     updated_accounts = [
@@ -281,11 +302,16 @@ def insert_accounts(
         "accounts",
         "account_periodic_values",
         "account_periodic_values",
+        quiet=quiet,
     )
 
 
 def insert_category_groups(
-    cur: sqlite3.Cursor, plan_id: str, category_groups: list[dict[str, Any]]
+    cur: sqlite3.Cursor,
+    plan_id: str,
+    category_groups: list[dict[str, Any]],
+    *,
+    quiet: bool = False,
 ) -> None:
     return insert_nested_entries(
         cur,
@@ -295,21 +321,30 @@ def insert_category_groups(
         "category_groups",
         "categories",
         "categories",
+        quiet=quiet,
     )
 
 
 def insert_payees(
-    cur: sqlite3.Cursor, plan_id: str, payees: list[dict[str, Any]]
+    cur: sqlite3.Cursor,
+    plan_id: str,
+    payees: list[dict[str, Any]],
+    *,
+    quiet: bool = False,
 ) -> None:
     if not payees:
         return
 
-    for payee in tldm(payees, desc="Payees"):
+    for payee in tldm(payees, desc="Payees", disable=quiet):
         insert_entry(cur, "payees", plan_id, payee)
 
 
 def insert_transactions(
-    cur: sqlite3.Cursor, plan_id: str, transactions: list[dict[str, Any]]
+    cur: sqlite3.Cursor,
+    plan_id: str,
+    transactions: list[dict[str, Any]],
+    *,
+    quiet: bool = False,
 ) -> None:
     return insert_nested_entries(
         cur,
@@ -319,11 +354,16 @@ def insert_transactions(
         "transactions",
         "subtransactions",
         "subtransactions",
+        quiet=quiet,
     )
 
 
 def insert_scheduled_transactions(
-    cur: sqlite3.Cursor, plan_id: str, scheduled_transactions: list[dict[str, Any]]
+    cur: sqlite3.Cursor,
+    plan_id: str,
+    scheduled_transactions: list[dict[str, Any]],
+    *,
+    quiet: bool = False,
 ) -> None:
     return insert_nested_entries(
         cur,
@@ -333,6 +373,7 @@ def insert_scheduled_transactions(
         "scheduled_transactions",
         "subtransactions",
         "scheduled_subtransactions",
+        quiet=quiet,
     )
 
 
@@ -345,6 +386,8 @@ def insert_nested_entries(
     entries_name: Literal["accounts"],
     subentries_name: Literal["account_periodic_values"],
     subentries_table_name: Literal["account_periodic_values"],
+    *,
+    quiet: bool = False,
 ) -> None: ...
 
 
@@ -357,6 +400,8 @@ def insert_nested_entries(
     entries_name: Literal["category_groups"],
     subentries_name: Literal["categories"],
     subentries_table_name: Literal["categories"],
+    *,
+    quiet: bool = False,
 ) -> None: ...
 
 
@@ -369,6 +414,8 @@ def insert_nested_entries(
     entries_name: Literal["transactions"],
     subentries_name: Literal["subtransactions"],
     subentries_table_name: Literal["subtransactions"],
+    *,
+    quiet: bool = False,
 ) -> None: ...
 
 
@@ -381,6 +428,8 @@ def insert_nested_entries(
     entries_name: Literal["scheduled_transactions"],
     subentries_name: Literal["subtransactions"],
     subentries_table_name: Literal["scheduled_subtransactions"],
+    *,
+    quiet: bool = False,
 ) -> None: ...
 
 
@@ -411,6 +460,8 @@ def insert_nested_entries(
         | Literal["subtransactions"]
         | Literal["scheduled_subtransactions"]
     ),
+    *,
+    quiet: bool = False,
 ) -> None:
     if not entries:
         return
@@ -418,6 +469,7 @@ def insert_nested_entries(
     with tldm(
         total=sum(1 + len(e[subentries_name]) for e in entries),
         desc=desc,
+        disable=quiet,
     ) as pbar:
         for entry in entries:
             insert_entry(
@@ -530,5 +582,7 @@ class YnabClient:
         raise AssertionError("unreachable")
 
 
-def main(argv: Sequence[str] | None = None) -> int:
-    return asyncio.run(async_main(argv))
+def main(
+    argv: Sequence[str] | None = None, *, token_override: str | None = None
+) -> int:
+    return asyncio.run(async_main(argv, token_override=token_override))

{sqlite_export_for_ynab-2.1.0 → sqlite_export_for_ynab-2.2.0/sqlite_export_for_ynab.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlite_export_for_ynab
-Version: 2.1.0
+Version: 2.2.0
 Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
 Home-page: https://github.com/mxr/sqlite-export-for-ynab
 Author: Max R
@@ -51,6 +51,7 @@ $ sqlite-export-for-ynab
 ```
 
 Running it again will pull only data that changed since the last pull (this is done with [Delta Requests](https://api.ynab.com/#deltas)). If you want to wipe the DB and pull all data again use the `--full-refresh` flag.
+Pass `--quiet` to suppress all CLI output, including progress bars.
 
 <a id="db-path"></a>You can specify the DB path with the following options
 1. The `--db` flag.
@@ -345,44 +346,80 @@ To compare assigned category values to a given account's balance:
 --   -cmd ".parameter set @account_name_like %Savings%" \
 --   -cmd ".parameter set @include_category_groups 'Home,Food'" \
 --   < query.sql
+CREATE TEMP TABLE excess_query_results AS
 WITH params AS (
     SELECT
         TRIM(COALESCE(@account_name_like, '')) AS account_name_like
+        , TRIM(COALESCE(@plan_id, '')) AS plan_id
         , TRIM(COALESCE(@include_category_groups, ''))
             AS include_category_groups
         , TRIM(COALESCE(@exclude_category_groups, ''))
             AS exclude_category_groups
 )
 
-, validation AS (
-    SELECT 'Set @account_name_like' AS error
-    FROM params AS p
-    WHERE p.account_name_like = ''
+, scoped_plans AS (
+    SELECT
+        p.id
+        , p.name
+    FROM plans AS p
+    CROSS JOIN params AS prm
+    WHERE prm.plan_id = '' OR p.id = prm.plan_id
+)
+
+, split_include_category_groups (value, rest) AS (
+    SELECT
+        ''
+        , prm.include_category_groups || ','
+    FROM params AS prm
     UNION ALL
     SELECT
-        'Set only one of @include_category_groups'
-        || ' or @exclude_category_groups' AS error
-    FROM params AS p
-    WHERE p.include_category_groups != '' AND p.exclude_category_groups != ''
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_include_category_groups
+    WHERE rest != ''
 )
 
-SELECT v.error AS error_message
-FROM validation AS v
-WHERE v.error IS NOT NULL
-;
+, include_category_groups AS (
+    SELECT value AS name
+    FROM split_include_category_groups
+    WHERE value != ''
+)
 
-WITH params AS (
+, split_exclude_category_groups (value, rest) AS (
     SELECT
-        TRIM(COALESCE(@account_name_like, '')) AS account_name_like
-        , TRIM(COALESCE(@include_category_groups, ''))
-            AS include_category_groups
-        , TRIM(COALESCE(@exclude_category_groups, ''))
-            AS exclude_category_groups
+        ''
+        , prm.exclude_category_groups || ','
+    FROM params AS prm
+    UNION ALL
+    SELECT
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_exclude_category_groups
+    WHERE rest != ''
+)
+
+, exclude_category_groups AS (
+    SELECT value AS name
+    FROM split_exclude_category_groups
+    WHERE value != ''
+)
+
+, matching_accounts AS (
+    SELECT
+        sp.id AS plan_id
+        , sp.name AS plan_name
+        , COUNT(*) AS matches
+    FROM scoped_plans AS sp
+    INNER JOIN accounts AS a ON sp.id = a.plan_id
+    CROSS JOIN params AS prm
+    WHERE NOT a.deleted AND a.name LIKE prm.account_name_like
+    GROUP BY sp.id, sp.name
 )
 
 , validation AS (
     SELECT
         p.account_name_like
+        , p.plan_id
         , p.include_category_groups
         , p.exclude_category_groups
     FROM params AS p
@@ -398,11 +435,62 @@ WITH params AS (
         || ' or @exclude_category_groups' AS error
     FROM validation AS v
     WHERE v.include_category_groups != '' AND v.exclude_category_groups != ''
+    UNION ALL
+    SELECT 'No plan matched @plan_id' AS error
+    FROM validation AS v
+    WHERE
+        v.plan_id != '' AND NOT EXISTS (
+            SELECT 1
+            FROM scoped_plans
+        )
+    UNION ALL
+    SELECT 'No account names matched @account_name_like' AS error
+    FROM validation AS v
+    WHERE
+        v.account_name_like != '' AND NOT EXISTS (
+            SELECT 1
+            FROM matching_accounts
+        )
+    UNION ALL
+    SELECT
+        'Matched more than 1 account in plan: '
+        || ma.plan_name AS error
+    FROM matching_accounts AS ma
+    WHERE ma.matches > 1
+    UNION ALL
+    SELECT
+        'Unknown include category group in plan '
+        || sp.name
+        || ': '
+        || icg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN include_category_groups AS icg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(icg.name)
+    WHERE cg.id IS NULL
+    UNION ALL
+    SELECT
+        'Unknown exclude category group in plan '
+        || sp.name
+        || ': '
+        || ecg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN exclude_category_groups AS ecg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(ecg.name)
+    WHERE cg.id IS NULL
 )
 
 , valid_params AS (
     SELECT
         v.account_name_like
+        , v.plan_id
         , v.include_category_groups
         , v.exclude_category_groups
     FROM validation AS v
@@ -425,7 +513,7 @@ WITH params AS (
         TRUE
         AND NOT a.deleted
         AND a.name LIKE v.account_name_like
-        AND (COALESCE(@plan_id, '') = '' OR p.id = @plan_id)
+        AND (v.plan_id = '' OR p.id = v.plan_id)
 )
 
 , category_totals AS (
@@ -440,36 +528,61 @@ WITH params AS (
         AND c.category_group_name != 'Internal Master Category'
         AND (
             v.include_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.include_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR EXISTS (
+                SELECT 1
+                FROM include_category_groups AS icg
+                WHERE LOWER(icg.name) = LOWER(c.category_group_name)
             )
-            > 0
         )
         AND (
             v.exclude_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.exclude_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR NOT EXISTS (
+                SELECT 1
+                FROM exclude_category_groups AS ecg
+                WHERE LOWER(ecg.name) = LOWER(c.category_group_name)
             )
-            = 0
         )
-        AND (COALESCE(@plan_id, '') = '' OR c.plan_id = @plan_id)
+        AND (v.plan_id = '' OR c.plan_id = v.plan_id)
     GROUP BY c.plan_id
 )
 
 SELECT
-    ma.plan_name AS "plan"
+    ve.error AS error_message
+    , NULL AS "plan"
+    , NULL AS account
+    , NULL AS total
+    , NULL AS excess
+FROM validation_errors AS ve
+
+UNION ALL
+
+SELECT
+    NULL AS error_message
+    , ma.plan_name AS "plan"
     , ma.account_name AS account
     , PRINTF('%.2f', COALESCE(ct.total, 0)) AS total
-    , PRINTF('%.2f', COALESCE(ct.total, 0) - ma.account_amount) AS excess
+    , PRINTF('%.2f', ma.account_amount - COALESCE(ct.total, 0)) AS excess
 FROM matched_accounts AS ma
 LEFT JOIN category_totals AS ct ON ma.plan_id = ct.plan_id
-ORDER BY "plan", account
+;
+
+SELECT error_message
+FROM excess_query_results
+WHERE error_message IS NOT NULL
+;
+
+SELECT
+    eqr."plan"
+    , eqr.account
+    , eqr.total
+    , eqr.excess
+FROM excess_query_results AS eqr
+WHERE
+    NOT EXISTS (
+        SELECT 1
+        FROM excess_query_results AS eqr_errors
+        WHERE eqr_errors.error_message IS NOT NULL
+    )
 ;
 ```
 

{sqlite_export_for_ynab-2.1.0 → sqlite_export_for_ynab-2.2.0}/testing/fixtures.py

@@ -8,7 +8,6 @@ from uuid import uuid4
 import pytest
 from aioresponses import aioresponses
 
-from sqlite_export_for_ynab._main import _row_factory
 from sqlite_export_for_ynab._main import contents
 
 PLAN_ID_1 = str(uuid4())
@@ -326,7 +325,7 @@ SCHEDULED_TRANSACTIONS: list[dict[str, Any]] = [
 @pytest.fixture
 def cur():
     with sqlite3.connect(":memory:") as con:
-        con.row_factory = _row_factory
+        con.row_factory = sqlite3.Row
         cursor = con.cursor()
         cursor.executescript(contents("create-relations.sql"))
         yield cursor
@@ -338,8 +337,8 @@ def mock_aioresponses():
         yield m
 
 
-def strip_nones(d: dict[str, Any]) -> dict[str, Any]:
-    return {k: v for k, v in d.items() if v is not None}
+def strip_nones(d: sqlite3.Row) -> dict[str, Any]:
+    return {k: v for k, v in dict(d).items() if v is not None}
 
 
 TOKEN = f"token-{uuid4()}"

{sqlite_export_for_ynab-2.1.0 → sqlite_export_for_ynab-2.2.0}/tests/_main_test.py

@@ -27,6 +27,7 @@ from sqlite_export_for_ynab._main import insert_scheduled_transactions
 from sqlite_export_for_ynab._main import insert_transactions
 from sqlite_export_for_ynab._main import main
 from sqlite_export_for_ynab._main import ProgressYnabClient
+from sqlite_export_for_ynab._main import resolve_token
 from sqlite_export_for_ynab._main import sync
 from sqlite_export_for_ynab._main import YnabClient
 from testing.fixtures import ACCOUNT_ID_1
@@ -106,7 +107,7 @@ def test_get_last_knowledge_of_server(cur):
 def test_insert_plans(cur):
     insert_plans(cur, PLANS, LKOS)
     cur.execute("SELECT * FROM plans ORDER BY name")
-    assert cur.fetchall() == [
+    assert [strip_nones(d) for d in cur.fetchall()] == [
         {
             "id": PLAN_ID_1,
             "name": PLANS[0]["name"],
@@ -170,7 +171,7 @@ def test_insert_accounts(cur):
     ]
 
     cur.execute("SELECT * FROM account_periodic_values ORDER BY name")
-    assert cur.fetchall() == [
+    assert [strip_nones(d) for d in cur.fetchall()] == [
         {
             "account_id": ACCOUNT_ID_1,
             "plan_id": PLAN_ID_1,
@@ -575,7 +576,7 @@ def test_main_ok(sync, tmp_path, monkeypatch):
     monkeypatch.setenv(_ENV_TOKEN, TOKEN)
 
     ret = main(("--db", str(tmp_path / "db.sqlite")))
-    sync.assert_called()
+    sync.assert_called_once_with(TOKEN, tmp_path / "db.sqlite", False, quiet=False)
     assert ret == 0
 
 
@@ -586,6 +587,40 @@ def test_main_no_token(tmp_path, monkeypatch):
         main(("--db", str(tmp_path / "db.sqlite")))
 
 
+@patch("sqlite_export_for_ynab._main.sync")
+def test_main_uses_token_override(sync, tmp_path, monkeypatch):
+    monkeypatch.delenv(_ENV_TOKEN, raising=False)
+
+    ret = main(("--db", str(tmp_path / "db.sqlite")), token_override="override-token")
+
+    sync.assert_called_once_with(
+        "override-token", tmp_path / "db.sqlite", False, quiet=False
+    )
+    assert ret == 0
+
+
+@patch("sqlite_export_for_ynab._main.sync")
+def test_main_quiet(sync, tmp_path, monkeypatch):
+    monkeypatch.setenv(_ENV_TOKEN, TOKEN)
+
+    ret = main(("--db", str(tmp_path / "db.sqlite"), "--quiet"))
+
+    sync.assert_called_once_with(TOKEN, tmp_path / "db.sqlite", False, quiet=True)
+    assert ret == 0
+
+
+def test_resolve_token_override(monkeypatch):
+    monkeypatch.delenv(_ENV_TOKEN, raising=False)
+
+    assert resolve_token("override-token") == "override-token"
+
+
+def test_resolve_token_env(monkeypatch):
+    monkeypatch.setenv(_ENV_TOKEN, TOKEN)
+
+    assert resolve_token() == TOKEN
+
+
 @pytest.mark.asyncio
 @pytest.mark.usefixtures(mock_aioresponses.__name__)
 async def test_sync_no_data(tmp_path, mock_aioresponses):
@@ -631,6 +666,54 @@ async def test_sync_no_data(tmp_path, mock_aioresponses):
     await sync(TOKEN, db, False)
 
 
+@pytest.mark.asyncio
+@pytest.mark.usefixtures(mock_aioresponses.__name__)
+async def test_sync_no_data_quiet(tmp_path, mock_aioresponses, capsys):
+    mock_aioresponses.get(
+        PLANS_ENDPOINT_RE, body=json.dumps({"data": {"plans": PLANS}})
+    )
+    mock_aioresponses.get(
+        ACCOUNTS_ENDPOINT_RE,
+        body=json.dumps({"data": {"accounts": []}}),
+        repeat=True,
+    )
+    mock_aioresponses.get(
+        CATEGORIES_ENDPOINT_RE,
+        body=json.dumps({"data": {"category_groups": []}}),
+        repeat=True,
+    )
+    mock_aioresponses.get(
+        PAYEES_ENDPOINT_RE, body=json.dumps({"data": {"payees": []}}), repeat=True
+    )
+    mock_aioresponses.get(
+        TRANSACTIONS_ENDPOINT_RE,
+        body=json.dumps(
+            {
+                "data": {
+                    "transactions": [],
+                    "server_knowledge": SERVER_KNOWLEDGE_1,
+                }
+            }
+        ),
+        repeat=True,
+    )
+    mock_aioresponses.get(
+        SCHEDULED_TRANSACTIONS_ENDPOINT_RE,
+        body=json.dumps({"data": {"scheduled_transactions": []}}),
+        repeat=True,
+    )
+
+    db = tmp_path / "db.sqlite"
+    with sqlite3.connect(db) as con:
+        con.executescript(contents("create-relations.sql"))
+
+    await sync(TOKEN, db, False, quiet=True)
+
+    out, err = capsys.readouterr()
+    assert out == ""
+    assert err == ""
+
+
 @pytest.mark.asyncio
 @pytest.mark.usefixtures(mock_aioresponses.__name__)
 async def test_sync(tmp_path, mock_aioresponses):