sqlite-export-for-ynab 2.1.0__tar.gz → 2.3.0__tar.gz

{sqlite_export_for_ynab-2.1.0/sqlite_export_for_ynab.egg-info → sqlite_export_for_ynab-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sqlite_export_for_ynab
-Version: 2.1.0
+Version: 2.3.0
 Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
 Home-page: https://github.com/mxr/sqlite-export-for-ynab
 Author: Max R
@@ -51,6 +51,7 @@ $ sqlite-export-for-ynab
 ```
 
 Running it again will pull only data that changed since the last pull (this is done with [Delta Requests](https://api.ynab.com/#deltas)). If you want to wipe the DB and pull all data again use the `--full-refresh` flag.
+Pass `--quiet` to suppress all CLI output, including progress bars.
 
 <a id="db-path"></a>You can specify the DB path with the following options
 1. The `--db` flag.
@@ -81,7 +82,7 @@ The relations are defined in [create-relations.sql](sqlite_export_for_ynab/ddl/c
 
 1. Some objects are pulled out into their own tables so they can be more cleanly modeled in SQLite (ex: subtransactions, loan account periodic values).
 1. Foreign keys are added as needed (ex: plan ID, transaction ID) so data across plans remains separate.
-1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted transactions/subtransactions and project payee/category fields to make querying more ergonomic.
+1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted/unapproved transactions/subtransactions and project payee/category fields to make querying more ergonomic.
 
 ## Querying
 
@@ -135,6 +136,7 @@ WITH used_payees AS (
     FROM transactions
     WHERE
         TRUE
+        AND approved
         AND payee_id IS NOT NULL
         AND NOT deleted
     UNION
@@ -345,44 +347,80 @@ To compare assigned category values to a given account's balance:
 --   -cmd ".parameter set @account_name_like %Savings%" \
 --   -cmd ".parameter set @include_category_groups 'Home,Food'" \
 --   < query.sql
+CREATE TEMP TABLE excess_query_results AS
 WITH params AS (
     SELECT
         TRIM(COALESCE(@account_name_like, '')) AS account_name_like
+        , TRIM(COALESCE(@plan_id, '')) AS plan_id
         , TRIM(COALESCE(@include_category_groups, ''))
             AS include_category_groups
         , TRIM(COALESCE(@exclude_category_groups, ''))
             AS exclude_category_groups
 )
 
-, validation AS (
-    SELECT 'Set @account_name_like' AS error
-    FROM params AS p
-    WHERE p.account_name_like = ''
+, scoped_plans AS (
+    SELECT
+        p.id
+        , p.name
+    FROM plans AS p
+    CROSS JOIN params AS prm
+    WHERE prm.plan_id = '' OR p.id = prm.plan_id
+)
+
+, split_include_category_groups (value, rest) AS (
+    SELECT
+        ''
+        , prm.include_category_groups || ','
+    FROM params AS prm
     UNION ALL
     SELECT
-        'Set only one of @include_category_groups'
-        || ' or @exclude_category_groups' AS error
-    FROM params AS p
-    WHERE p.include_category_groups != '' AND p.exclude_category_groups != ''
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_include_category_groups
+    WHERE rest != ''
 )
 
-SELECT v.error AS error_message
-FROM validation AS v
-WHERE v.error IS NOT NULL
-;
+, include_category_groups AS (
+    SELECT value AS name
+    FROM split_include_category_groups
+    WHERE value != ''
+)
 
-WITH params AS (
+, split_exclude_category_groups (value, rest) AS (
     SELECT
-        TRIM(COALESCE(@account_name_like, '')) AS account_name_like
-        , TRIM(COALESCE(@include_category_groups, ''))
-            AS include_category_groups
-        , TRIM(COALESCE(@exclude_category_groups, ''))
-            AS exclude_category_groups
+        ''
+        , prm.exclude_category_groups || ','
+    FROM params AS prm
+    UNION ALL
+    SELECT
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_exclude_category_groups
+    WHERE rest != ''
+)
+
+, exclude_category_groups AS (
+    SELECT value AS name
+    FROM split_exclude_category_groups
+    WHERE value != ''
+)
+
+, matching_accounts AS (
+    SELECT
+        sp.id AS plan_id
+        , sp.name AS plan_name
+        , COUNT(*) AS matches
+    FROM scoped_plans AS sp
+    INNER JOIN accounts AS a ON sp.id = a.plan_id
+    CROSS JOIN params AS prm
+    WHERE NOT a.deleted AND a.name LIKE prm.account_name_like
+    GROUP BY sp.id, sp.name
 )
 
 , validation AS (
     SELECT
         p.account_name_like
+        , p.plan_id
         , p.include_category_groups
         , p.exclude_category_groups
     FROM params AS p
@@ -398,11 +436,62 @@ WITH params AS (
         || ' or @exclude_category_groups' AS error
     FROM validation AS v
     WHERE v.include_category_groups != '' AND v.exclude_category_groups != ''
+    UNION ALL
+    SELECT 'No plan matched @plan_id' AS error
+    FROM validation AS v
+    WHERE
+        v.plan_id != '' AND NOT EXISTS (
+            SELECT 1
+            FROM scoped_plans
+        )
+    UNION ALL
+    SELECT 'No account names matched @account_name_like' AS error
+    FROM validation AS v
+    WHERE
+        v.account_name_like != '' AND NOT EXISTS (
+            SELECT 1
+            FROM matching_accounts
+        )
+    UNION ALL
+    SELECT
+        'Matched more than 1 account in plan: '
+        || ma.plan_name AS error
+    FROM matching_accounts AS ma
+    WHERE ma.matches > 1
+    UNION ALL
+    SELECT
+        'Unknown include category group in plan '
+        || sp.name
+        || ': '
+        || icg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN include_category_groups AS icg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(icg.name)
+    WHERE cg.id IS NULL
+    UNION ALL
+    SELECT
+        'Unknown exclude category group in plan '
+        || sp.name
+        || ': '
+        || ecg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN exclude_category_groups AS ecg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(ecg.name)
+    WHERE cg.id IS NULL
 )
 
 , valid_params AS (
     SELECT
         v.account_name_like
+        , v.plan_id
         , v.include_category_groups
         , v.exclude_category_groups
     FROM validation AS v
@@ -425,7 +514,7 @@ WITH params AS (
         TRUE
         AND NOT a.deleted
         AND a.name LIKE v.account_name_like
-        AND (COALESCE(@plan_id, '') = '' OR p.id = @plan_id)
+        AND (v.plan_id = '' OR p.id = v.plan_id)
 )
 
 , category_totals AS (
@@ -440,36 +529,61 @@ WITH params AS (
         AND c.category_group_name != 'Internal Master Category'
         AND (
             v.include_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.include_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR EXISTS (
+                SELECT 1
+                FROM include_category_groups AS icg
+                WHERE LOWER(icg.name) = LOWER(c.category_group_name)
             )
-            > 0
         )
         AND (
             v.exclude_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.exclude_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR NOT EXISTS (
+                SELECT 1
+                FROM exclude_category_groups AS ecg
+                WHERE LOWER(ecg.name) = LOWER(c.category_group_name)
             )
-            = 0
         )
-        AND (COALESCE(@plan_id, '') = '' OR c.plan_id = @plan_id)
+        AND (v.plan_id = '' OR c.plan_id = v.plan_id)
     GROUP BY c.plan_id
 )
 
 SELECT
-    ma.plan_name AS "plan"
+    ve.error AS error_message
+    , NULL AS "plan"
+    , NULL AS account
+    , NULL AS total
+    , NULL AS excess
+FROM validation_errors AS ve
+
+UNION ALL
+
+SELECT
+    NULL AS error_message
+    , ma.plan_name AS "plan"
     , ma.account_name AS account
     , PRINTF('%.2f', COALESCE(ct.total, 0)) AS total
-    , PRINTF('%.2f', COALESCE(ct.total, 0) - ma.account_amount) AS excess
+    , PRINTF('%.2f', ma.account_amount - COALESCE(ct.total, 0)) AS excess
 FROM matched_accounts AS ma
 LEFT JOIN category_totals AS ct ON ma.plan_id = ct.plan_id
-ORDER BY "plan", account
+;
+
+SELECT error_message
+FROM excess_query_results
+WHERE error_message IS NOT NULL
+;
+
+SELECT
+    eqr."plan"
+    , eqr.account
+    , eqr.total
+    , eqr.excess
+FROM excess_query_results AS eqr
+WHERE
+    NOT EXISTS (
+        SELECT 1
+        FROM excess_query_results AS eqr_errors
+        WHERE eqr_errors.error_message IS NOT NULL
+    )
 ;
 ```
 

{sqlite_export_for_ynab-2.1.0 → sqlite_export_for_ynab-2.3.0}/README.md

@@ -31,6 +31,7 @@ $ sqlite-export-for-ynab
 ```
 
 Running it again will pull only data that changed since the last pull (this is done with [Delta Requests](https://api.ynab.com/#deltas)). If you want to wipe the DB and pull all data again use the `--full-refresh` flag.
+Pass `--quiet` to suppress all CLI output, including progress bars.
 
 <a id="db-path"></a>You can specify the DB path with the following options
 1. The `--db` flag.
@@ -61,7 +62,7 @@ The relations are defined in [create-relations.sql](sqlite_export_for_ynab/ddl/c
 
 1. Some objects are pulled out into their own tables so they can be more cleanly modeled in SQLite (ex: subtransactions, loan account periodic values).
 1. Foreign keys are added as needed (ex: plan ID, transaction ID) so data across plans remains separate.
-1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted transactions/subtransactions and project payee/category fields to make querying more ergonomic.
+1. Two new views called `flat_transactions` and `scheduled_flat_transactions`. These allow you to query split and non-split transactions easily, without needing to also query `subtransactions` and `scheduled_subtransactions` respectively. They also filter out deleted/unapproved transactions/subtransactions and project payee/category fields to make querying more ergonomic.
 
 ## Querying
 
@@ -115,6 +116,7 @@ WITH used_payees AS (
     FROM transactions
     WHERE
         TRUE
+        AND approved
         AND payee_id IS NOT NULL
         AND NOT deleted
     UNION
@@ -325,44 +327,80 @@ To compare assigned category values to a given account's balance:
 --   -cmd ".parameter set @account_name_like %Savings%" \
 --   -cmd ".parameter set @include_category_groups 'Home,Food'" \
 --   < query.sql
+CREATE TEMP TABLE excess_query_results AS
 WITH params AS (
     SELECT
         TRIM(COALESCE(@account_name_like, '')) AS account_name_like
+        , TRIM(COALESCE(@plan_id, '')) AS plan_id
         , TRIM(COALESCE(@include_category_groups, ''))
             AS include_category_groups
         , TRIM(COALESCE(@exclude_category_groups, ''))
             AS exclude_category_groups
 )
 
-, validation AS (
-    SELECT 'Set @account_name_like' AS error
-    FROM params AS p
-    WHERE p.account_name_like = ''
+, scoped_plans AS (
+    SELECT
+        p.id
+        , p.name
+    FROM plans AS p
+    CROSS JOIN params AS prm
+    WHERE prm.plan_id = '' OR p.id = prm.plan_id
+)
+
+, split_include_category_groups (value, rest) AS (
+    SELECT
+        ''
+        , prm.include_category_groups || ','
+    FROM params AS prm
     UNION ALL
     SELECT
-        'Set only one of @include_category_groups'
-        || ' or @exclude_category_groups' AS error
-    FROM params AS p
-    WHERE p.include_category_groups != '' AND p.exclude_category_groups != ''
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_include_category_groups
+    WHERE rest != ''
 )
 
-SELECT v.error AS error_message
-FROM validation AS v
-WHERE v.error IS NOT NULL
-;
+, include_category_groups AS (
+    SELECT value AS name
+    FROM split_include_category_groups
+    WHERE value != ''
+)
 
-WITH params AS (
+, split_exclude_category_groups (value, rest) AS (
     SELECT
-        TRIM(COALESCE(@account_name_like, '')) AS account_name_like
-        , TRIM(COALESCE(@include_category_groups, ''))
-            AS include_category_groups
-        , TRIM(COALESCE(@exclude_category_groups, ''))
-            AS exclude_category_groups
+        ''
+        , prm.exclude_category_groups || ','
+    FROM params AS prm
+    UNION ALL
+    SELECT
+        TRIM(SUBSTR(rest, 1, INSTR(rest, ',') - 1))
+        , SUBSTR(rest, INSTR(rest, ',') + 1)
+    FROM split_exclude_category_groups
+    WHERE rest != ''
+)
+
+, exclude_category_groups AS (
+    SELECT value AS name
+    FROM split_exclude_category_groups
+    WHERE value != ''
+)
+
+, matching_accounts AS (
+    SELECT
+        sp.id AS plan_id
+        , sp.name AS plan_name
+        , COUNT(*) AS matches
+    FROM scoped_plans AS sp
+    INNER JOIN accounts AS a ON sp.id = a.plan_id
+    CROSS JOIN params AS prm
+    WHERE NOT a.deleted AND a.name LIKE prm.account_name_like
+    GROUP BY sp.id, sp.name
 )
 
 , validation AS (
     SELECT
         p.account_name_like
+        , p.plan_id
         , p.include_category_groups
         , p.exclude_category_groups
     FROM params AS p
@@ -378,11 +416,62 @@ WITH params AS (
         || ' or @exclude_category_groups' AS error
     FROM validation AS v
     WHERE v.include_category_groups != '' AND v.exclude_category_groups != ''
+    UNION ALL
+    SELECT 'No plan matched @plan_id' AS error
+    FROM validation AS v
+    WHERE
+        v.plan_id != '' AND NOT EXISTS (
+            SELECT 1
+            FROM scoped_plans
+        )
+    UNION ALL
+    SELECT 'No account names matched @account_name_like' AS error
+    FROM validation AS v
+    WHERE
+        v.account_name_like != '' AND NOT EXISTS (
+            SELECT 1
+            FROM matching_accounts
+        )
+    UNION ALL
+    SELECT
+        'Matched more than 1 account in plan: '
+        || ma.plan_name AS error
+    FROM matching_accounts AS ma
+    WHERE ma.matches > 1
+    UNION ALL
+    SELECT
+        'Unknown include category group in plan '
+        || sp.name
+        || ': '
+        || icg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN include_category_groups AS icg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(icg.name)
+    WHERE cg.id IS NULL
+    UNION ALL
+    SELECT
+        'Unknown exclude category group in plan '
+        || sp.name
+        || ': '
+        || ecg.name AS error
+    FROM scoped_plans AS sp
+    CROSS JOIN exclude_category_groups AS ecg
+    LEFT JOIN category_groups AS cg
+        ON
+            sp.id = cg.plan_id
+            AND NOT COALESCE(cg.deleted, 0)
+            AND LOWER(cg.name) = LOWER(ecg.name)
+    WHERE cg.id IS NULL
 )
 
 , valid_params AS (
     SELECT
         v.account_name_like
+        , v.plan_id
         , v.include_category_groups
         , v.exclude_category_groups
     FROM validation AS v
@@ -405,7 +494,7 @@ WITH params AS (
         TRUE
         AND NOT a.deleted
         AND a.name LIKE v.account_name_like
-        AND (COALESCE(@plan_id, '') = '' OR p.id = @plan_id)
+        AND (v.plan_id = '' OR p.id = v.plan_id)
 )
 
 , category_totals AS (
@@ -420,36 +509,61 @@ WITH params AS (
         AND c.category_group_name != 'Internal Master Category'
         AND (
             v.include_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.include_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR EXISTS (
+                SELECT 1
+                FROM include_category_groups AS icg
+                WHERE LOWER(icg.name) = LOWER(c.category_group_name)
             )
-            > 0
         )
         AND (
             v.exclude_category_groups = ''
-            OR INSTR(
-                ','
-                || LOWER(REPLACE(v.exclude_category_groups, ', ', ','))
-                || ','
-                , ',' || LOWER(c.category_group_name) || ','
+            OR NOT EXISTS (
+                SELECT 1
+                FROM exclude_category_groups AS ecg
+                WHERE LOWER(ecg.name) = LOWER(c.category_group_name)
             )
-            = 0
         )
-        AND (COALESCE(@plan_id, '') = '' OR c.plan_id = @plan_id)
+        AND (v.plan_id = '' OR c.plan_id = v.plan_id)
     GROUP BY c.plan_id
 )
 
 SELECT
-    ma.plan_name AS "plan"
+    ve.error AS error_message
+    , NULL AS "plan"
+    , NULL AS account
+    , NULL AS total
+    , NULL AS excess
+FROM validation_errors AS ve
+
+UNION ALL
+
+SELECT
+    NULL AS error_message
+    , ma.plan_name AS "plan"
     , ma.account_name AS account
     , PRINTF('%.2f', COALESCE(ct.total, 0)) AS total
-    , PRINTF('%.2f', COALESCE(ct.total, 0) - ma.account_amount) AS excess
+    , PRINTF('%.2f', ma.account_amount - COALESCE(ct.total, 0)) AS excess
 FROM matched_accounts AS ma
 LEFT JOIN category_totals AS ct ON ma.plan_id = ct.plan_id
-ORDER BY "plan", account
+;
+
+SELECT error_message
+FROM excess_query_results
+WHERE error_message IS NOT NULL
+;
+
+SELECT
+    eqr."plan"
+    , eqr.account
+    , eqr.total
+    , eqr.excess
+FROM excess_query_results AS eqr
+WHERE
+    NOT EXISTS (
+        SELECT 1
+        FROM excess_query_results AS eqr_errors
+        WHERE eqr_errors.error_message IS NOT NULL
+    )
 ;
 ```
 

{sqlite_export_for_ynab-2.1.0 → sqlite_export_for_ynab-2.3.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = sqlite_export_for_ynab
-version = 2.1.0
+version = 2.3.0
 description = SQLite Export for YNAB - Export YNAB Data to SQLite
 long_description = file: README.md
 long_description_content_type = text/markdown