sqlite-export-for-ynab 0.0.1__tar.gz

sqlite_export_for_ynab-0.0.1/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.1
+Name: sqlite_export_for_ynab
+Version: 0.0.1
+Summary: SQLite Export for YNAB - Export YNAB Budget data to SQLite
+Home-page: https://github.com/mxr/sqlite-export-for-ynab
+Author: Max R
+Author-email: maxr@outlook.com
+License: MIT
+Keywords: ynab,sqlite,sql,budget,cli
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp
+Requires-Dist: tqdm

sqlite_export_for_ynab-0.0.1/pyproject.toml

@@ -0,0 +1,20 @@
+[tool.mypy]
+check_untyped_defs = true
+disallow_any_generics = true
+disallow_incomplete_defs = true
+disallow_untyped_defs = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+
+[tool.ruff]
+target-version = "py312"
+
+[tool.ruff.lint]
+extend-select = [
+  "UP",  # see pyupgrade
+  "B",  # see flake8-bugbear
+  "A",  # see flake8-builtins
+  "C4",  # see flake8-comprehension
+  "SIM",  # see flake8-simplify
+  "TCH"  # see flake8-type-checking
+]

sqlite_export_for_ynab-0.0.1/setup.cfg

@@ -0,0 +1,40 @@
+[metadata]
+name = sqlite_export_for_ynab
+version = 0.0.1
+description = SQLite Export for YNAB - Export YNAB Budget data to SQLite
+long_description = file: README.md
+long_description_content_type = text/markdown
+url = https://github.com/mxr/sqlite-export-for-ynab
+author = Max R
+author_email = maxr@outlook.com
+license = MIT
+license_files = LICENSE
+classifiers = 
+	License :: OSI Approved :: MIT License
+	Programming Language :: Python :: 3
+	Programming Language :: Python :: 3 :: Only
+	Programming Language :: Python :: Implementation :: CPython
+	Programming Language :: Python :: Implementation :: PyPy
+keywords = ynab, sqlite, sql, budget, cli
+
+[options]
+packages = find:
+install_requires = 
+	aiohttp
+	tqdm
+python_requires = >=3.12
+
+[options.entry_points]
+console_scripts = 
+	sqlite-export-for-ynab = sqlite_export_for_ynab._main:main
+
+[options.package_data]
+* = *.sqlite
+
+[bdist_wheel]
+universal = True
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

sqlite_export_for_ynab-0.0.1/setup.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from setuptools import setup
+
+setup()

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from sqlite_export_for_ynab._main import DEFAULT_DB
+from sqlite_export_for_ynab._main import sync
+
+__all__ = ["sync", "DEFAULT_DB"]

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab/__main__.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+from sqlite_export_for_ynab._main import main
+
+if __name__ == "__main__":
+    raise SystemExit(main())

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab/_main.py

@@ -0,0 +1,328 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import os
+import sqlite3
+from dataclasses import dataclass
+from importlib import resources
+from typing import Any
+from typing import ClassVar
+from typing import Literal
+from typing import overload
+from typing import Protocol
+from typing import TYPE_CHECKING
+from urllib.parse import urlencode
+from urllib.parse import urljoin
+from urllib.parse import urlunparse
+
+import aiohttp
+from tqdm import tqdm
+
+from sqlite_export_for_ynab import ddl
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable
+    from typing import Never
+
+
+_EntryTable = (
+    Literal["payees"]
+    | Literal["transactions"]
+    | Literal["subtransactions"]
+    | Literal["category_groups"]
+    | Literal["categories"]
+)
+_ALL_TABLES = frozenset(
+    ("budgets",) + tuple(lit.__args__[0] for lit in _EntryTable.__args__)
+)
+
+_ENV_TOKEN = "YNAB_PERSONAL_ACCESS_TOKEN"
+
+DEFAULT_DB = "sqlite-export-for-ynab.db"
+
+
+async def async_main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--db", help="The SQLite database file.", default=DEFAULT_DB)
+    parser.add_argument(
+        "--full-refresh",
+        action="store_true",
+        help="Drop all tables and fetch the budget.",
+    )
+
+    args = parser.parse_args()
+    db: str = args.db
+    full_refresh: bool = args.full_refresh
+
+    token = os.environ.get(_ENV_TOKEN)
+    if not token:
+        raise ValueError(
+            f"Must set YNAB access token as {_ENV_TOKEN!r} "
+            "environment variable. See "
+            "https://api.ynab.com/#personal-access-tokens"
+        )
+
+    await sync(db, full_refresh, token)
+
+    return 0
+
+
+async def sync(db: str, full_refresh: bool, token: str) -> None:
+    async with aiohttp.ClientSession() as session:
+        budgets = (await YnabClient(token, session)("budgets"))["budgets"]
+
+    budget_ids = [b["id"] for b in budgets]
+
+    with sqlite3.connect(db) as con:
+        con.row_factory = lambda c, row: dict(
+            zip([name for name, *_ in c.description], row, strict=True)
+        )
+        cur = con.cursor()
+
+        if full_refresh:
+            print("Dropping tables...")
+            cur.executescript(contents("drop-tables.sqlite"))
+            con.commit()
+            print("Done")
+
+        tables = get_tables(cur)
+        if tables != _ALL_TABLES:
+            print("Recreating tables...")
+            cur.executescript(contents("create-tables.sqlite"))
+            con.commit()
+            print("Done")
+
+        print("Fetching budget data...")
+        lkos = get_last_knowledge_of_server(cur)
+        async with aiohttp.ClientSession() as session:
+            with tqdm(desc="Budget Data", total=len(budgets) * 3) as pbar:
+                yc = ProgressYnabClient(
+                    YnabClient(os.environ["YNAB_PERSONAL_ACCESS_TOKEN"], session), pbar
+                )
+
+                txn_jobs = jobs(yc, "transactions", budget_ids, lkos)
+                payee_jobs = jobs(yc, "payees", budget_ids, lkos)
+                cat_jobs = jobs(yc, "categories", budget_ids, lkos)
+
+                data = await asyncio.gather(*txn_jobs, *payee_jobs, *cat_jobs)
+
+            all_txn_data = data[: len(txn_jobs)]
+            all_payee_data = data[len(txn_jobs) : len(txn_jobs) + len(payee_jobs)]
+            all_cat_data = data[len(txn_jobs) + len(payee_jobs) :]
+
+            new_lkos = {
+                bid: t["server_knowledge"]
+                for bid, t in zip(budget_ids, all_txn_data, strict=True)
+            }
+        print("Done")
+
+        if (
+            not any(t["transactions"] for t in all_txn_data)
+            and not any(p["payees"] for p in all_payee_data)
+            and not any(c["category_groups"] for c in all_cat_data)
+        ):
+            print("No data fetched")
+        else:
+            print("Inserting budget data...")
+            insert_budgets(cur, budgets, new_lkos)
+            for bid, txn_data in zip(budget_ids, all_txn_data, strict=True):
+                insert_transactions(cur, bid, txn_data["transactions"])
+            for bid, payee_data in zip(budget_ids, all_payee_data, strict=True):
+                insert_payees(cur, bid, payee_data["payees"])
+            for bid, cat_data in zip(budget_ids, all_cat_data, strict=True):
+                insert_category_groups(cur, bid, cat_data["category_groups"])
+            print("Done")
+
+
+def contents(filename: str) -> str:
+    return (resources.files(ddl) / filename).read_text()
+
+
+def jobs(
+    yc: SupportsYnabClient,
+    endpoint: Literal["transactions"] | Literal["categories"] | Literal["payees"],
+    budget_ids: list[str],
+    lkos: dict[str, int],
+) -> list[Awaitable[dict[str, Any]]]:
+    return [
+        yc(f"budgets/{bid}/{endpoint}", last_knowledge_of_server=lkos.get(bid))
+        for bid in budget_ids
+    ]
+
+
+def get_tables(cur: sqlite3.Cursor) -> set[str]:
+    return {
+        t["name"]
+        for t in cur.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'"
+        ).fetchall()
+    }
+
+
+def get_last_knowledge_of_server(cur: sqlite3.Cursor) -> dict[str, int]:
+    return {
+        r["id"]: r["last_knowledge_of_server"]
+        for r in cur.execute(
+            "SELECT id, last_knowledge_of_server FROM budgets",
+        ).fetchall()
+    }
+
+
+def insert_budgets(
+    cur: sqlite3.Cursor, budgets: list[dict[str, Any]], lkos: dict[str, int]
+) -> None:
+    cur.executemany(
+        "INSERT OR REPLACE INTO budgets (id, name, last_knowledge_of_server) VALUES (?, ?, ?)",
+        ((bid := b["id"], b["name"], lkos[bid]) for b in budgets),
+    )
+
+
+def insert_transactions(
+    cur: sqlite3.Cursor, budget_id: str, transactions: list[dict[str, Any]]
+) -> None:
+    return insert_nested_entries(
+        cur, budget_id, transactions, "Transactions", "transactions", "subtransactions"
+    )
+
+
+@overload
+def insert_nested_entries(
+    cur: sqlite3.Cursor,
+    budget_id: str,
+    entries: list[dict[str, Any]],
+    desc: Literal["Transactions"],
+    entries_name: Literal["transactions"],
+    subentries_name: Literal["subtransactions"],
+) -> None: ...
+
+
+@overload
+def insert_nested_entries(
+    cur: sqlite3.Cursor,
+    budget_id: str,
+    entries: list[dict[str, Any]],
+    desc: Literal["Categories"],
+    entries_name: Literal["category_groups"],
+    subentries_name: Literal["categories"],
+) -> None: ...
+
+
+def insert_nested_entries(
+    cur: sqlite3.Cursor,
+    budget_id: str,
+    entries: list[dict[str, Any]],
+    desc: Literal["Transactions"] | Literal["Categories"],
+    entries_name: Literal["transactions"] | Literal["category_groups"],
+    subentries_name: Literal["subtransactions"] | Literal["categories"],
+) -> None:
+    if not entries:
+        return
+
+    with tqdm(
+        total=sum(1 + len(e[subentries_name]) for e in entries),
+        desc=desc,
+    ) as pbar:
+        for entry in entries:
+            subentries = entry.pop(subentries_name, [])
+            insert_entry(cur, entries_name, budget_id, entry)
+            pbar.update()
+
+            for subentry in subentries:
+                insert_entry(cur, subentries_name, budget_id, subentry)
+                pbar.update()
+
+
+def insert_payees(
+    cur: sqlite3.Cursor, budget_id: str, payees: list[dict[str, Any]]
+) -> None:
+    if not payees:
+        return
+
+    for payee in tqdm(payees, desc="Payees"):
+        insert_entry(cur, "payees", budget_id, payee)
+
+
+def insert_category_groups(
+    cur: sqlite3.Cursor, budget_id: str, category_groups: list[dict[str, Any]]
+) -> None:
+    return insert_nested_entries(
+        cur, budget_id, category_groups, "Categories", "category_groups", "categories"
+    )
+
+
+def insert_entry(
+    cur: sqlite3.Cursor,
+    table: _EntryTable,
+    budget_id: str,
+    entry: dict[str, Any],
+) -> None:
+    ekeys, evalues = zip(*entry.items(), strict=True)
+    keys, values = ekeys + ("budget_id",), evalues + (budget_id,)
+
+    cur.execute(
+        f'INSERT OR REPLACE INTO {table} ({", ".join(keys)}) VALUES ({", ".join("?" * len(values))})',
+        values,
+    )
+
+
+class SupportsYnabClient(Protocol):
+    async def __call__(
+        self, path: str, last_knowledge_of_server: int | None = None
+    ) -> dict[str, Any]: ...
+
+
+@dataclass
+class ProgressYnabClient:
+    yc: YnabClient
+    pbar: tqdm[Never]
+
+    async def __call__(
+        self, path: str, last_knowledge_of_server: int | None = None
+    ) -> dict[str, Any]:
+        try:
+            return await self.yc(path, last_knowledge_of_server)
+        finally:
+            self.pbar.update()
+
+
+@dataclass
+class YnabClient:
+    BASE_SCHEME: ClassVar[str] = "https"
+    BASE_NETLOC: ClassVar[str] = "api.ynab.com"
+    BASE_PATH: ClassVar[str] = "v1/"
+
+    token: str
+    session: aiohttp.ClientSession
+
+    async def __call__(
+        self, path: str, last_knowledge_of_server: int | None = None
+    ) -> dict[str, Any]:
+        headers = {
+            "Authorization": f"Bearer {self.token}",
+            "Content-Type": "application/json",
+        }
+        url = urlunparse(
+            (
+                self.BASE_SCHEME,
+                self.BASE_NETLOC,
+                urljoin(self.BASE_PATH, path),
+                "",
+                urlencode(
+                    {"last_knowledge_of_server": last_knowledge_of_server}
+                    if last_knowledge_of_server
+                    else {}
+                ),
+                "",
+            )
+        )
+
+        async with self.session.get(url, headers=headers) as resp:
+            body = await resp.text()
+
+        return json.loads(body)["data"]
+
+
+def main() -> int:
+    return asyncio.run(async_main())

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab/ddl/create-tables.sqlite

@@ -0,0 +1,98 @@
+CREATE TABLE IF NOT EXISTS budgets (id TEXT primary key, name TEXT, last_knowledge_of_server INT)
+;
+
+CREATE TABLE IF NOT EXISTS payees (
+    id TEXT primary key,
+    budget_id TEXT,
+    name TEXT,
+    transfer_account_id TEXT,
+    deleted BOOLEAN,
+    foreign key (budget_id) references budgets (id)
+)
+;
+
+CREATE TABLE IF NOT EXISTS category_groups (
+    id TEXT primary key,
+    budget_id TEXT,
+    name TEXT,
+    hidden BOOLEAN,
+    deleted BOOLEAN,
+    foreign key (budget_id) references budgets (id)
+)
+;
+
+CREATE TABLE IF NOT EXISTS categories (
+    id TEXT primary key,
+    budget_id TEXT,
+    category_group_id TEXT,
+    category_group_name TEXT,
+    name TEXT,
+    hidden BOOLEAN,
+    original_category_group_id TEXT,
+    note TEXT,
+    budgeted INT,
+    activity INT,
+    balance INT,
+    goal_type TEXT,
+    goal_needs_whole_amount BOOLEAN,
+    goal_day INT,
+    goal_cadence INT,
+    goal_cadence_frequency INT,
+    goal_creation_month text,
+    goal_target INT,
+    goal_target_month text,
+    goal_percentage_complete INT,
+    goal_months_to_budget INT,
+    goal_under_funded INT,
+    goal_overall_funded INT,
+    goal_overall_left INT,
+    deleted BOOLEAN,
+    foreign key (budget_id) references budgets (id),
+    foreign key (category_group_id) references category_groups (id)
+)
+;
+
+CREATE TABLE IF NOT EXISTS transactions (
+    id TEXT primary key,
+    budget_id TEXT,
+    account_id TEXT,
+    account_name TEXT,
+    amount INT,
+    approved BOOLEAN,
+    category_id TEXT,
+    category_name TEXT,
+    cleared TEXT,
+    DATE TEXT,
+    debt_transaction_type TEXT,
+    deleted BOOLEAN,
+    flag_color TEXT,
+    flag_name TEXT,
+    import_id TEXT,
+    import_payee_name TEXT,
+    import_payee_name_original TEXT,
+    matched_transaction_id TEXT,
+    memo TEXT,
+    payee_id TEXT,
+    payee_name TEXT,
+    transfer_account_id TEXT,
+    transfer_transaction_id TEXT,
+    foreign key (budget_id) references budgets (id)
+)
+;
+
+CREATE TABLE IF NOT EXISTS subtransactions (
+    id TEXT primary key,
+    budget_id TEXT,
+    amount INT,
+    category_id TEXT,
+    category_name TEXT,
+    deleted BOOLEAN,
+    memo TEXT,
+    payee_id TEXT,
+    payee_name TEXT,
+    transaction_id TEXT,
+    transfer_account_id TEXT,
+    transfer_transaction_id TEXT,
+    foreign key (budget_id) references budget (id),
+    foreign key (transaction_id) references transaction_id (id)
+)

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab/ddl/drop-tables.sqlite

@@ -0,0 +1,17 @@
+DROP TABLE IF EXISTS budgets
+;
+
+DROP TABLE IF EXISTS payees
+;
+
+DROP TABLE IF EXISTS category_groups
+;
+
+DROP TABLE IF EXISTS categories
+;
+
+DROP TABLE IF EXISTS transactions
+;
+
+DROP TABLE IF EXISTS subtransactions
+;

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab.egg-info/PKG-INFO

@@ -0,0 +1,18 @@
+Metadata-Version: 2.1
+Name: sqlite_export_for_ynab
+Version: 0.0.1
+Summary: SQLite Export for YNAB - Export YNAB Budget data to SQLite
+Home-page: https://github.com/mxr/sqlite-export-for-ynab
+Author: Max R
+Author-email: maxr@outlook.com
+License: MIT
+Keywords: ynab,sqlite,sql,budget,cli
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: aiohttp
+Requires-Dist: tqdm

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+pyproject.toml
+setup.cfg
+setup.py
+sqlite_export_for_ynab/__init__.py
+sqlite_export_for_ynab/__main__.py
+sqlite_export_for_ynab/_main.py
+sqlite_export_for_ynab.egg-info/PKG-INFO
+sqlite_export_for_ynab.egg-info/SOURCES.txt
+sqlite_export_for_ynab.egg-info/dependency_links.txt
+sqlite_export_for_ynab.egg-info/entry_points.txt
+sqlite_export_for_ynab.egg-info/requires.txt
+sqlite_export_for_ynab.egg-info/top_level.txt
+sqlite_export_for_ynab/ddl/__init__.py
+sqlite_export_for_ynab/ddl/create-tables.sqlite
+sqlite_export_for_ynab/ddl/drop-tables.sqlite

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab.egg-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+sqlite-export-for-ynab = sqlite_export_for_ynab._main:main

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab.egg-info/requires.txt

@@ -0,0 +1,2 @@
+aiohttp
+tqdm

sqlite_export_for_ynab-0.0.1/sqlite_export_for_ynab.egg-info/top_level.txt

@@ -0,0 +1 @@
+sqlite_export_for_ynab