sqlite-export-for-ynab 2.4.0__tar.gz → 2.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {sqlite_export_for_ynab-2.4.0/sqlite_export_for_ynab.egg-info → sqlite_export_for_ynab-2.5.0}/PKG-INFO +3 -2
  2. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/setup.cfg +3 -2
  3. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab/_main.py +182 -170
  4. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0/sqlite_export_for_ynab.egg-info}/PKG-INFO +3 -2
  5. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab.egg-info/requires.txt +2 -1
  6. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/testing/fixtures.py +3 -17
  7. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/tests/_main_test.py +81 -68
  8. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/LICENSE +0 -0
  9. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/README.md +0 -0
  10. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/pyproject.toml +0 -0
  11. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/setup.py +0 -0
  12. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab/__init__.py +0 -0
  13. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab/__main__.py +0 -0
  14. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab/ddl/__init__.py +0 -0
  15. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab/ddl/create-relations.sql +0 -0
  16. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab/ddl/drop-relations.sql +0 -0
  17. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab/py.typed +0 -0
  18. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab.egg-info/SOURCES.txt +0 -0
  19. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab.egg-info/dependency_links.txt +0 -0
  20. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab.egg-info/entry_points.txt +0 -0
  21. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/sqlite_export_for_ynab.egg-info/top_level.txt +0 -0
  22. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/testing/__init__.py +0 -0
  23. {sqlite_export_for_ynab-2.4.0 → sqlite_export_for_ynab-2.5.0}/tests/__init__.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sqlite_export_for_ynab
3
- Version: 2.4.0
3
+ Version: 2.5.0
4
4
  Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
5
5
  Home-page: https://github.com/mxr/sqlite-export-for-ynab
6
6
  Author: Max R
@@ -15,8 +15,9 @@ Requires-Python: >=3.12
15
15
  Description-Content-Type: text/markdown
16
16
  License-File: LICENSE
17
17
  Requires-Dist: aiohttp>=3
18
+ Requires-Dist: aiopathlib
18
19
  Requires-Dist: aiosqlite
19
- Requires-Dist: tldm
20
+ Requires-Dist: rich
20
21
  Dynamic: license-file
21
22
 
22
23
  # sqlite-export-for-ynab
@@ -1,6 +1,6 @@
1
1
  [metadata]
2
2
  name = sqlite_export_for_ynab
3
- version = 2.4.0
3
+ version = 2.5.0
4
4
  description = SQLite Export for YNAB - Export YNAB Data to SQLite
5
5
  long_description = file: README.md
6
6
  long_description_content_type = text/markdown
@@ -20,8 +20,9 @@ keywords = ynab, sqlite, sql, budget, plan, cli
20
20
  packages = find:
21
21
  install_requires =
22
22
  aiohttp>=3
23
+ aiopathlib
23
24
  aiosqlite
24
- tldm
25
+ rich
25
26
  python_requires = >=3.12
26
27
 
27
28
  [options.entry_points]
@@ -4,6 +4,8 @@ import argparse
4
4
  import asyncio
5
5
  import json
6
6
  import os
7
+ from contextlib import asynccontextmanager
8
+ from contextlib import contextmanager
7
9
  from dataclasses import dataclass
8
10
  from importlib import resources
9
11
  from importlib.metadata import version
@@ -21,13 +23,18 @@ from urllib.parse import urlunparse
21
23
 
22
24
  import aiohttp
23
25
  import aiosqlite
24
- from tldm import tldm
26
+ from aiopathlib import AsyncPath
27
+ from rich.progress import BarColumn
28
+ from rich.progress import MofNCompleteColumn
29
+ from rich.progress import Progress
30
+ from rich.progress import TaskID
31
+ from rich.progress import TextColumn
32
+ from rich.progress import TimeElapsedColumn
25
33
 
26
34
  from sqlite_export_for_ynab import ddl
27
35
 
28
36
  if TYPE_CHECKING:
29
- from collections.abc import Awaitable, Sequence
30
- from typing import Never
37
+ from collections.abc import AsyncIterator, Awaitable, Iterator, Sequence
31
38
 
32
39
 
33
40
  _EntryTable = (
@@ -60,6 +67,13 @@ _PACKAGE = "sqlite-export-for-ynab"
60
67
 
61
68
  _BATCH_SIZE = 100
62
69
 
70
+ _PROGRESS_COLUMNS = (
71
+ TextColumn("[progress.description]{task.description}"),
72
+ BarColumn(),
73
+ MofNCompleteColumn(),
74
+ TimeElapsedColumn(),
75
+ )
76
+
63
77
 
64
78
  def resolve_token(token_override: str | None = None) -> str:
65
79
  token = token_override or os.environ.get(_ENV_TOKEN)
@@ -124,69 +138,92 @@ def _print(message: str, *, quiet: bool) -> None:
124
138
  print(message)
125
139
 
126
140
 
141
+ @dataclass
142
+ class _Context:
143
+ session: aiohttp.ClientSession
144
+ progress: Progress
145
+ con: aiosqlite.Connection
146
+
147
+
148
+ @asynccontextmanager
149
+ async def _context(db: Path, *, quiet: bool) -> AsyncIterator[_Context]:
150
+ progress = Progress(*_PROGRESS_COLUMNS, disable=quiet)
151
+ async with aiohttp.ClientSession() as session, aiosqlite.connect(db) as con:
152
+ con.row_factory = aiosqlite.Row
153
+ yield _Context(session, progress, con)
154
+
155
+
156
+ @contextmanager
157
+ def _progress(context: _Context) -> Iterator[None]:
158
+ context.progress.start()
159
+ try:
160
+ yield
161
+ finally:
162
+ context.progress.stop()
163
+ for task_id in context.progress.task_ids:
164
+ context.progress.remove_task(task_id)
165
+
166
+
127
167
  async def sync(
128
168
  token: str, db: Path, full_refresh: bool, *, quiet: bool = False
129
169
  ) -> None:
130
- async with aiohttp.ClientSession() as session:
131
- plans = (await YnabClient(token, session)("plans"))["plans"]
132
-
133
- plan_ids = [plan["id"] for plan in plans]
170
+ await AsyncPath(db).parent.mkdir(parents=True, exist_ok=True)
134
171
 
135
- if not db.exists():
136
- db.parent.mkdir(parents=True, exist_ok=True)
172
+ async with _context(db, quiet=quiet) as context:
173
+ plans = (await YnabClient(token, context.session)("plans"))["plans"]
137
174
 
138
- async with aiosqlite.connect(db) as con:
139
- con.row_factory = aiosqlite.Row
175
+ plan_ids = [plan["id"] for plan in plans]
140
176
 
141
177
  if full_refresh:
142
178
  _print("Dropping relations...", quiet=quiet)
143
- async with con.cursor() as cur:
144
- await cur.executescript(contents("drop-relations.sql"))
145
- await con.commit()
179
+ async with context.con.cursor() as cur:
180
+ await cur.executescript(await contents("drop-relations.sql"))
181
+ await context.con.commit()
146
182
  _print("Done", quiet=quiet)
147
183
 
148
- async with con.cursor() as cur:
184
+ async with context.con.cursor() as cur:
149
185
  relations = await get_relations(cur)
150
186
  if relations != _ALL_RELATIONS:
151
187
  _print("Recreating relations...", quiet=quiet)
152
- await cur.executescript(contents("create-relations.sql"))
153
- await con.commit()
188
+ await cur.executescript(await contents("create-relations.sql"))
189
+ await context.con.commit()
154
190
  _print("Done", quiet=quiet)
155
191
 
156
192
  _print("Fetching plan data...", quiet=quiet)
157
- async with con.cursor() as cur:
193
+ async with context.con.cursor() as cur:
158
194
  lkos = await get_last_knowledge_of_server(cur)
159
- async with aiohttp.ClientSession() as session:
160
- with tldm(
161
- desc="Plan Data", total=len(plans) * len(_ENDPOINTS), disable=quiet
162
- ) as pbar:
163
- yc = ProgressYnabClient(YnabClient(token, session), pbar)
164
-
165
- endpoint_data = dict(
166
- zip(
167
- _ENDPOINTS,
168
- await asyncio.gather(
169
- *(
170
- asyncio.gather(*jobs(yc, endpoint, plan_ids, lkos))
171
- for endpoint in _ENDPOINTS
172
- )
173
- ),
174
- strict=True,
175
- )
195
+ with _progress(context):
196
+ yc = ProgressYnabClient(
197
+ YnabClient(token, context.session),
198
+ context,
199
+ context.progress.add_task(
200
+ "Plan Data", total=len(plans) * len(_ENDPOINTS)
201
+ ),
202
+ )
203
+
204
+ endpoint_data = dict(
205
+ zip(
206
+ _ENDPOINTS,
207
+ await asyncio.gather(
208
+ *(
209
+ asyncio.gather(*jobs(yc, endpoint, plan_ids, lkos))
210
+ for endpoint in _ENDPOINTS
211
+ )
212
+ ),
213
+ strict=True,
176
214
  )
215
+ )
177
216
 
178
- all_account_data = endpoint_data["accounts"]
179
- all_cat_data = endpoint_data["categories"]
180
- all_payee_data = endpoint_data["payees"]
181
- all_txn_data = endpoint_data["transactions"]
182
- all_sched_txn_data = endpoint_data["scheduled_transactions"]
217
+ all_account_data = endpoint_data["accounts"]
218
+ all_cat_data = endpoint_data["categories"]
219
+ all_payee_data = endpoint_data["payees"]
220
+ all_txn_data = endpoint_data["transactions"]
221
+ all_sched_txn_data = endpoint_data["scheduled_transactions"]
183
222
 
184
- new_lkos = {
185
- plan_id: transaction_data["server_knowledge"]
186
- for plan_id, transaction_data in zip(
187
- plan_ids, all_txn_data, strict=True
188
- )
189
- }
223
+ new_lkos = {
224
+ plan_id: transaction_data["server_knowledge"]
225
+ for plan_id, transaction_data in zip(plan_ids, all_txn_data, strict=True)
226
+ }
190
227
  _print("Done", quiet=quiet)
191
228
 
192
229
  if (
@@ -199,63 +236,24 @@ async def sync(
199
236
  _print("No new data fetched", quiet=quiet)
200
237
  else:
201
238
  _print("Inserting plan data...", quiet=quiet)
202
- await insert_plans(con, plans, new_lkos)
203
- await asyncio.gather(
204
- *(
205
- insert_accounts(
206
- con,
207
- plan_id,
208
- account_data["accounts"],
209
- quiet=quiet,
210
- )
211
- for plan_id, account_data in zip(
212
- plan_ids, all_account_data, strict=True
213
- )
214
- ),
215
- *(
216
- insert_category_groups(
217
- con,
218
- plan_id,
219
- cat_data["category_groups"],
220
- quiet=quiet,
221
- )
222
- for plan_id, cat_data in zip(plan_ids, all_cat_data, strict=True)
223
- ),
224
- *(
225
- insert_payees(con, plan_id, payee_data["payees"], quiet=quiet)
226
- for plan_id, payee_data in zip(
227
- plan_ids, all_payee_data, strict=True
228
- )
229
- ),
230
- )
231
- await asyncio.gather(
232
- *(
233
- insert_transactions(
234
- con,
235
- plan_id,
236
- txn_data["transactions"],
237
- quiet=quiet,
238
- )
239
- for plan_id, txn_data in zip(plan_ids, all_txn_data, strict=True)
240
- ),
241
- *(
242
- insert_scheduled_transactions(
243
- con,
244
- plan_id,
245
- sched_txn_data["scheduled_transactions"],
246
- quiet=quiet,
247
- )
248
- for plan_id, sched_txn_data in zip(
249
- plan_ids, all_sched_txn_data, strict=True
250
- )
251
- ),
252
- )
253
- await con.commit()
239
+ with _progress(context):
240
+ await insert_plan_data(
241
+ context,
242
+ plans,
243
+ plan_ids,
244
+ all_account_data,
245
+ all_cat_data,
246
+ all_payee_data,
247
+ all_txn_data,
248
+ all_sched_txn_data,
249
+ new_lkos,
250
+ )
251
+ await context.con.commit()
254
252
  _print("Done", quiet=quiet)
255
253
 
256
254
 
257
- def contents(filename: str) -> str:
258
- return (resources.files(ddl) / filename).read_text()
255
+ async def contents(filename: str) -> str:
256
+ return await AsyncPath(resources.files(ddl) / filename).read_text()
259
257
 
260
258
 
261
259
  async def get_relations(cur: aiosqlite.Cursor) -> set[str]:
@@ -272,10 +270,52 @@ async def get_last_knowledge_of_server(cur: aiosqlite.Cursor) -> dict[str, int]:
272
270
  return {r["id"]: r["last_knowledge_of_server"] for r in await cur.fetchall()}
273
271
 
274
272
 
273
+ async def insert_plan_data(
274
+ context: _Context,
275
+ plans: list[dict[str, Any]],
276
+ plan_ids: list[str],
277
+ all_account_data: list[dict[str, Any]],
278
+ all_cat_data: list[dict[str, Any]],
279
+ all_payee_data: list[dict[str, Any]],
280
+ all_txn_data: list[dict[str, Any]],
281
+ all_sched_txn_data: list[dict[str, Any]],
282
+ new_lkos: dict[str, int],
283
+ ) -> None:
284
+ await insert_plans(context, plans, new_lkos)
285
+ await asyncio.gather(
286
+ *(
287
+ insert_accounts(context, plan_id, account_data["accounts"])
288
+ for plan_id, account_data in zip(plan_ids, all_account_data, strict=True)
289
+ ),
290
+ *(
291
+ insert_category_groups(context, plan_id, cat_data["category_groups"])
292
+ for plan_id, cat_data in zip(plan_ids, all_cat_data, strict=True)
293
+ ),
294
+ *(
295
+ insert_payees(context, plan_id, payee_data["payees"])
296
+ for plan_id, payee_data in zip(plan_ids, all_payee_data, strict=True)
297
+ ),
298
+ )
299
+ await asyncio.gather(
300
+ *(
301
+ insert_transactions(context, plan_id, txn_data["transactions"])
302
+ for plan_id, txn_data in zip(plan_ids, all_txn_data, strict=True)
303
+ ),
304
+ *(
305
+ insert_scheduled_transactions(
306
+ context, plan_id, sched_txn_data["scheduled_transactions"]
307
+ )
308
+ for plan_id, sched_txn_data in zip(
309
+ plan_ids, all_sched_txn_data, strict=True
310
+ )
311
+ ),
312
+ )
313
+
314
+
275
315
  async def insert_plans(
276
- con: aiosqlite.Connection, plans: list[dict[str, Any]], lkos: dict[str, int]
316
+ context: _Context, plans: list[dict[str, Any]], lkos: dict[str, int]
277
317
  ) -> None:
278
- async with con.cursor() as cur:
318
+ async with context.con.cursor() as cur:
279
319
  for plan_batch in batched(plans, _BATCH_SIZE):
280
320
  await cur.executemany(
281
321
  """
@@ -316,11 +356,9 @@ _LOAN_ACCOUNT_PERIODIC_VALUES = frozenset(
316
356
 
317
357
 
318
358
  async def insert_accounts(
319
- con: aiosqlite.Connection,
359
+ context: _Context,
320
360
  plan_id: str,
321
361
  accounts: list[dict[str, Any]],
322
- *,
323
- quiet: bool = False,
324
362
  ) -> None:
325
363
  # YNAB's LoanAccountPeriodValues are untyped dicts so we need to turn them into a more standard sub-entry view
326
364
  updated_accounts = [
@@ -341,146 +379,126 @@ async def insert_accounts(
341
379
  ]
342
380
 
343
381
  await insert_nested_entries(
344
- con,
382
+ context,
345
383
  plan_id,
346
384
  updated_accounts,
347
385
  "Accounts",
348
386
  "accounts",
349
387
  "account_periodic_values",
350
388
  "account_periodic_values",
351
- quiet=quiet,
352
389
  )
353
390
 
354
391
 
355
392
  async def insert_category_groups(
356
- con: aiosqlite.Connection,
393
+ context: _Context,
357
394
  plan_id: str,
358
395
  category_groups: list[dict[str, Any]],
359
- *,
360
- quiet: bool = False,
361
396
  ) -> None:
362
397
  await insert_nested_entries(
363
- con,
398
+ context,
364
399
  plan_id,
365
400
  category_groups,
366
401
  "Categories",
367
402
  "category_groups",
368
403
  "categories",
369
404
  "categories",
370
- quiet=quiet,
371
405
  )
372
406
 
373
407
 
374
408
  async def insert_payees(
375
- con: aiosqlite.Connection,
409
+ context: _Context,
376
410
  plan_id: str,
377
411
  payees: list[dict[str, Any]],
378
- *,
379
- quiet: bool = False,
380
412
  ) -> None:
381
413
  if not payees:
382
414
  return
383
415
 
384
- with tldm(total=len(payees), desc="Payees", disable=quiet) as pbar:
385
- await insert_entries(con, "payees", plan_id, payees, pbar)
416
+ task_id = context.progress.add_task("Payees", total=len(payees))
417
+ await insert_entries(context, "payees", plan_id, payees, task_id)
386
418
 
387
419
 
388
420
  async def insert_transactions(
389
- con: aiosqlite.Connection,
421
+ context: _Context,
390
422
  plan_id: str,
391
423
  transactions: list[dict[str, Any]],
392
- *,
393
- quiet: bool = False,
394
424
  ) -> None:
395
425
  await insert_nested_entries(
396
- con,
426
+ context,
397
427
  plan_id,
398
428
  transactions,
399
429
  "Transactions",
400
430
  "transactions",
401
431
  "subtransactions",
402
432
  "subtransactions",
403
- quiet=quiet,
404
433
  )
405
434
 
406
435
 
407
436
  async def insert_scheduled_transactions(
408
- con: aiosqlite.Connection,
437
+ context: _Context,
409
438
  plan_id: str,
410
439
  scheduled_transactions: list[dict[str, Any]],
411
- *,
412
- quiet: bool = False,
413
440
  ) -> None:
414
441
  await insert_nested_entries(
415
- con,
442
+ context,
416
443
  plan_id,
417
444
  scheduled_transactions,
418
445
  "Scheduled Transactions",
419
446
  "scheduled_transactions",
420
447
  "subtransactions",
421
448
  "scheduled_subtransactions",
422
- quiet=quiet,
423
449
  )
424
450
 
425
451
 
426
452
  @overload
427
453
  async def insert_nested_entries(
428
- con: aiosqlite.Connection,
454
+ context: _Context,
429
455
  plan_id: str,
430
456
  entries: list[dict[str, Any]],
431
457
  desc: Literal["Accounts"],
432
458
  entries_name: Literal["accounts"],
433
459
  subentries_name: Literal["account_periodic_values"],
434
460
  subentries_table_name: Literal["account_periodic_values"],
435
- *,
436
- quiet: bool = False,
437
461
  ) -> None: ...
438
462
 
439
463
 
440
464
  @overload
441
465
  async def insert_nested_entries(
442
- con: aiosqlite.Connection,
466
+ context: _Context,
443
467
  plan_id: str,
444
468
  entries: list[dict[str, Any]],
445
469
  desc: Literal["Categories"],
446
470
  entries_name: Literal["category_groups"],
447
471
  subentries_name: Literal["categories"],
448
472
  subentries_table_name: Literal["categories"],
449
- *,
450
- quiet: bool = False,
451
473
  ) -> None: ...
452
474
 
453
475
 
454
476
  @overload
455
477
  async def insert_nested_entries(
456
- con: aiosqlite.Connection,
478
+ context: _Context,
457
479
  plan_id: str,
458
480
  entries: list[dict[str, Any]],
459
481
  desc: Literal["Transactions"],
460
482
  entries_name: Literal["transactions"],
461
483
  subentries_name: Literal["subtransactions"],
462
484
  subentries_table_name: Literal["subtransactions"],
463
- *,
464
- quiet: bool = False,
465
485
  ) -> None: ...
466
486
 
467
487
 
468
488
  @overload
469
489
  async def insert_nested_entries(
470
- con: aiosqlite.Connection,
490
+ context: _Context,
471
491
  plan_id: str,
472
492
  entries: list[dict[str, Any]],
473
493
  desc: Literal["Scheduled Transactions"],
474
494
  entries_name: Literal["scheduled_transactions"],
475
495
  subentries_name: Literal["subtransactions"],
476
496
  subentries_table_name: Literal["scheduled_subtransactions"],
477
- *,
478
- quiet: bool = False,
479
497
  ) -> None: ...
480
498
 
481
499
 
482
500
  async def insert_nested_entries(
483
- con: aiosqlite.Connection,
501
+ context: _Context,
484
502
  plan_id: str,
485
503
  entries: list[dict[str, Any]],
486
504
  desc: (
@@ -506,42 +524,35 @@ async def insert_nested_entries(
506
524
  | Literal["subtransactions"]
507
525
  | Literal["scheduled_subtransactions"]
508
526
  ),
509
- *,
510
- quiet: bool = False,
511
527
  ) -> None:
512
528
  if not entries:
513
529
  return
514
530
 
515
- with tldm(
516
- total=sum(1 + len(e[subentries_name]) for e in entries),
517
- desc=desc,
518
- disable=quiet,
519
- ) as pbar:
520
- await insert_entries(
521
- con,
522
- entries_name,
523
- plan_id,
524
- [
525
- {k: v for k, v in entry.items() if k != subentries_name}
526
- for entry in entries
527
- ],
528
- pbar,
529
- )
530
- await insert_entries(
531
- con,
532
- subentries_table_name,
533
- plan_id,
534
- [subentry for entry in entries for subentry in entry[subentries_name]],
535
- pbar,
536
- )
531
+ task_id = context.progress.add_task(
532
+ desc, total=sum(1 + len(e[subentries_name]) for e in entries)
533
+ )
534
+ await insert_entries(
535
+ context,
536
+ entries_name,
537
+ plan_id,
538
+ [{k: v for k, v in entry.items() if k != subentries_name} for entry in entries],
539
+ task_id,
540
+ )
541
+ await insert_entries(
542
+ context,
543
+ subentries_table_name,
544
+ plan_id,
545
+ [subentry for entry in entries for subentry in entry[subentries_name]],
546
+ task_id,
547
+ )
537
548
 
538
549
 
539
550
  async def insert_entries(
540
- con: aiosqlite.Connection,
551
+ context: _Context,
541
552
  table: _EntryTable,
542
553
  plan_id: str,
543
554
  entries: list[dict[str, Any]],
544
- pbar: tldm[Never],
555
+ task_id: TaskID,
545
556
  ) -> None:
546
557
  if not entries:
547
558
  return
@@ -549,14 +560,14 @@ async def insert_entries(
549
560
  entry_keys = tuple(entries[0])
550
561
  sql = f"INSERT OR REPLACE INTO {table} ({', '.join(entry_keys + ('plan_id',))}) VALUES ({', '.join('?' * (len(entry_keys) + 1))})"
551
562
 
552
- async with con.cursor() as cur:
563
+ async with context.con.cursor() as cur:
553
564
  for entry_batch in batched(entries, _BATCH_SIZE):
554
565
  values_batch = [
555
566
  tuple(entry[key] for key in entry_keys) + (plan_id,)
556
567
  for entry in entry_batch
557
568
  ]
558
569
  await cur.executemany(sql, values_batch)
559
- pbar.update(len(values_batch))
570
+ context.progress.update(task_id, advance=len(values_batch))
560
571
 
561
572
 
562
573
  def jobs(
@@ -580,7 +591,8 @@ class SupportsYnabClient(Protocol):
580
591
  @dataclass
581
592
  class ProgressYnabClient:
582
593
  yc: YnabClient
583
- pbar: tldm[Never]
594
+ context: _Context
595
+ task_id: TaskID
584
596
 
585
597
  async def __call__(
586
598
  self, path: str, last_knowledge_of_server: int | None = None
@@ -588,7 +600,7 @@ class ProgressYnabClient:
588
600
  try:
589
601
  return await self.yc(path, last_knowledge_of_server)
590
602
  finally:
591
- self.pbar.update()
603
+ self.context.progress.update(self.task_id, advance=1)
592
604
 
593
605
 
594
606
  @dataclass
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: sqlite_export_for_ynab
3
- Version: 2.4.0
3
+ Version: 2.5.0
4
4
  Summary: SQLite Export for YNAB - Export YNAB Data to SQLite
5
5
  Home-page: https://github.com/mxr/sqlite-export-for-ynab
6
6
  Author: Max R
@@ -15,8 +15,9 @@ Requires-Python: >=3.12
15
15
  Description-Content-Type: text/markdown
16
16
  License-File: LICENSE
17
17
  Requires-Dist: aiohttp>=3
18
+ Requires-Dist: aiopathlib
18
19
  Requires-Dist: aiosqlite
19
- Requires-Dist: tldm
20
+ Requires-Dist: rich
20
21
  Dynamic: license-file
21
22
 
22
23
  # sqlite-export-for-ynab
@@ -1,3 +1,4 @@
1
1
  aiohttp>=3
2
+ aiopathlib
2
3
  aiosqlite
3
- tldm
4
+ rich
@@ -2,14 +2,14 @@ from __future__ import annotations
2
2
 
3
3
  import re
4
4
  from typing import Any
5
+ from typing import TYPE_CHECKING
5
6
  from uuid import uuid4
6
7
 
7
- import aiosqlite
8
8
  import pytest
9
- import pytest_asyncio
10
9
  from aioresponses import aioresponses
11
10
 
12
- from sqlite_export_for_ynab._main import contents
11
+ if TYPE_CHECKING:
12
+ import aiosqlite
13
13
 
14
14
  PLAN_ID_1 = str(uuid4())
15
15
  PLAN_ID_2 = str(uuid4())
@@ -353,20 +353,6 @@ SCHEDULED_TRANSACTIONS: list[dict[str, Any]] = [
353
353
  ]
354
354
 
355
355
 
356
- @pytest_asyncio.fixture
357
- async def con():
358
- async with aiosqlite.connect(":memory:") as con:
359
- con.row_factory = aiosqlite.Row
360
- await con.executescript(contents("create-relations.sql"))
361
- yield con
362
-
363
-
364
- @pytest_asyncio.fixture
365
- async def cur(con):
366
- async with con.cursor() as cursor:
367
- yield cursor
368
-
369
-
370
356
  @pytest.fixture
371
357
  def mock_aioresponses():
372
358
  with aioresponses() as m:
@@ -9,13 +9,16 @@ from unittest.mock import patch
9
9
  import aiohttp
10
10
  import aiosqlite
11
11
  import pytest
12
+ import pytest_asyncio
12
13
  from aiohttp.http_exceptions import HttpProcessingError
13
- from tldm import tldm
14
+ from rich.progress import Progress
14
15
 
15
16
  from sqlite_export_for_ynab import default_db_path
16
17
  from sqlite_export_for_ynab._main import _ALL_RELATIONS
18
+ from sqlite_export_for_ynab._main import _Context
17
19
  from sqlite_export_for_ynab._main import _ENV_TOKEN
18
20
  from sqlite_export_for_ynab._main import _PACKAGE
21
+ from sqlite_export_for_ynab._main import _PROGRESS_COLUMNS
19
22
  from sqlite_export_for_ynab._main import contents
20
23
  from sqlite_export_for_ynab._main import get_last_knowledge_of_server
21
24
  from sqlite_export_for_ynab._main import get_relations
@@ -49,8 +52,6 @@ from testing.fixtures import CATEGORY_NAME_1
49
52
  from testing.fixtures import CATEGORY_NAME_2
50
53
  from testing.fixtures import CATEGORY_NAME_3
51
54
  from testing.fixtures import CATEGORY_NAME_4
52
- from testing.fixtures import con
53
- from testing.fixtures import cur
54
55
  from testing.fixtures import EXAMPLE_ENDPOINT_RE
55
56
  from testing.fixtures import LKOS
56
57
  from testing.fixtures import mock_aioresponses
@@ -87,6 +88,18 @@ async def fetchall(con, query):
87
88
  return await cur.fetchall()
88
89
 
89
90
 
91
+ @pytest_asyncio.fixture
92
+ async def context():
93
+ with Progress(*_PROGRESS_COLUMNS, disable=True) as progress:
94
+ async with (
95
+ aiohttp.ClientSession(loop=asyncio.get_event_loop()) as session,
96
+ aiosqlite.connect(":memory:") as con,
97
+ ):
98
+ con.row_factory = aiosqlite.Row
99
+ await con.executescript(await contents("create-relations.sql"))
100
+ yield _Context(session, progress, con)
101
+
102
+
90
103
  @pytest.mark.parametrize(
91
104
  ("xdg_data_home", "expected_prefix"),
92
105
  (
@@ -99,26 +112,25 @@ def test_default_db_path(monkeypatch, xdg_data_home, expected_prefix):
99
112
  assert default_db_path() == expected_prefix / "sqlite-export-for-ynab" / "db.sqlite"
100
113
 
101
114
 
102
- @pytest.mark.usefixtures(cur.__name__)
103
115
  @pytest.mark.asyncio
104
- async def test_get_relations(cur):
105
- assert await get_relations(cur) == _ALL_RELATIONS
116
+ async def test_get_relations(context):
117
+ async with context.con.cursor() as cur:
118
+ assert await get_relations(cur) == _ALL_RELATIONS
106
119
 
107
120
 
108
- @pytest.mark.usefixtures(con.__name__)
109
121
  @pytest.mark.asyncio
110
- async def test_get_last_knowledge_of_server(con):
111
- await insert_plans(con, PLANS, LKOS)
112
- async with con.cursor() as cur:
122
+ async def test_get_last_knowledge_of_server(context):
123
+ await insert_plans(context, PLANS, LKOS)
124
+ async with context.con.cursor() as cur:
113
125
  assert await get_last_knowledge_of_server(cur) == LKOS
114
126
 
115
127
 
116
- @pytest.mark.usefixtures(con.__name__)
117
128
  @pytest.mark.asyncio
118
- async def test_insert_plans(con):
119
- await insert_plans(con, PLANS, LKOS)
129
+ async def test_insert_plans(context):
130
+ await insert_plans(context, PLANS, LKOS)
120
131
  assert [
121
- strip_nones(d) for d in await fetchall(con, "SELECT * FROM plans ORDER BY name")
132
+ strip_nones(d)
133
+ for d in await fetchall(context.con, "SELECT * FROM plans ORDER BY name")
122
134
  ] == [
123
135
  {
124
136
  "id": PLAN_ID_1,
@@ -147,17 +159,16 @@ async def test_insert_plans(con):
147
159
  ]
148
160
 
149
161
 
150
- @pytest.mark.usefixtures(con.__name__)
151
162
  @pytest.mark.asyncio
152
- async def test_insert_accounts(con):
153
- await insert_accounts(con, PLAN_ID_1, [])
154
- assert not await fetchall(con, "SELECT * FROM accounts")
155
- assert not await fetchall(con, "SELECT * FROM account_periodic_values")
163
+ async def test_insert_accounts(context):
164
+ await insert_accounts(context, PLAN_ID_1, [])
165
+ assert not await fetchall(context.con, "SELECT * FROM accounts")
166
+ assert not await fetchall(context.con, "SELECT * FROM account_periodic_values")
156
167
 
157
- await insert_accounts(con, PLAN_ID_1, ACCOUNTS)
168
+ await insert_accounts(context, PLAN_ID_1, ACCOUNTS)
158
169
  assert [
159
170
  strip_nones(d)
160
- for d in await fetchall(con, "SELECT * FROM accounts ORDER BY name")
171
+ for d in await fetchall(context.con, "SELECT * FROM accounts ORDER BY name")
161
172
  ] == [
162
173
  {
163
174
  "id": ACCOUNT_ID_1,
@@ -188,7 +199,7 @@ async def test_insert_accounts(con):
188
199
  assert [
189
200
  strip_nones(d)
190
201
  for d in await fetchall(
191
- con, "SELECT * FROM account_periodic_values ORDER BY name"
202
+ context.con, "SELECT * FROM account_periodic_values ORDER BY name"
192
203
  )
193
204
  ] == [
194
205
  {
@@ -208,17 +219,18 @@ async def test_insert_accounts(con):
208
219
  ]
209
220
 
210
221
 
211
- @pytest.mark.usefixtures(con.__name__)
212
222
  @pytest.mark.asyncio
213
- async def test_insert_category_groups(con):
214
- await insert_category_groups(con, PLAN_ID_1, [])
215
- assert not await fetchall(con, "SELECT * FROM category_groups")
216
- assert not await fetchall(con, "SELECT * FROM categories")
223
+ async def test_insert_category_groups(context):
224
+ await insert_category_groups(context, PLAN_ID_1, [])
225
+ assert not await fetchall(context.con, "SELECT * FROM category_groups")
226
+ assert not await fetchall(context.con, "SELECT * FROM categories")
217
227
 
218
- await insert_category_groups(con, PLAN_ID_1, CATEGORY_GROUPS)
228
+ await insert_category_groups(context, PLAN_ID_1, CATEGORY_GROUPS)
219
229
  assert [
220
230
  strip_nones(d)
221
- for d in await fetchall(con, "SELECT * FROM category_groups ORDER BY name")
231
+ for d in await fetchall(
232
+ context.con, "SELECT * FROM category_groups ORDER BY name"
233
+ )
222
234
  ] == [
223
235
  {
224
236
  "id": CATEGORY_GROUP_ID_1,
@@ -234,7 +246,7 @@ async def test_insert_category_groups(con):
234
246
 
235
247
  assert [
236
248
  strip_nones(d)
237
- for d in await fetchall(con, "SELECT * FROM categories ORDER BY name")
249
+ for d in await fetchall(context.con, "SELECT * FROM categories ORDER BY name")
238
250
  ] == [
239
251
  {
240
252
  "id": CATEGORY_ID_1,
@@ -300,20 +312,21 @@ async def test_insert_category_groups(con):
300
312
  ]
301
313
 
302
314
 
303
- @pytest.mark.usefixtures(con.__name__)
304
315
  @pytest.mark.asyncio
305
- async def test_insert_category_group_without_categories(con):
316
+ async def test_insert_category_group_without_categories(context):
306
317
  category_group = {
307
318
  "id": CATEGORY_GROUP_ID_1,
308
319
  "name": CATEGORY_GROUP_NAME_1,
309
320
  "categories": [],
310
321
  }
311
322
 
312
- await insert_category_groups(con, PLAN_ID_1, [category_group])
323
+ await insert_category_groups(context, PLAN_ID_1, [category_group])
313
324
 
314
325
  assert [
315
326
  strip_nones(d)
316
- for d in await fetchall(con, "SELECT * FROM category_groups ORDER BY name")
327
+ for d in await fetchall(
328
+ context.con, "SELECT * FROM category_groups ORDER BY name"
329
+ )
317
330
  ] == [
318
331
  {
319
332
  "id": CATEGORY_GROUP_ID_1,
@@ -321,19 +334,18 @@ async def test_insert_category_group_without_categories(con):
321
334
  "plan_id": PLAN_ID_1,
322
335
  },
323
336
  ]
324
- assert not await fetchall(con, "SELECT * FROM categories")
337
+ assert not await fetchall(context.con, "SELECT * FROM categories")
325
338
 
326
339
 
327
- @pytest.mark.usefixtures(con.__name__)
328
340
  @pytest.mark.asyncio
329
- async def test_insert_payees(con):
330
- await insert_payees(con, PLAN_ID_1, [])
331
- assert not await fetchall(con, "SELECT * FROM payees")
341
+ async def test_insert_payees(context):
342
+ await insert_payees(context, PLAN_ID_1, [])
343
+ assert not await fetchall(context.con, "SELECT * FROM payees")
332
344
 
333
- await insert_payees(con, PLAN_ID_1, PAYEES)
345
+ await insert_payees(context, PLAN_ID_1, PAYEES)
334
346
  assert [
335
347
  strip_nones(d)
336
- for d in await fetchall(con, "SELECT * FROM payees ORDER BY name")
348
+ for d in await fetchall(context.con, "SELECT * FROM payees ORDER BY name")
337
349
  ] == [
338
350
  {
339
351
  "id": PAYEE_ID_1,
@@ -348,18 +360,17 @@ async def test_insert_payees(con):
348
360
  ]
349
361
 
350
362
 
351
- @pytest.mark.usefixtures(con.__name__)
352
363
  @pytest.mark.asyncio
353
- async def test_insert_transactions(con):
354
- await insert_transactions(con, PLAN_ID_1, [])
355
- assert not await fetchall(con, "SELECT * FROM transactions")
356
- assert not await fetchall(con, "SELECT * FROM subtransactions")
364
+ async def test_insert_transactions(context):
365
+ await insert_transactions(context, PLAN_ID_1, [])
366
+ assert not await fetchall(context.con, "SELECT * FROM transactions")
367
+ assert not await fetchall(context.con, "SELECT * FROM subtransactions")
357
368
 
358
- await insert_category_groups(con, PLAN_ID_1, CATEGORY_GROUPS)
359
- await insert_transactions(con, PLAN_ID_1, TRANSACTIONS)
369
+ await insert_category_groups(context, PLAN_ID_1, CATEGORY_GROUPS)
370
+ await insert_transactions(context, PLAN_ID_1, TRANSACTIONS)
360
371
  assert [
361
372
  strip_nones(d)
362
- for d in await fetchall(con, "SELECT * FROM transactions ORDER BY date")
373
+ for d in await fetchall(context.con, "SELECT * FROM transactions ORDER BY date")
363
374
  ] == [
364
375
  {
365
376
  "id": TRANSACTION_ID_1,
@@ -401,7 +412,9 @@ async def test_insert_transactions(con):
401
412
 
402
413
  assert [
403
414
  strip_nones(d)
404
- for d in await fetchall(con, "SELECT * FROM subtransactions ORDER BY amount")
415
+ for d in await fetchall(
416
+ context.con, "SELECT * FROM subtransactions ORDER BY amount"
417
+ )
405
418
  ] == [
406
419
  {
407
420
  "id": SUBTRANSACTION_ID_1,
@@ -429,7 +442,9 @@ async def test_insert_transactions(con):
429
442
 
430
443
  assert [
431
444
  strip_nones(d)
432
- for d in await fetchall(con, "SELECT * FROM flat_transactions ORDER BY amount")
445
+ for d in await fetchall(
446
+ context.con, "SELECT * FROM flat_transactions ORDER BY amount"
447
+ )
433
448
  ] == [
434
449
  {
435
450
  "transaction_id": TRANSACTION_ID_1,
@@ -462,19 +477,18 @@ async def test_insert_transactions(con):
462
477
  ]
463
478
 
464
479
 
465
- @pytest.mark.usefixtures(con.__name__)
466
480
  @pytest.mark.asyncio
467
- async def test_insert_scheduled_transactions(con):
468
- await insert_scheduled_transactions(con, PLAN_ID_1, [])
469
- assert not await fetchall(con, "SELECT * FROM scheduled_transactions")
470
- assert not await fetchall(con, "SELECT * FROM scheduled_subtransactions")
481
+ async def test_insert_scheduled_transactions(context):
482
+ await insert_scheduled_transactions(context, PLAN_ID_1, [])
483
+ assert not await fetchall(context.con, "SELECT * FROM scheduled_transactions")
484
+ assert not await fetchall(context.con, "SELECT * FROM scheduled_subtransactions")
471
485
 
472
- await insert_category_groups(con, PLAN_ID_1, CATEGORY_GROUPS)
473
- await insert_scheduled_transactions(con, PLAN_ID_1, SCHEDULED_TRANSACTIONS)
486
+ await insert_category_groups(context, PLAN_ID_1, CATEGORY_GROUPS)
487
+ await insert_scheduled_transactions(context, PLAN_ID_1, SCHEDULED_TRANSACTIONS)
474
488
  assert [
475
489
  strip_nones(d)
476
490
  for d in await fetchall(
477
- con, "SELECT * FROM scheduled_transactions ORDER BY amount"
491
+ context.con, "SELECT * FROM scheduled_transactions ORDER BY amount"
478
492
  )
479
493
  ] == [
480
494
  {
@@ -515,7 +529,7 @@ async def test_insert_scheduled_transactions(con):
515
529
  assert [
516
530
  strip_nones(d)
517
531
  for d in await fetchall(
518
- con, "SELECT * FROM scheduled_subtransactions ORDER BY amount"
532
+ context.con, "SELECT * FROM scheduled_subtransactions ORDER BY amount"
519
533
  )
520
534
  ] == [
521
535
  {
@@ -545,7 +559,7 @@ async def test_insert_scheduled_transactions(con):
545
559
  assert [
546
560
  strip_nones(d)
547
561
  for d in await fetchall(
548
- con, "SELECT * FROM scheduled_flat_transactions ORDER BY amount"
562
+ context.con, "SELECT * FROM scheduled_flat_transactions ORDER BY amount"
549
563
  )
550
564
  ] == [
551
565
  {
@@ -594,14 +608,13 @@ async def test_insert_scheduled_transactions(con):
594
608
 
595
609
  @pytest.mark.asyncio
596
610
  @pytest.mark.usefixtures(mock_aioresponses.__name__)
597
- async def test_progress_ynab_client_ok(mock_aioresponses):
611
+ async def test_progress_ynab_client_ok(context, mock_aioresponses):
598
612
  expected = {"example": [{"id": 1, "value": 2}, {"id": 3, "value": 4}]}
599
613
  mock_aioresponses.get(EXAMPLE_ENDPOINT_RE, body=json.dumps({"data": expected}))
600
614
 
601
- with tldm(disable=True) as pbar:
602
- async with aiohttp.ClientSession(loop=asyncio.get_event_loop()) as session:
603
- pyc = ProgressYnabClient(YnabClient(TOKEN, session), pbar)
604
- entries = await pyc("example")
615
+ task_id = context.progress.add_task("Example", total=1)
616
+ pyc = ProgressYnabClient(YnabClient(TOKEN, context.session), context, task_id)
617
+ entries = await pyc("example")
605
618
 
606
619
  assert entries == expected
607
620
 
@@ -722,7 +735,7 @@ async def test_sync_no_data(tmp_path, mock_aioresponses):
722
735
  # create the db and tables to exercise all code branches
723
736
  db = tmp_path / "db.sqlite"
724
737
  async with aiosqlite.connect(db) as con:
725
- await con.executescript(contents("create-relations.sql"))
738
+ await con.executescript(await contents("create-relations.sql"))
726
739
 
727
740
  await sync(TOKEN, db, False)
728
741
 
@@ -766,7 +779,7 @@ async def test_sync_no_data_quiet(tmp_path, mock_aioresponses, capsys):
766
779
 
767
780
  db = tmp_path / "db.sqlite"
768
781
  async with aiosqlite.connect(db) as con:
769
- await con.executescript(contents("create-relations.sql"))
782
+ await con.executescript(await contents("create-relations.sql"))
770
783
 
771
784
  await sync(TOKEN, db, False, quiet=True)
772
785