dimwit 0.3.0__tar.gz → 0.3.2__tar.gz

{dimwit-0.3.0 → dimwit-0.3.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dimwit
-Version: 0.3.0
+Version: 0.3.2
 Summary: A package containing various functions and classes for fetching, transforming, and visualising data for personal metrics.
 Author-email: Daniel Soutar <danielsoutar144@gmail.com>
 Project-URL: repository, https://github.com/danielsoutar/dimwit

{dimwit-0.3.0 → dimwit-0.3.2}/dimwit/main.py

@@ -314,119 +314,6 @@ def group_data(
     return grouped_ts, grouped_events
 
 
-# This enables reading from a (chunked) database with good compression and
-# substantially faster reads for older data, and only a small amount of syncing
-# required with the slower database.
-# This does assume the presence of a 'date' column in the un-chunked database,
-# however. This enables the filtering on the slower database.
-def get_notion_pages_from_db_sync_with_chunked(
-    db_id: str,
-    chunked_db_id: str,
-    headers: NotionHeaders,
-    extract_chunked_func: Callable,
-    decompression_map: dict,
-    compression_map: dict,
-    extract_func: Callable,
-    compress_func: Callable,
-    patch_chunked_page: Callable,
-    post_new_chunked_page: Callable,
-):
-    # Read all the data from chunked_db in ascending order of id.
-    chunked_pages = get_notion_pages_from_db(
-        chunked_db_id,
-        headers,
-        sort_column="id",
-        sort_direction="ascending",
-    )
-
-    # Get the latest date from the chunked data.
-    latest_chunked_page = [chunked_pages[-1]] if chunked_pages else []
-    chunked_page_ts, chunked_page_events = get_all_entries_chunked(
-        latest_chunked_page, extract_chunked_func, decompression_map
-    )
-    latest_chunked_date: dat.datetime | None = (
-        chunked_page_ts[-1] if chunked_page_ts else None
-    )
-    chunked_page_id: str | None = chunked_pages[-1]["id"] if chunked_pages else None
-    print(f"Latest chunked timestamp: {latest_chunked_date}")
-    print(f"Latest chunked page ID: {chunked_page_id = }")
-
-    # Read all the data from slower_db s.t. date >= latest chunked date.
-    # Typicall this should be a small amount of data - only on an empty chunked
-    # database should this pay the full cost of loading/syncing all the data.
-    filter_by = None
-    if latest_chunked_date:
-        filter_by = {
-            "property": "date",
-            "date": {"on_or_after": latest_chunked_date.isoformat()},
-        }
-    unchunked_pages = get_notion_pages_from_db(
-        db_id,
-        headers,
-        sort_column="date",
-        sort_direction="ascending",
-        filter_by=filter_by,
-    )
-    unchunked_ts, unchunked_events = get_all_entries(
-        unchunked_pages,
-        extract_func,
-    )
-
-    if not latest_chunked_date:
-        # Useful debugging utility when creating a new chunked DB.
-        unchunked_event_freqs = {}
-        for event in unchunked_events:
-            if event not in unchunked_event_freqs:
-                unchunked_event_freqs[event] = 0
-            unchunked_event_freqs[event] += 1
-        print(f"{unchunked_event_freqs = }")
-
-    earliest_unchunked = None if len(unchunked_ts) == 0 else unchunked_ts[0]
-    latest_unchunked = None if len(unchunked_ts) == 0 else unchunked_ts[-1]
-    print(f"Earliest unchunked timestamp: {earliest_unchunked}")
-    print(f"Latest unchunked timestamp: {latest_unchunked}")
-
-    # De-duplicate the data and transform into standardised chunked pages (i.e.
-    # update the latest chunked page and add any new chunked pages).
-    if latest_chunked_date:
-        dedup_chunked_pairs = [
-            (t, e)
-            for t, e in zip(chunked_page_ts, chunked_page_events)
-            if t < latest_chunked_date
-        ]
-        dedup_ts, dedup_events = zip(*dedup_chunked_pairs)
-        chunked_page_ts = list(dedup_ts)
-        chunked_page_events = list(dedup_events)
-
-    ts = chunked_page_ts + unchunked_ts
-    events = chunked_page_events + unchunked_events
-    # Group data for better compression.
-    grouped_ts, grouped_events = group_data(ts, events)
-    new_chunked_pages = compress_event_data(
-        grouped_ts,
-        grouped_events,
-        compression_map,
-        chunked_db_id,
-    )
-
-    # PATCH/POST the latest chunked page and any new chunked pages.
-    for idx, chunked_page in enumerate(new_chunked_pages):
-        if idx == 0 and chunked_page_id:
-            patch_chunked_page(
-                headers,
-                chunked_page_id,
-                chunked_page,
-            )
-            continue
-        post_new_chunked_page(
-            headers,
-            chunked_page,
-        )
-
-    # Return the chunked pages.
-    return chunked_pages[:-1] + new_chunked_pages
-
-
 def patch_chunked_page(
     headers: NotionHeaders,
     page_id: str,
@@ -633,6 +520,118 @@ def get_all_entries_chunked(pages, add_data_entry, decompression_map):
     return timestamps, data
 
 
+# This enables reading from a (chunked) database with good compression and
+# substantially faster reads for older data, and only a small amount of syncing
+# required with the slower database.
+# This does assume the presence of a 'date' column in the un-chunked database,
+# however. This enables the filtering on the slower database.
+def get_notion_pages_from_db_sync_with_chunked(
+    db_id: str,
+    chunked_db_id: str,
+    headers: NotionHeaders,
+    decompression_map: dict,
+    compression_map: dict,
+    extract_chunked_func: Callable = extract_chunked_categorical_entry,
+    extract_func: Callable = extract_categorical_entry,
+    patch_chunked_page_func: Callable = patch_chunked_page,
+    post_new_chunked_page_func: Callable = post_new_chunked_page,
+):
+    # Read all the data from chunked_db in ascending order of id.
+    chunked_pages = get_notion_pages_from_db(
+        chunked_db_id,
+        headers,
+        sort_column="id",
+        sort_direction="ascending",
+    )
+
+    # Get the latest date from the chunked data.
+    latest_chunked_page = [chunked_pages[-1]] if chunked_pages else []
+    chunked_page_ts, chunked_page_events = get_all_entries_chunked(
+        latest_chunked_page, extract_chunked_func, decompression_map
+    )
+    latest_chunked_date: dat.datetime | None = (
+        chunked_page_ts[-1] if chunked_page_ts else None
+    )
+    chunked_page_id: str | None = chunked_pages[-1]["id"] if chunked_pages else None
+    print(f"Latest chunked timestamp: {latest_chunked_date}")
+    print(f"Latest chunked page ID: {chunked_page_id = }")
+
+    # Read all the data from slower_db s.t. date >= latest chunked date.
+    # Typicall this should be a small amount of data - only on an empty chunked
+    # database should this pay the full cost of loading/syncing all the data.
+    filter_by = None
+    if latest_chunked_date:
+        filter_by = {
+            "property": "date",
+            "date": {"on_or_after": latest_chunked_date.isoformat()},
+        }
+    unchunked_pages = get_notion_pages_from_db(
+        db_id,
+        headers,
+        sort_column="date",
+        sort_direction="ascending",
+        filter_by=filter_by,
+    )
+    unchunked_ts, unchunked_events = get_all_entries(
+        unchunked_pages,
+        extract_func,
+    )
+
+    if not latest_chunked_date:
+        # Useful debugging utility when creating a new chunked DB.
+        unchunked_event_freqs = {}
+        for event in unchunked_events:
+            if event not in unchunked_event_freqs:
+                unchunked_event_freqs[event] = 0
+            unchunked_event_freqs[event] += 1
+        print(f"{unchunked_event_freqs = }")
+
+    earliest_unchunked = None if len(unchunked_ts) == 0 else unchunked_ts[0]
+    latest_unchunked = None if len(unchunked_ts) == 0 else unchunked_ts[-1]
+    print(f"Earliest unchunked timestamp: {earliest_unchunked}")
+    print(f"Latest unchunked timestamp: {latest_unchunked}")
+
+    # De-duplicate the data and transform into standardised chunked pages (i.e.
+    # update the latest chunked page and add any new chunked pages).
+    if latest_chunked_date:
+        dedup_chunked_pairs = [
+            (t, e)
+            for t, e in zip(chunked_page_ts, chunked_page_events)
+            if t < latest_chunked_date
+        ]
+        dedup_ts, dedup_events = zip(*dedup_chunked_pairs)
+        chunked_page_ts = list(dedup_ts)
+        chunked_page_events = list(dedup_events)
+
+    ts = chunked_page_ts + unchunked_ts
+    events = chunked_page_events + unchunked_events
+    # Group data for better compression.
+    grouped_ts, grouped_events = group_data(ts, events)
+    new_chunked_pages = compress_event_data(
+        grouped_ts,
+        grouped_events,
+        compression_map,
+        chunked_db_id,
+    )
+
+    # PATCH/POST the latest chunked page and any new chunked pages.
+    for idx, chunked_page in enumerate(new_chunked_pages):
+        if idx == 0 and chunked_page_id:
+            patch_chunked_page_func(
+                headers,
+                chunked_page_id,
+                chunked_page,
+            )
+            continue
+        post_new_chunked_page_func(
+            headers,
+            chunked_page,
+        )
+
+    # Return the chunked pages.
+    return chunked_pages[:-1] + new_chunked_pages
+
+
 def get_n_weeks_ago(n):
     now = dat.datetime.now().astimezone()
     current_week_start = now - dat.timedelta(days=now.weekday())

{dimwit-0.3.0 → dimwit-0.3.2}/dimwit.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dimwit
-Version: 0.3.0
+Version: 0.3.2
 Summary: A package containing various functions and classes for fetching, transforming, and visualising data for personal metrics.
 Author-email: Daniel Soutar <danielsoutar144@gmail.com>
 Project-URL: repository, https://github.com/danielsoutar/dimwit

{dimwit-0.3.0 → dimwit-0.3.2}/pyproject.toml

@@ -14,7 +14,7 @@ dependencies = [
     "tqdm>=4.67.1",
 ]
 name = "dimwit"
-version = "0.3.0"
+version = "0.3.2"
 description = "A package containing various functions and classes for fetching, transforming, and visualising data for personal metrics."
 readme = "README.md"