memberjojo 2.1__py3-none-any.whl → 2.3__py3-none-any.whl

memberjojo/mojo_loader.py

@@ -1,12 +1,33 @@
 #!/usr/bin/env python3
 """
-Helper module for importing a CSV into a SQLite database.
+Helper module for importing a CSV into a SQLite database
 """
 
+from collections import defaultdict, Counter
 from csv import DictReader
+from dataclasses import dataclass
+from io import StringIO
 from pathlib import Path
+from typing import Any, IO, Tuple
+
 import re
-from collections import defaultdict, Counter
+import sqlite3 as sqlite3_builtin
+
+import requests
+
+
+@dataclass(frozen=True)
+class DiffRow:
+    """
+    Represents a single diff result
+
+    - diff_type: 'added' | 'deleted' | 'changed'
+    - preview: tuple of values, with preview[0] == key
+    """
+
+    diff_type: str
+    preview: Tuple[Any, ...]
+
 
 # -----------------------
 # Normalization & Type Guessing
@@ -15,11 +36,11 @@ from collections import defaultdict, Counter
 
 def _normalize(name: str) -> str:
     """
-    Normalize a column name: lowercase, remove symbols, convert to snake case.
+    Normalize a column name: lowercase, remove symbols, convert to snake case
 
-    :param name: Raw name to normalize.
+    :param name: Raw name to normalize
 
-    :return: Normalized lowercase string in snake case with no symbols.
+    :return: Normalized lowercase string in snake case with no symbols
     """
     name = name.strip().lower()
     name = re.sub(r"[^a-z0-9]+", "_", name)
@@ -28,7 +49,11 @@ def _normalize(name: str) -> str:
 
 def _guess_type(value: any) -> str:
     """
-    Guess SQLite data type of a CSV value: 'INTEGER', 'REAL', or 'TEXT'.
+    Guess SQLite data type of a CSV value: 'INTEGER', 'REAL', or 'TEXT'
+
+    :param value: entry from sqlite database to guess the type of
+
+    :return: string of the type, TEXT, INTEGER, REAL
     """
     if value is None:
         return "TEXT"
@@ -49,8 +74,12 @@ def _guess_type(value: any) -> str:
 
 def infer_columns_from_rows(rows: list[dict]) -> dict[str, str]:
     """
-    Infer column types from CSV rows.
-    Returns mapping: normalized column name -> SQLite type.
+    Infer column types from CSV rows
+    Returns mapping: normalized column name -> SQLite type
+
+    :param rows: list of rows to use for inference
+
+    :return: dict of name, type for columns
     """
     type_counters = defaultdict(Counter)
 
@@ -78,18 +107,21 @@ def infer_columns_from_rows(rows: list[dict]) -> dict[str, str]:
 
 def _create_table_from_columns(table_name: str, columns: dict[str, str]) -> str:
     """
-    Generate CREATE TABLE SQL from column type mapping.
+    Generate CREATE TABLE SQL from column type mapping
+    Adds an auto-incrementing rowid as the primary key
 
-    :param table_name: Table to use when creating columns.
-    :param columns: dict of columns to create.
+    :param table_name: Table to use when creating columns
+    :param columns: dict of columns to create
 
-    :return: SQL commands to create the table.
+    :return: SQL commands to create the table
     """
-    column_defs = [f'"{col}" {col_type}' for col, col_type in columns.items()]
+    col_defs = ["rowid INTEGER PRIMARY KEY AUTOINCREMENT"]
+
+    for col, col_type in columns.items():
+        col_defs.append(f'"{col}" {col_type}')
+
     return (
-        f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n'
-        + ",\n".join(column_defs)
-        + "\n)"
+        f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n' + ",\n".join(col_defs) + "\n)"
     )
 
 
@@ -98,14 +130,47 @@ def _create_table_from_columns(table_name: str, columns: dict[str, str]) -> str:
 # -----------------------
 
 
+def import_data(conn, table_name: str, reader: DictReader):
+    """
+    Import data in the DictReader into the SQLite3 database at conn
+
+    :param conn: SQLite database connection to use
+    :param table_name: Name of the table to import into
+    :param reader: A Dictreader object to import from
+    """
+    inferred_cols = infer_columns_from_rows(reader)
+
+    cursor = conn.cursor()
+    # Drop existing table
+    cursor.execute(f'DROP TABLE IF EXISTS "{table_name}";')
+
+    # Create table
+    create_sql = _create_table_from_columns(table_name, inferred_cols)
+    cursor.execute(create_sql)
+
+    # Insert rows
+    cols = list(reader[0].keys())
+    norm_map = {c: _normalize(c) for c in cols}
+    colnames = ",".join(f'"{norm_map[c]}"' for c in cols)
+    placeholders = ",".join("?" for _ in cols)
+    insert_sql = f'INSERT INTO "{table_name}" ({colnames}) VALUES ({placeholders})'
+
+    for row in reader:
+        values = [row[c] if row[c] != "" else None for c in cols]
+        cursor.execute(insert_sql, values)
+
+    cursor.close()
+    conn.commit()
+
+
 def import_csv_helper(conn, table_name: str, csv_path: Path):
     """
-    Import CSV into database using given cursor.
-    Column types inferred automatically.
+    Import CSV into database using given cursor
+    Column types inferred automatically
 
-    :param conn: SQLite database connection to use.
-    :param table_name: Table to import the CSV into.
-    :param csv_path: Path like path of the CSV file to import.
+    :param conn: SQLite database connection to use
+    :param table_name: Table to import the CSV into
+    :param csv_path: Path like path of the CSV file to import
     """
     if not csv_path.exists():
         raise FileNotFoundError(f"CSV file not found: {csv_path}")
@@ -115,26 +180,205 @@ def import_csv_helper(conn, table_name: str, csv_path: Path):
         reader = list(DictReader(f))
         if not reader:
             raise ValueError("CSV file is empty.")
-        inferred_cols = infer_columns_from_rows(reader)
+        import_data(conn, table_name, reader)
+
+
+# -----------------------
+# diff generation
+# -----------------------
+
+
+def _diffrow_from_sql_row(row: sqlite3_builtin.Row) -> DiffRow:
+    """
+    Convert a sqlite3.Row from generate_sql_diff into DiffRow
+    Row shape:
+        (diff_type, col1, col2, col3, ...)
 
-        cursor = conn.cursor()
-        # Drop existing table
-        cursor.execute(f'DROP TABLE IF EXISTS "{table_name}";')
+    :param row: Row from sqlite3 database to create a dataclass entry from
 
-        # Create table
-        create_sql = _create_table_from_columns(table_name, inferred_cols)
-        cursor.execute(create_sql)
+    :return: A dataclass of the row
+    """
+    return DiffRow(
+        diff_type=row[0],
+        preview=tuple(row[1:]),
+    )
 
-        # Insert rows
-        cols = list(reader[0].keys())
-        norm_map = {c: _normalize(c) for c in cols}
-        colnames = ",".join(f'"{norm_map[c]}"' for c in cols)
-        placeholders = ",".join("?" for _ in cols)
-        insert_sql = f'INSERT INTO "{table_name}" ({colnames}) VALUES ({placeholders})'
 
-        for row in reader:
-            values = [row[c] if row[c] != "" else None for c in cols]
-            cursor.execute(insert_sql, values)
+def diff_cipher_tables(
+    conn,
+    *,
+    new_table: str,
+    old_table: str,
+) -> list[DiffRow]:
+    """
+    Copy old and new tables from SQLCipher into a single
+    in-memory sqlite3 database and diff them there.
 
-    cursor.close()
-    conn.commit()
+    :param conn: sqlite connection to the db
+    :param new_table: name of the new table for comparison
+    :param old_table: name of the old table for comparison
+
+    :return: a list of DiffRow entries of the changed rows
+    """
+
+    plain = sqlite3_builtin.connect(":memory:")
+    plain.row_factory = sqlite3_builtin.Row
+
+    try:
+        for table in (old_table, new_table):
+            # Clone schema using SQLite itself
+            schema_sql = conn.execute(
+                """
+                SELECT sql
+                FROM sqlite_master
+                WHERE type='table' AND name=?
+                """,
+                (table,),
+            ).fetchone()
+
+            if schema_sql is None:
+                raise RuntimeError(f"Table {table!r} not found in cipher DB")
+
+            plain.execute(schema_sql[0])
+
+            # 2. Copy data
+            rows = conn.execute(f"SELECT * FROM {table}")
+            cols = [d[0] for d in rows.description]
+
+            col_list = ", ".join(cols)
+            placeholders = ", ".join("?" for _ in cols)
+
+            plain.executemany(
+                f"INSERT INTO {table} ({col_list}) VALUES ({placeholders})",
+                rows.fetchall(),
+            )
+
+        # 3. Run sqlite-only diff
+        rows = _generate_sql_diff(
+            plain,
+            new_table=new_table,
+            old_table=old_table,
+        )
+
+        return [_diffrow_from_sql_row(r) for r in rows]
+
+    finally:
+        plain.close()
+
+
+def _generate_sql_diff(
+    conn: sqlite3_builtin.Connection,
+    *,
+    new_table: str,
+    old_table: str,
+) -> list[sqlite3_builtin.Row]:
+    """
+    Generate a diff between two tables using standard SQLite features
+
+    - Uses rowid as the primary key for joining
+    - Returned row shape:
+            (diff_type, preview_col1, preview_col2, preview_col3, ...)
+
+    :param conn: sqlite connection to the db, using python builtin sqlite
+    :param new_table: name of the new table to use for comparison
+    :param old_table: name of the old table to use for comparison
+
+    :return: list of sqlite rows that are changed
+    """
+
+    # Introspect schema (order-preserving)
+    cols_info = conn.execute(f"PRAGMA table_info({new_table})").fetchall()
+
+    if not cols_info:
+        raise RuntimeError(f"Table {new_table!r} has no columns")
+
+    cols = [row[1] for row in cols_info]
+
+    # Exclude rowid from consideration for key or data
+    main_cols = [c for c in cols if c != "rowid"]
+
+    # First column is key, others are for comparison
+    key = main_cols[0]
+    non_key_cols = main_cols[1:]
+
+    #  Preview columns (key first, then others for readability)
+    preview_cols = [key] + non_key_cols[:5]
+
+    new_preview = ", ".join(f"n.{c}" for c in preview_cols)
+    old_preview = ", ".join(f"o.{c}" for c in preview_cols)
+
+    # Row-value comparison (NULL-safe)
+    if non_key_cols:
+        changed_predicate = (
+            f"({', '.join(f'n.{c}' for c in non_key_cols)}) "
+            f"IS NOT "
+            f"({', '.join(f'o.{c}' for c in non_key_cols)})"
+        )
+    else:
+        # Key-only table
+        changed_predicate = "0"
+
+    sql = f"""
+        WITH
+            added AS (
+                SELECT 'added' AS diff_type, {new_preview}
+                FROM {new_table} n
+                LEFT JOIN {old_table} o USING ({key})
+                WHERE o.{key} IS NULL
+            ),
+            deleted AS (
+                SELECT 'deleted' AS diff_type, {old_preview}
+                FROM {old_table} o
+                LEFT JOIN {new_table} n USING ({key})
+                WHERE n.{key} IS NULL
+            ),
+            changed AS (
+                SELECT 'changed' AS diff_type, {new_preview}
+                FROM {new_table} n
+                JOIN {old_table} o USING ({key})
+                WHERE {changed_predicate}
+            )
+        SELECT * FROM added
+        UNION ALL
+        SELECT * FROM deleted
+        UNION ALL
+        SELECT * FROM changed
+        ORDER BY {key};
+        """
+
+    return list(conn.execute(sql))
+
+
+def download_csv_helper(
+    conn, table_name: str, url: str, session: requests.Session
+) -> IO[str]:
+    """
+    Download url into a StringIO file object using streaming
+    and import into database
+
+    :param conn: The SQLite3 database connection to use
+    :param table_name: The name of the table to import it into
+    :param url: URL of the csv to download
+    :param session: A requests session to use for the download
+    """
+
+    print(f"Downloading from: {url}")
+
+    # Enable streaming
+    with session.get(url, stream=True) as resp:
+        resp.raise_for_status()
+
+        # Initialize the string buffer
+        string_buffer = StringIO()
+
+        # Stream decoded text
+        # decode_unicode=True uses the encoding from the response headers
+        for chunk in resp.iter_content(chunk_size=8192, decode_unicode=True):
+            if chunk:
+                string_buffer.write(chunk)
+
+        # Reset pointer to the beginning for DictReader
+        string_buffer.seek(0)
+
+        print(f"✅ Downloaded with encoding {resp.encoding}.")
+        import_data(conn, table_name, list(DictReader(string_buffer)))

memberjojo/mojo_member.py

@@ -1,25 +1,28 @@
 """
-Member module for creating and interacting with a SQLite database.
+Member module for creating and interacting with a SQLite database
 
 This module loads data from a `members.csv` file downloaded from Membermojo,
-stores it in SQLite, and provides helper functions for member lookups.
+stores it in SQLite, and provides helper functions for member lookups
 """
 
+from difflib import get_close_matches
 from pathlib import Path
+from pprint import pprint
 from typing import Optional
 from .mojo_common import MojoSkel
 
 
 class Member(MojoSkel):
     """
-    Subclass of MojoSkel providing member-specific database functions.
+    Subclass of MojoSkel providing member-specific database functions
 
     This class connects to a SQLite database and supports importing member data
-    from CSV and performing queries like lookup by name or member number.
+    from CSV and performing queries like lookup by name or member number
 
-    :param member_db_path (Path): Path to the SQLite database file.
-    :param table_name (str): (optional) Table name to use. Defaults to "members".
-    :param db_key: (optional) key to unlock the encrypted sqlite database, unencrypted if unset.
+    :param member_db_path (Path): Path to the SQLite database file
+    :param db_key: key to unlock the encrypted sqlite database,
+        unencrypted if sqlcipher3 not installed or unset
+    :param table_name (str): (optional) Table name to use. Defaults to "members"
     """
 
     def __init__(
@@ -29,23 +32,50 @@ class Member(MojoSkel):
         table_name: str = "members",
     ):
         """
-        Initialize the Member database handler.
+        Initialize the Member database handler
         """
         super().__init__(member_db_path, db_key, table_name)
 
+    def get_bool(self, entry_name: str, member_number: int) -> bool:
+        """
+        Return a bool for a member entry that is a tick box on membermojo
+
+        :param entry_name: The entry name to return as a bool
+        :param member_number: The member number to check value of entry_name
+
+        :return: True is entry is yes otherwise False
+
+        :raises ValueError: If entry name not found
+        """
+        sql = f"""
+            SELECT "{entry_name}"
+            FROM "{self.table_name}"
+            WHERE "member_number" = ?
+        """
+        self.cursor.execute(sql, (member_number,))
+
+        row = self.cursor.fetchone()
+        if row is None:
+            raise ValueError(
+                f"❌ Cannot find: {entry_name} for member {member_number}."
+            )
+
+        value = row[0]
+        return str(value).lower() == "yes"
+
     def get_number_first_last(
         self, first_name: str, last_name: str, found_error: bool = False
     ) -> Optional[int]:
         """
-        Find a member number based on first and last name (case-insensitive).
+        Find a member number based on first and last name (case-insensitive)
 
-        :param first_name: First name of the member.
-        :param last_name: Last name of the member.
-        :param found_error: (optional): If True, raises ValueError if not found.
+        :param first_name: First name of the member
+        :param last_name: Last name of the member
+        :param found_error: (optional): If True, raises ValueError if not found
 
-        :return: The member number if found, otherwise None.
+        :return: The member number if found, otherwise None
 
-        :raises ValueError: If not found and `found_error` is True.
+        :raises ValueError: If not found and `found_error` is True
         """
         sql = f"""
             SELECT "member_number"
@@ -54,6 +84,12 @@ class Member(MojoSkel):
         """
         self.cursor.execute(sql, (first_name, last_name))
         result = self.cursor.fetchone()
+        if self.debug:
+            print("Sql:")
+            pprint(sql)
+            print(f"First Name: {first_name} Last Name: {last_name}")
+            print("Result:")
+            pprint(result[0])
 
         if not result and found_error:
             raise ValueError(
@@ -64,18 +100,22 @@ class Member(MojoSkel):
 
     def get_number(self, full_name: str, found_error: bool = False) -> Optional[int]:
         """
-        Find a member number by passed full_name.
+        Find a member number by passed full_name
         Tries first and last, and then middle last if 3 words,
-        Then initial of first name if initials passed.
+        Then initial of first name if initials passed
+        Finnaly a fuzzy lookup is tried
 
-        :param full_name: Full name of the member.
-        :param found_error: (optional) Raise ValueError if not found.
+        :param full_name: Full name of the member
+        :param found_error: (optional) Raise ValueError if not found
 
-        :return: Member number if found, else None.
+        :return: Member number if found, else None
 
-        :raises ValueError: If not found and `found_error` is True.
+        :raises ValueError: If not found and `found_error` is True
         """
-        result = self.get_mojo_name(full_name, found_error)
+        result = self.get_mojo_name(full_name)
+        if not result:
+            result = self.get_fuzz_name(full_name, found_error)
+
         if result:
             return self.get_number_first_last(result[0], result[1])
         return None
@@ -123,7 +163,7 @@ class Member(MojoSkel):
         self, full_name: str, found_error: bool = False
     ) -> Optional[tuple]:
         """
-        Resolve a member name from a free-text full name.
+        Resolve a member name from a free-text full name
 
         **Search order**
 
@@ -132,14 +172,14 @@ class Member(MojoSkel):
         3. initial 1st letter + last
         4. initial 2nd letter + last (for two-letter initials)
 
-        Returns (first_name, last_name) or None.
+        Returns (first_name, last_name) or None
 
-        :param full_name: Full name of the member to find.
-        :param found_error: (optional) Raise ValueError if not found.
+        :param full_name: Full name of the member to find
+        :param found_error: (optional) Raise ValueError if not found
 
-        :return: Membermojo name if found, else None.
+        :return: Membermojo name if found, else None
 
-        :raises ValueError: If not found and `found_error` is True.
+        :raises ValueError: If not found and `found_error` is True
         """
 
         parts = full_name.strip().split()
@@ -207,11 +247,11 @@ class Member(MojoSkel):
 
     def get_first_last_name(self, member_number: int) -> Optional[str]:
         """
-        Get full name for a given member number.
+        Get full name for a given member number
 
-        :param member_number: Member number to look up.
+        :param member_number: Member number to look up
 
-        :return: Full name as tuple, or None if not found.
+        :return: Full name as tuple, or None if not found
         """
         sql = f"""
             SELECT "first_name", "last_name"
@@ -225,11 +265,11 @@ class Member(MojoSkel):
 
     def get_name(self, member_number: int) -> Optional[str]:
         """
-        Get full name for a given member number.
+        Get full name for a given member number
 
-        :param member_number: Member number to look up.
+        :param member_number: Member number to look up
 
-        :return: Full name as "First Last", or None if not found.
+        :return: Full name as "First Last", or None if not found
         """
 
         result = self.get_first_last_name(member_number)
@@ -238,3 +278,39 @@ class Member(MojoSkel):
             first_name, last_name = result
             return f"{first_name} {last_name}"
         return None
+
+    def get_fuzz_name(self, name: str, found_error: bool = False):
+        """
+        Fuzzy search for members by name using partial matching
+        Searches across first_name and last_name fields
+
+        :param name: Free text name to search for (partial match)
+
+
+        :return: Tuple of (first_name, last_name) or None
+
+        :raises ValueError: If not found and `found_error` is True
+        """
+
+        name = name.strip().lower()
+
+        # Get all members
+        self.cursor.execute(
+            f'SELECT *, LOWER(first_name || " " || last_name) AS full FROM "{self.table_name}"'
+        )
+        rows = self.cursor.fetchall()
+
+        choices = [row["full"] for row in rows]
+        matches = get_close_matches(name, choices, n=1, cutoff=0.7)
+
+        if not matches:
+            if found_error:
+                raise ValueError(
+                    f"❌ Cannot find {name} in member database with fuzzy match."
+                )
+            return None
+
+        match = matches[0]
+        # return the sqlite row for the best match
+        row = next(r for r in rows if r["full"] == match)
+        return (row["first_name"], row["last_name"])

memberjojo/mojo_transaction.py

@@ -1,8 +1,8 @@
 """
-Module to import and interact with Membermojo completed_payments.csv data in SQLite.
+Module to import and interact with Membermojo completed_payments.csv data in SQLite
 
 Provides automatic column type inference, robust CSV importing, and
-helper methods for querying the database.
+helper methods for querying the database
 """
 
 from .mojo_common import MojoSkel
@@ -10,14 +10,15 @@ from .mojo_common import MojoSkel
 
 class Transaction(MojoSkel):
     """
-    Handles importing and querying completed payment data.
+    Handles importing and querying completed payment data
 
     Extends:
-        MojoSkel: Base class with transaction database operations.
+        MojoSkel: Base class with transaction database operations
 
-    :param payment_db_path: Path to the SQLite database.
-    :param table_name: (optional) Name of the table. Defaults to "payments".
-    :param db_key: (optional) key to unlock the encrypted sqlite database, unencrypted if unset.
+    :param payment_db_path: Path to the SQLite database
+    :param db_key: key to unlock the encrypted sqlite database,
+        unencrypted if sqlcipher3 not installed or unset
+    :param table_name: (optional) Name of the table. Defaults to "payments"
     """
 
     def __init__(
@@ -27,6 +28,6 @@ class Transaction(MojoSkel):
         table_name: str = "payments",
     ):
         """
-        Initialize the Transaction object.
+        Initialize the Transaction object
         """
         super().__init__(payment_db_path, db_key, table_name)

memberjojo/sql_query.py

@@ -0,0 +1,12 @@
+"""
+Classes for use in sqlite row matching
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Like:
+    """Marker type for SQL LIKE comparisons"""
+
+    pattern: str