memberjojo 2.2__py3-none-any.whl → 3.0__py3-none-any.whl

memberjojo/mojo_loader.py

@@ -1,22 +1,25 @@
 #!/usr/bin/env python3
 """
-Helper module for importing a CSV into a SQLite database.
+Helper module for importing a CSV into a SQLite database
 """
 
 from collections import defaultdict, Counter
 from csv import DictReader
 from dataclasses import dataclass
+from io import StringIO
 from pathlib import Path
-from typing import Any, Tuple
+from typing import Any, IO, Tuple
 
 import re
 import sqlite3 as sqlite3_builtin
 
+import requests
+
 
 @dataclass(frozen=True)
 class DiffRow:
     """
-    Represents a single diff result.
+    Represents a single diff result
 
     - diff_type: 'added' | 'deleted' | 'changed'
     - preview: tuple of values, with preview[0] == key
@@ -33,11 +36,11 @@ class DiffRow:
 
 def _normalize(name: str) -> str:
     """
-    Normalize a column name: lowercase, remove symbols, convert to snake case.
+    Normalize a column name: lowercase, remove symbols, convert to snake case
 
-    :param name: Raw name to normalize.
+    :param name: Raw name to normalize
 
-    :return: Normalized lowercase string in snake case with no symbols.
+    :return: Normalized lowercase string in snake case with no symbols
     """
     name = name.strip().lower()
     name = re.sub(r"[^a-z0-9]+", "_", name)
@@ -46,7 +49,7 @@ def _normalize(name: str) -> str:
 
 def _guess_type(value: any) -> str:
     """
-    Guess SQLite data type of a CSV value: 'INTEGER', 'REAL', or 'TEXT'.
+    Guess SQLite data type of a CSV value: 'INTEGER', 'REAL', or 'TEXT'
 
     :param value: entry from sqlite database to guess the type of
 
@@ -71,8 +74,8 @@ def _guess_type(value: any) -> str:
 
 def infer_columns_from_rows(rows: list[dict]) -> dict[str, str]:
     """
-    Infer column types from CSV rows.
-    Returns mapping: normalized column name -> SQLite type.
+    Infer column types from CSV rows
+    Returns mapping: normalized column name -> SQLite type
 
     :param rows: list of rows to use for inference
 
@@ -104,75 +107,100 @@ def infer_columns_from_rows(rows: list[dict]) -> dict[str, str]:
 
 def _create_table_from_columns(table_name: str, columns: dict[str, str]) -> str:
     """
-    Generate CREATE TABLE SQL from column type mapping.
+    Generate CREATE TABLE SQL from column type mapping
+    Adds an auto-incrementing rowid as the primary key
 
-    :param table_name: Table to use when creating columns.
-    :param columns: dict of columns to create.
+    :param table_name: Table to use when creating columns
+    :param columns: dict of columns to create
 
-    :return: SQL commands to create the table.
+    :return: SQL commands to create the table
     """
-    col_defs = []
-    first = True
+    col_defs = ["rowid INTEGER PRIMARY KEY AUTOINCREMENT"]
 
     for col, col_type in columns.items():
-        if first:
-            col_defs.append(f'"{col}" {col_type} PRIMARY KEY')
-            first = False
-        else:
-            col_defs.append(f'"{col}" {col_type}')
+        col_defs.append(f'"{col}" {col_type}')
 
     return (
         f'CREATE TABLE IF NOT EXISTS "{table_name}" (\n' + ",\n".join(col_defs) + "\n)"
     )
 
 
+def table_exists(cursor, table_name: str) -> bool:
+    """
+    Return True or False if a table exists
+
+    :param cursor: SQLite cursor of db to find table in
+    :param table_name: name of the table to check existance of
+
+    :return: bool of existence
+    """
+    cursor.execute(
+        "SELECT 1 FROM sqlite_master WHERE type='table' AND name=? LIMIT 1;",
+        (table_name,),
+    )
+    return cursor.fetchone() is not None
+
+
 # -----------------------
 # CSV Import
 # -----------------------
 
 
-def import_csv_helper(conn, table_name: str, csv_path: Path):
+def import_data(conn, table_name: str, reader: DictReader, merge: bool = False):
     """
-    Import CSV into database using given cursor.
-    Column types inferred automatically.
+    Import data in the DictReader into the SQLite3 database at conn
 
-    :param conn: SQLite database connection to use.
-    :param table_name: Table to import the CSV into.
-    :param csv_path: Path like path of the CSV file to import.
+    :param conn: SQLite database connection to use
+    :param table_name: Name of the table to import into
+    :param reader: A Dictreader object to import from
+    :param merge: (optional) If True, merge into existing table. Defaults to False.
     """
-    if not csv_path.exists():
-        raise FileNotFoundError(f"CSV file not found: {csv_path}")
 
-    # Read CSV rows
-    with csv_path.open(newline="", encoding="utf-8") as f:
-        reader = list(DictReader(f))
-        if not reader:
-            raise ValueError("CSV file is empty.")
-        inferred_cols = infer_columns_from_rows(reader)
-
-        cursor = conn.cursor()
+    cursor = conn.cursor()
+    if not merge or not table_exists(cursor, table_name):
         # Drop existing table
         cursor.execute(f'DROP TABLE IF EXISTS "{table_name}";')
-
+        inferred_cols = infer_columns_from_rows(reader)
         # Create table
         create_sql = _create_table_from_columns(table_name, inferred_cols)
         cursor.execute(create_sql)
 
-        # Insert rows
-        cols = list(reader[0].keys())
-        norm_map = {c: _normalize(c) for c in cols}
-        colnames = ",".join(f'"{norm_map[c]}"' for c in cols)
-        placeholders = ",".join("?" for _ in cols)
-        insert_sql = f'INSERT INTO "{table_name}" ({colnames}) VALUES ({placeholders})'
+    # Insert rows
+    cols = list(reader[0].keys())
+    norm_map = {c: _normalize(c) for c in cols}
+    colnames = ",".join(f'"{norm_map[c]}"' for c in cols)
+    placeholders = ",".join("?" for _ in cols)
+    insert_sql = f'INSERT INTO "{table_name}" ({colnames}) VALUES ({placeholders})'
 
-        for row in reader:
-            values = [row[c] if row[c] != "" else None for c in cols]
-            cursor.execute(insert_sql, values)
+    for row in reader:
+        values = [row[c] if row[c] != "" else None for c in cols]
+        cursor.execute(insert_sql, values)
 
     cursor.close()
     conn.commit()
 
 
+def import_csv_helper(conn, table_name: str, csv_path: Path, merge: bool = False):
+    """
+    Import CSV into database using given cursor
+    Column types inferred automatically
+
+    :param conn: SQLite database connection to use
+    :param table_name: Table to import the CSV into
+    :param csv_path: Path like path of the CSV file to import
+    :param merge: (optional) If True, merge into existing table. Defaults to False.
+    """
+    if not csv_path.exists():
+        raise FileNotFoundError(f"CSV file not found: {csv_path}")
+
+    # Read CSV rows
+    with csv_path.open(newline="", encoding="utf-8") as f:
+        reader = list(DictReader(f))
+        if not reader:
+            raise ValueError("CSV file is empty.")
+        import_data(conn, table_name, reader, merge=merge)
+
+
 # -----------------------
 # diff generation
 # -----------------------
@@ -180,7 +208,7 @@ def import_csv_helper(conn, table_name: str, csv_path: Path):
 
 def _diffrow_from_sql_row(row: sqlite3_builtin.Row) -> DiffRow:
     """
-    Convert a sqlite3.Row from generate_sql_diff into DiffRow.
+    Convert a sqlite3.Row from generate_sql_diff into DiffRow
     Row shape:
         (diff_type, col1, col2, col3, ...)
 
@@ -195,7 +223,7 @@ def _diffrow_from_sql_row(row: sqlite3_builtin.Row) -> DiffRow:
 
 
 def diff_cipher_tables(
-    cipher_conn,
+    conn,
     *,
     new_table: str,
     old_table: str,
@@ -204,7 +232,7 @@ def diff_cipher_tables(
     Copy old and new tables from SQLCipher into a single
     in-memory sqlite3 database and diff them there.
 
-    :param cipher_conn: sqlite connection to the encrypted db
+    :param conn: sqlite connection to the db
     :param new_table: name of the new table for comparison
     :param old_table: name of the old table for comparison
 
@@ -216,8 +244,8 @@ def diff_cipher_tables(
 
     try:
         for table in (old_table, new_table):
-            # 1. Clone schema using SQLite itself
-            schema_sql = cipher_conn.execute(
+            # Clone schema using SQLite itself
+            schema_sql = conn.execute(
                 """
                 SELECT sql
                 FROM sqlite_master
@@ -232,7 +260,7 @@ def diff_cipher_tables(
             plain.execute(schema_sql[0])
 
             # 2. Copy data
-            rows = cipher_conn.execute(f"SELECT * FROM {table}")
+            rows = conn.execute(f"SELECT * FROM {table}")
             cols = [d[0] for d in rows.description]
 
             col_list = ", ".join(cols)
@@ -263,9 +291,9 @@ def _generate_sql_diff(
     old_table: str,
 ) -> list[sqlite3_builtin.Row]:
     """
-    Generate a diff between two tables using standard SQLite features.
+    Generate a diff between two tables using standard SQLite features
 
-    - The FIRST column is the primary key.
+    - Uses rowid as the primary key for joining
     - Returned row shape:
             (diff_type, preview_col1, preview_col2, preview_col3, ...)
 
@@ -276,7 +304,7 @@ def _generate_sql_diff(
     :return: list of sqlite rows that are changed
     """
 
-    # 1. Introspect schema (order-preserving)
+    # Introspect schema (order-preserving)
     cols_info = conn.execute(f"PRAGMA table_info({new_table})").fetchall()
 
     if not cols_info:
@@ -284,16 +312,20 @@ def _generate_sql_diff(
 
     cols = [row[1] for row in cols_info]
 
-    key = cols[0]
-    non_key_cols = cols[1:]
+    # Exclude rowid from consideration for key or data
+    main_cols = [c for c in cols if c != "rowid"]
 
-    # 2. Preview columns (key first, limit for readability)
+    # First column is key, others are for comparison
+    key = main_cols[0]
+    non_key_cols = main_cols[1:]
+
+    #  Preview columns (key first, then others for readability)
     preview_cols = [key] + non_key_cols[:5]
 
     new_preview = ", ".join(f"n.{c}" for c in preview_cols)
     old_preview = ", ".join(f"o.{c}" for c in preview_cols)
 
-    # 3. Row-value comparison (NULL-safe)
+    # Row-value comparison (NULL-safe)
     if non_key_cols:
         changed_predicate = (
             f"({', '.join(f'n.{c}' for c in non_key_cols)}) "
@@ -333,3 +365,39 @@ def _generate_sql_diff(
         """
 
     return list(conn.execute(sql))
+
+
+def download_csv_helper(
+    conn, table_name: str, url: str, session: requests.Session, merge: bool = False
+) -> IO[str]:
+    """
+    Download url into a StringIO file object using streaming
+    and import into database
+
+    :param conn: The SQLite3 database connection to use
+    :param table_name: The name of the table to import it into
+    :param url: URL of the csv to download
+    :param session: A requests session to use for the download
+    :param merge: (optional) If True, merge into existing table. Defaults to False.
+    """
+
+    print(f"☁️ Downloading from: {url}")
+
+    # Enable streaming
+    with session.get(url, stream=True) as resp:
+        resp.raise_for_status()
+
+        # Initialize the string buffer
+        string_buffer = StringIO()
+
+        # Stream decoded text
+        # decode_unicode=True uses the encoding from the response headers
+        for chunk in resp.iter_content(chunk_size=8192, decode_unicode=True):
+            if chunk:
+                string_buffer.write(chunk)
+
+        # Reset pointer to the beginning for DictReader
+        string_buffer.seek(0)
+
+        print(f"✅ Downloaded with encoding {resp.encoding}.")
+        import_data(conn, table_name, list(DictReader(string_buffer)), merge=merge)

memberjojo/mojo_member.py

@@ -1,25 +1,28 @@
 """
-Member module for creating and interacting with a SQLite database.
+Member module for creating and interacting with a SQLite database
 
 This module loads data from a `members.csv` file downloaded from Membermojo,
-stores it in SQLite, and provides helper functions for member lookups.
+stores it in SQLite, and provides helper functions for member lookups
 """
 
+from difflib import get_close_matches
 from pathlib import Path
+from pprint import pprint
 from typing import Optional
 from .mojo_common import MojoSkel
 
 
 class Member(MojoSkel):
     """
-    Subclass of MojoSkel providing member-specific database functions.
+    Subclass of MojoSkel providing member-specific database functions
 
     This class connects to a SQLite database and supports importing member data
-    from CSV and performing queries like lookup by name or member number.
+    from CSV and performing queries like lookup by name or member number
 
-    :param member_db_path (Path): Path to the SQLite database file.
-    :param table_name (str): (optional) Table name to use. Defaults to "members".
-    :param db_key: (optional) key to unlock the encrypted sqlite database, unencrypted if unset.
+    :param member_db_path (Path): Path to the SQLite database file
+    :param db_key: key to unlock the encrypted sqlite database,
+        unencrypted if sqlcipher3 not installed or unset
+    :param table_name (str): (optional) Table name to use. Defaults to "members"
     """
 
     def __init__(
@@ -29,23 +32,63 @@ class Member(MojoSkel):
         table_name: str = "members",
     ):
         """
-        Initialize the Member database handler.
+        Initialize the Member database handler
         """
         super().__init__(member_db_path, db_key, table_name)
 
+    def get_bool(self, entry_name: str, member_number: int) -> bool:
+        """
+        Return a bool for a member entry that is a tick box on membermojo
+
+        :param entry_name: The entry name to return as a bool
+        :param member_number: The member number to check value of entry_name
+
+        :return: True is entry is yes otherwise False
+
+        :raises ValueError: If entry name not found
+        """
+        sql = f"""
+            SELECT "{entry_name}"
+            FROM "{self.table_name}"
+            WHERE "member_number" = ?
+        """
+        self.cursor.execute(sql, (member_number,))
+
+        row = self.cursor.fetchone()
+        if row is None:
+            raise ValueError(
+                f"❌ Cannot find: {entry_name} for member {member_number}."
+            )
+
+        value = row[0]
+        return str(value).lower() == "yes"
+
+    def member_type_count(self, membership_type: str):
+        """
+        Count members by membership type string
+
+        :param membership_type: the string to match, can use percent to match
+            remaining or preceeding text
+                - Full (match only Full)
+                - Full% (match Full and any words after)
+                - %Full% ( match Full in the middle)
+        """
+        query = "WHERE membership LIKE ?"
+        return self.run_count_query(query, (f"{membership_type}",))
+
     def get_number_first_last(
         self, first_name: str, last_name: str, found_error: bool = False
     ) -> Optional[int]:
         """
-        Find a member number based on first and last name (case-insensitive).
+        Find a member number based on first and last name (case-insensitive)
 
-        :param first_name: First name of the member.
-        :param last_name: Last name of the member.
-        :param found_error: (optional): If True, raises ValueError if not found.
+        :param first_name: First name of the member
+        :param last_name: Last name of the member
+        :param found_error: (optional): If True, raises ValueError if not found
 
-        :return: The member number if found, otherwise None.
+        :return: The member number if found, otherwise None
 
-        :raises ValueError: If not found and `found_error` is True.
+        :raises ValueError: If not found and `found_error` is True
         """
         sql = f"""
             SELECT "member_number"
@@ -54,6 +97,12 @@ class Member(MojoSkel):
         """
         self.cursor.execute(sql, (first_name, last_name))
         result = self.cursor.fetchone()
+        if self.debug:
+            print("Sql:")
+            pprint(sql)
+            print(f"First Name: {first_name} Last Name: {last_name}")
+            print("Result:")
+            pprint(result[0])
 
         if not result and found_error:
             raise ValueError(
@@ -64,18 +113,22 @@ class Member(MojoSkel):
 
     def get_number(self, full_name: str, found_error: bool = False) -> Optional[int]:
         """
-        Find a member number by passed full_name.
+        Find a member number by passed full_name
         Tries first and last, and then middle last if 3 words,
-        Then initial of first name if initials passed.
+        Then initial of first name if initials passed
+        Finnaly a fuzzy lookup is tried
 
-        :param full_name: Full name of the member.
-        :param found_error: (optional) Raise ValueError if not found.
+        :param full_name: Full name of the member
+        :param found_error: (optional) Raise ValueError if not found
 
-        :return: Member number if found, else None.
+        :return: Member number if found, else None
 
-        :raises ValueError: If not found and `found_error` is True.
+        :raises ValueError: If not found and `found_error` is True
         """
-        result = self.get_mojo_name(full_name, found_error)
+        result = self.get_mojo_name(full_name)
+        if not result:
+            result = self.get_fuzz_name(full_name, found_error)
+
         if result:
             return self.get_number_first_last(result[0], result[1])
         return None
@@ -123,7 +176,7 @@ class Member(MojoSkel):
         self, full_name: str, found_error: bool = False
     ) -> Optional[tuple]:
         """
-        Resolve a member name from a free-text full name.
+        Resolve a member name from a free-text full name
 
         **Search order**
 
@@ -132,14 +185,14 @@ class Member(MojoSkel):
         3. initial 1st letter + last
         4. initial 2nd letter + last (for two-letter initials)
 
-        Returns (first_name, last_name) or None.
+        Returns (first_name, last_name) or None
 
-        :param full_name: Full name of the member to find.
-        :param found_error: (optional) Raise ValueError if not found.
+        :param full_name: Full name of the member to find
+        :param found_error: (optional) Raise ValueError if not found
 
-        :return: Membermojo name if found, else None.
+        :return: Membermojo name if found, else None
 
-        :raises ValueError: If not found and `found_error` is True.
+        :raises ValueError: If not found and `found_error` is True
         """
 
         parts = full_name.strip().split()
@@ -207,11 +260,11 @@ class Member(MojoSkel):
 
     def get_first_last_name(self, member_number: int) -> Optional[str]:
         """
-        Get full name for a given member number.
+        Get full name for a given member number
 
-        :param member_number: Member number to look up.
+        :param member_number: Member number to look up
 
-        :return: Full name as tuple, or None if not found.
+        :return: Full name as tuple, or None if not found
         """
         sql = f"""
             SELECT "first_name", "last_name"
@@ -225,11 +278,11 @@ class Member(MojoSkel):
 
     def get_name(self, member_number: int) -> Optional[str]:
         """
-        Get full name for a given member number.
+        Get full name for a given member number
 
-        :param member_number: Member number to look up.
+        :param member_number: Member number to look up
 
-        :return: Full name as "First Last", or None if not found.
+        :return: Full name as "First Last", or None if not found
         """
 
         result = self.get_first_last_name(member_number)
@@ -238,3 +291,39 @@ class Member(MojoSkel):
             first_name, last_name = result
             return f"{first_name} {last_name}"
         return None
+
+    def get_fuzz_name(self, name: str, found_error: bool = False):
+        """
+        Fuzzy search for members by name using partial matching
+        Searches across first_name and last_name fields
+
+        :param name: Free text name to search for (partial match)
+
+
+        :return: Tuple of (first_name, last_name) or None
+
+        :raises ValueError: If not found and `found_error` is True
+        """
+
+        name = name.strip().lower()
+
+        # Get all members
+        self.cursor.execute(
+            f'SELECT *, LOWER(first_name || " " || last_name) AS full FROM "{self.table_name}"'
+        )
+        rows = self.cursor.fetchall()
+
+        choices = [row["full"] for row in rows]
+        matches = get_close_matches(name, choices, n=1, cutoff=0.7)
+
+        if not matches:
+            if found_error:
+                raise ValueError(
+                    f"❌ Cannot find {name} in member database with fuzzy match."
+                )
+            return None
+
+        match = matches[0]
+        # return the sqlite row for the best match
+        row = next(r for r in rows if r["full"] == match)
+        return (row["first_name"], row["last_name"])

memberjojo/mojo_transaction.py

@@ -1,8 +1,8 @@
 """
-Module to import and interact with Membermojo completed_payments.csv data in SQLite.
+Module to import and interact with Membermojo completed_payments.csv data in SQLite
 
 Provides automatic column type inference, robust CSV importing, and
-helper methods for querying the database.
+helper methods for querying the database
 """
 
 from .mojo_common import MojoSkel
@@ -10,14 +10,15 @@ from .mojo_common import MojoSkel
 
 class Transaction(MojoSkel):
     """
-    Handles importing and querying completed payment data.
+    Handles importing and querying completed payment data
 
     Extends:
-        MojoSkel: Base class with transaction database operations.
+        MojoSkel: Base class with transaction database operations
 
-    :param payment_db_path: Path to the SQLite database.
-    :param table_name: (optional) Name of the table. Defaults to "payments".
-    :param db_key: (optional) key to unlock the encrypted sqlite database, unencrypted if unset.
+    :param payment_db_path: Path to the SQLite database
+    :param db_key: key to unlock the encrypted sqlite database,
+        unencrypted if sqlcipher3 not installed or unset
+    :param table_name: (optional) Name of the table. Defaults to "payments"
     """
 
     def __init__(
@@ -27,6 +28,6 @@ class Transaction(MojoSkel):
         table_name: str = "payments",
     ):
         """
-        Initialize the Transaction object.
+        Initialize the Transaction object
         """
         super().__init__(payment_db_path, db_key, table_name)

memberjojo/sql_query.py

@@ -0,0 +1,12 @@
+"""
+Classes for use in sqlite row matching
+"""
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class Like:
+    """Marker type for SQL LIKE comparisons"""
+
+    pattern: str