monopyly 1.4.7__py3-none-any.whl → 1.5.0__py3-none-any.whl

monopyly/credit/routes.py

@@ -4,7 +4,16 @@ Routes for credit card financials.
 
 from itertools import islice
 
-from flask import flash, g, jsonify, redirect, render_template, request, url_for
+from flask import (
+    flash,
+    g,
+    jsonify,
+    redirect,
+    render_template,
+    request,
+    session,
+    url_for,
+)
 from fuisce.database import db_transaction
 from sqlalchemy.exc import MultipleResultsFound
 from werkzeug.exceptions import abort
@@ -16,7 +25,11 @@ from ..banking.banks import BankHandler
 from ..banking.transactions import BankTransactionHandler
 from ..common.forms import form_err_msg
 from ..common.forms.utils import extend_field_list_for_ajax
-from ..common.transactions import get_linked_transaction
+from ..common.transactions import (
+    categorize,
+    get_linked_transaction,
+    highlight_unmatched_transactions,
+)
 from ..common.utils import dedelimit_float, parse_date, sort_by_frequency
 from .accounts import CreditAccountHandler
 from .actions import (
@@ -24,6 +37,7 @@ from .actions import (
     get_potential_preceding_card,
     get_statement_and_transactions,
     make_payment,
+    parse_request_transaction_data,
     transfer_credit_card_statement,
 )
 from .blueprint import bp
@@ -31,6 +45,11 @@ from .cards import CreditCardHandler, save_card
 from .forms import CardStatementTransferForm, CreditCardForm, CreditTransactionForm
 from .statements import CreditStatementHandler
 from .transactions import CreditTagHandler, CreditTransactionHandler, save_transaction
+from .transactions.activity import (
+    ActivityMatchmaker,
+    TransactionActivities,
+    parse_transaction_activity_file,
+)
 
 
 @bp.route("/cards")
@@ -179,6 +198,7 @@ def update_statements_display():
 @login_required
 def load_statement_details(statement_id):
     statement, transactions = get_statement_and_transactions(statement_id)
+    categories = categorize(transactions)
     # Get bank accounts for potential payments
     bank_accounts = BankAccountHandler.get_accounts()
     return render_template(
@@ -186,6 +206,7 @@ def load_statement_details(statement_id):
         statement=statement,
         statement_transactions=transactions,
         bank_accounts=bank_accounts,
+        chart_data=categories.assemble_chart_data(exclude=["Credit payments"]),
     )
 
 
@@ -232,6 +253,68 @@ def pay_credit_card(card_id, statement_id):
     return jsonify((summary_template, transactions_table_template))
 
 
+@bp.route("/_reconcile_activity/<int:statement_id>")
+@login_required
+def reconcile_activity(statement_id):
+    return render_template(
+        "credit/statement_reconciliation/statement_reconciliation_inquiry.html",
+        statement_id=statement_id,
+    )
+
+
+@bp.route("/reconciliation/<int:statement_id>", methods=("GET", "POST"))
+@login_required
+def load_statement_reconciliation_details(statement_id):
+    if request.method == "POST":
+        activity_file = request.files.get("activity-file")
+        # Parse the data and match transactions to activities
+        if activities := parse_transaction_activity_file(activity_file):
+            session["reconciliation_info"] = (statement_id, activities.jsonify())
+    else:
+        activity_data = session.get("reconciliation_info", (None, []))[1]
+        activities = TransactionActivities(activity_data)
+    if activities:
+        statement, transactions = get_statement_and_transactions(statement_id)
+        matchmaker = ActivityMatchmaker(transactions, activities)
+        non_matches = matchmaker.unmatched_transactions
+        transactions = list(highlight_unmatched_transactions(transactions, non_matches))
+        # Calculate the amount charged/refunded during this statement timeframe
+        prior_statement = CreditStatementHandler.get_prior_statement(statement)
+        prior_statement_balance = prior_statement.balance if prior_statement else 0
+        statement_transaction_balance = statement.balance - prior_statement_balance
+        return render_template(
+            "credit/statement_reconciliation/statement_reconciliation_page.html",
+            statement=statement,
+            statement_transactions=transactions,
+            discrepant_records=matchmaker.match_discrepancies,
+            discrepant_amount=abs(statement_transaction_balance - activities.total),
+            unrecorded_activities=matchmaker.unmatched_activities,
+        )
+    else:
+        flash("ERROR")
+        return redirect(
+            url_for("credit.load_statement_details", statement_id=statement_id)
+        )
+
+
+@bp.before_app_request
+def clear_reconciliation_info():
+    exempt_endpoints = (
+        "credit.reconcile_activity",
+        "credit.load_statement_reconciliation_details",
+        "credit.expand_transaction",
+        "credit.add_transaction",
+        "credit.update_transaction",
+        "credit.infer_statement",
+        "credit.suggest_transaction_autocomplete",
+        "credit.delete_transacton",
+        "static",
+        None,
+    )
+    if request.endpoint not in exempt_endpoints:
+        session.pop("reconciliation_info", None)
+
+
 @bp.route("/transactions", defaults={"card_id": None})
 @bp.route("/transactions/<int:card_id>")
 @login_required
@@ -333,12 +416,16 @@ def add_transaction(card_id, statement_id):
             update=False,
         )
     else:
+        transaction_data = parse_request_transaction_data(request.args)
         if statement_id:
-            statement = CreditStatementHandler.get_entry(statement_id)
-            form = form.prepopulate(statement)
+            entry = CreditStatementHandler.get_entry(statement_id)
         elif card_id:
-            card = CreditCardHandler.get_entry(card_id)
-            form = form.prepopulate(card)
+            entry = CreditCardHandler.get_entry(card_id)
+        else:
+            entry = None
+        form = form.prepopulate(
+            entry, data=transaction_data, suggestion_fields=["merchant"]
+        )
     # Display the form for accepting user input
     return render_template(
         "credit/transaction_form/transaction_form_page_new.html", form=form
@@ -360,8 +447,11 @@ def update_transaction(transaction_id):
             update=True,
         )
     else:
+        transaction_data = parse_request_transaction_data(request.args)
         transaction = CreditTransactionHandler.get_entry(transaction_id)
-        form = form.prepopulate(transaction)
+        form = form.prepopulate(
+            transaction, data=transaction_data, suggestion_fields=["amount"]
+        )
     # Display the form for accepting user input
     return render_template(
         "credit/transaction_form/transaction_form_page_update.html",

monopyly/credit/transactions/_transactions.py

@@ -81,6 +81,21 @@ class CreditTransactionHandler(
         )
         return transactions
 
+    @classmethod
+    @DatabaseViewHandler.view_query
+    def get_merchants(cls):
+        """
+        Get a credit card merchants from the database.
+
+        Returns
+        -------
+        merchants : sqlalchemy.engine.ScalarResult
+            All known credit card transaction merchants from the
+            database.
+        """
+        query = cls.model.select_for_user(cls.model.merchant).distinct()
+        return cls._db.session.scalars(query)
+
     @classmethod
     def add_entry(cls, **field_values):
         """

monopyly/credit/transactions/activity/__init__.py

@@ -0,0 +1,3 @@
+from .data import TransactionActivities
+from .parser import parse_transaction_activity_file
+from .reconciliation import ActivityMatchmaker

monopyly/credit/transactions/activity/data.py

@@ -0,0 +1,161 @@
+"""Data structures for working with credit card activity data."""
+
+import datetime
+from collections import UserList, namedtuple
+from pathlib import Path
+
+from flask import current_app
+from werkzeug.utils import secure_filename
+
+
+class TransactionActivities(UserList):
+    """
+    A list-like datatype for storing transaction activity information.
+
+    A subclass of `UserList` that stores transaction activity data in an
+    easily accessible format. The object is constructed by passing a
+    normal list of ordered lists/tuples, and converts each row into an
+    equivalent `namedtuple` object, with data recorded according to
+    its column type.
+
+    Parameters
+    ----------
+    data : list
+        A list of ordered lists/tuples that contain the data to be
+        converted into this `TransactionActivities` instance.
+    """
+
+    column_types = ("transaction_date", "total", "description")
+
+    def __init__(self, data=()):
+        super().__init__([self._load_activity_from_data(*row) for row in data])
+
+    @classmethod
+    def _load_activity_from_data(cls, date, total, description):
+        activity_cls = namedtuple("TransactionActivity", cls.column_types)
+        if isinstance(date, datetime.date):
+            transaction_date = date
+        else:
+            try:
+                transaction_date = datetime.datetime.strptime(date, "%Y-%m-%d").date()
+            except ValueError:
+                raise ValueError(
+                    f"The given date '{date}' of type `{type(date)}` is not recognized. "
+                    "Dates must be native `datetime.date` objects or strings given in "
+                    "the form 'YYYY-MM-DD'."
+                )
+        return activity_cls(transaction_date, total, description)
+
+    @property
+    def total(self):
+        """The sum of the totals of each activity in the list."""
+        return sum(_.total for _ in self.data)
+
+    def jsonify(self):
+        """Return a JSON serializable representation of the activities."""
+        return [(str(_.transaction_date), _.total, _.description) for _ in self.data]
+
+
+class TransactionActivityGroup(UserList):
+    """A minimalistic class for aggregating individual transaction activities."""
+
+    def __init__(self, transaction_activities):
+        transaction_activities = list(transaction_activities)
+        self._check_grouping_validity(transaction_activities)
+        super().__init__(transaction_activities)
+
+    @property
+    def transaction_date(self):
+        return self.data[0].transaction_date
+
+    @property
+    def total(self):
+        return sum(activity.total for activity in self.data)
+
+    @property
+    def description(self):
+        return self.data[0].description
+
+    def _check_grouping_validity(self, activities):
+        self._ensure_field_commonality("transaction_date", activities)
+        self._ensure_field_commonality("description", activities)
+
+    @staticmethod
+    def _ensure_field_commonality(field, activities):
+        field_value = getattr(activities[0], field)
+        if not all(getattr(activity, field) == field_value for activity in activities):
+            raise ValueError(
+                "All transaction activities in a grouping must share the same value "
+                f"for the '{field}' field."
+            )
+
+
+class ActivityLoadingError(RuntimeError):
+    """A special exception indicating that an activity CSV failed to load."""
+
+    def __init__(self, msg="", *args, **kwargs):
+        self.msg = msg
+        super().__init__(msg, *args, **kwargs)
+
+
+class TransactionActivityLoader:
+    """
+    A tool to load transaction activity CSV files.
+
+    This is an object designed to load CSV files provided by a user. It
+    loads the file, stores it in an app-local directory, and then
+    provides access to uploaded file path.
+
+    Parameters
+    ----------
+    activity_dir : pathlib.Path
+        The path to the directory where uploaded activity files will be
+        stored. The default path is a directory named `.credit_activity`
+        within the app's instance directory.
+
+    Attributes
+    ----------
+    activity_dir : pathlib.Path
+        The path to the directory where uploaded activity files will be
+        stored. If left unset, the default path will be a directory
+        named `.credit_activity` within the app's instance directory.
+    loaded_files : list
+        A list of paths to loaded files; this list is empty before
+        adding any files and after cleaning up any previously loaded
+        files.
+    """
+
+    def __init__(self, activity_dir=None):
+        # Create and use a directory to store uploaded activity files
+        default_activity_dir = Path(current_app.instance_path) / ".credit_activity"
+        self.activity_dir = Path(activity_dir) if activity_dir else default_activity_dir
+        self.activity_dir.mkdir(exist_ok=True)
+        self.loaded_files = []
+
+    def upload(self, activity_file):
+        """
+        Upload a CSV file containing credit transaction activity.
+
+        Parameters
+        ----------
+        activity_file : werkzeug.datastructures.FileStorage
+            The file object to be loaded.
+
+        Returns
+        -------
+        activity_filepath : pathlib.Path
+            The path to the uploaded activity file.
+        """
+        activity_filename = secure_filename(activity_file.filename)
+        if not activity_filename:
+            raise ActivityLoadingError("No activity file was specified.")
+        activity_filepath = self.activity_dir / activity_filename
+        activity_file.save(activity_filepath)
+        self.loaded_files.append(activity_filepath)
+        return activity_filepath
+
+    def cleanup(self):
+        """Clean all loaded files from the activity directory."""
+        while self.loaded_files:
+            activity_filepath = self.loaded_files.pop()
+            activity_filepath.unlink()

monopyly/credit/transactions/activity/parser.py

@@ -0,0 +1,274 @@
+"""Define a parser and associated functionality for reading activity data CSV files."""
+
+import csv
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from flask import current_app
+from werkzeug.utils import secure_filename
+
+from ....common.utils import parse_date
+from .data import ActivityLoadingError, TransactionActivities, TransactionActivityLoader
+
+
+def parse_transaction_activity_file(transaction_file):
+    """
+    Parse a CSV file containing reported transaction activity.
+
+    Parameters
+    ----------
+    activity_file : werkzeug.datastructures.FileStorage, pathlib.Path
+        The file object containing transaction activity data or a path
+        to a file containing transaction activity data.
+
+    Returns
+    -------
+    activities : TransactionActivities
+        The list-like object containing credit transaction activity
+        data.
+    """
+    try:
+        return _TransactionActivityParser(transaction_file).data
+    except ActivityLoadingError:
+        return None
+
+
+class _ColumnIdentifier(ABC):
+    """
+    An object to aid in identifying the column matching a certain type.
+
+    Parameters
+    ----------
+    raw_header : list
+        The header row as a list of column titles.
+    """
+
+    def __init__(self, raw_header):
+        self._raw_header = raw_header
+        self._potential_column_indices = []
+
+    def determine_index(self):
+        """
+        Given the data header, determine the current column type index.
+
+        Returns
+        -------
+        index : int, None
+            The index of the column corresponding to the current column
+            type.
+        """
+        for column_index, column_title in enumerate(self._raw_header):
+            standardized_title = column_title.strip().casefold()
+            column_match = self.check(standardized_title)
+            if column_match is True:
+                return column_index
+            elif column_match is None:
+                self._potential_column_indices.append(column_index)
+        return self._infer_column_index()
+
+    @abstractmethod
+    def check(self, standardized_title):
+        raise NotImplementedError(
+            "Define how a column identification check is performed in a subclass. "
+            "A check should return `True` if the title does describe the class's "
+            "column type, `False` if the title does *not* describe the class's column "
+            "type, and `None` if the title may describe the column type but more "
+            "information is required."
+        )
+
+    def _infer_column_index(self):
+        # Attempt to infer the current column type index from a potential match
+        if len(self._potential_column_indices) == 1:
+            return self._potential_column_indices[0]
+        return None
+
+
+class _TransactionDateColumnIdentifier(_ColumnIdentifier):
+    def check(self, standardized_title):
+        """Check if the title indicates this column contains a transaction date."""
+        if standardized_title == "transaction date":
+            return True
+        elif "date" in standardized_title.split():
+            if "trans." in standardized_title:
+                return True
+            elif standardized_title == "date":
+                return None
+        return False
+
+
+class _TransactionTotalColumnIdentifier(_ColumnIdentifier):
+    def check(self, standardized_title):
+        """Check if the title indicates this column contains an amount."""
+        return standardized_title in ("amount", "total")
+
+
+class _TransactionDescriptionColumnIdentifier(_ColumnIdentifier):
+    def check(self, standardized_title):
+        """Check if the title indicates this column contains a description."""
+        return standardized_title in ("description", "desc.")
+
+
+class _TransactionCategoryColumnIdentifier(_ColumnIdentifier):
+    def check(self, standardized_title):
+        """Check if the title indicates this column contains a category."""
+        return standardized_title == "category"
+
+
+class _TransactionTypeColumnIdentifier(_ColumnIdentifier):
+    def check(self, standardized_title):
+        """Check if the title indicates this column contains a transaction type."""
+        return standardized_title == "type"
+
+
+class _TransactionActivityParser:
+    """
+    A parser for arbitrary CSV files containing transaction activity.
+
+    This object contains logic and utilities for parsing aribtrary CSV
+    files containing transaction activity data. It is generalized to
+    work with various CSV labeling schemes, inferring information about
+    the CSV data from the contents of the file.
+
+    Parameters
+    ----------
+    activity_file : werkzeug.datastructures.FileStorage, pathlib.Path
+        The file object containing transaction activity data or a path
+        to a file containing transaction activity data.
+    activity_dir : pathlib.Path
+        The path to the directory where activity files to be parsed will
+        be stored after uploading.
+    """
+
+    _column_identifiers = {
+        "transaction_date": _TransactionDateColumnIdentifier,
+        "total": _TransactionTotalColumnIdentifier,
+        "description": _TransactionDescriptionColumnIdentifier,
+        "category": _TransactionCategoryColumnIdentifier,
+        "type": _TransactionTypeColumnIdentifier,
+    }
+    _raw_column_types = list(_column_identifiers.keys())
+    column_types = _raw_column_types[:3]
+
+    def __init__(self, activity_file, activity_dir=None):
+        file_loader = TransactionActivityLoader(activity_dir=activity_dir)
+        if isinstance(activity_file, Path):
+            activity_filepath = activity_file
+        else:
+            activity_filepath = file_loader.upload(activity_file)
+        # Load data from the activity file
+        raw_header, raw_data = self._load_data(activity_filepath)
+        if not raw_data:
+            raise ActivityLoadingError("The activity file contains no actionable data.")
+        # Parse the loaded activity data
+        self._raw_column_indices = self._determine_column_indices(raw_header)
+        self._negative_charges = self._determine_expenditure_sign(raw_data)
+        self.column_indices = {name: i for i, name in enumerate(self.column_types)}
+        self.data = TransactionActivities(self._process_data(row) for row in raw_data)
+        # Remove the loaded activity file
+        file_loader.cleanup()
+
+    @staticmethod
+    def _load_data(transaction_filepath):
+        # Load raw header information and data from the transaction file
+        with transaction_filepath.open() as csv_file:
+            csv_reader = csv.reader(csv_file)
+            raw_header = next(csv_reader)
+            raw_data = list(csv_reader)
+        return raw_header, raw_data
+
+    def _determine_column_indices(self, raw_header):
+        # Determine the indices of various columns in the raw header/data
+        raw_column_indices = {
+            column_type: identifier(raw_header).determine_index()
+            for column_type, identifier in self._column_identifiers.items()
+        }
+        for column_type in self.column_types:
+            if raw_column_indices[column_type] is None:
+                raise RuntimeError(
+                    f"The '{column_type}' column could not be identified in the data."
+                )
+        return raw_column_indices
+
+    def _determine_expenditure_sign(self, raw_data):
+        # Determine the sign of expenditures
+        # - Charges may be reported as either positive or negative amounts
+        # - Negatively valued charges (positively valued payments) return `True`;
+        #   positively valued charges (negatively valued payments) return `False`
+        # Note: This method assumes that an activity file will not report a standard
+        #       transaction as a "payment"
+        contextual_column_types = ("category", "description", "type")
+        contextual_column_indices = [
+            index
+            for column_type, index in self._raw_column_indices.items()
+            if column_type in contextual_column_types and index is not None
+        ]
+
+        def _infer_payment_row(row):
+            # Infer whether the row constitutes a payment transaction
+            contextual_info = [row[i].lower() for i in contextual_column_indices]
+            return any("payment" in element for element in contextual_info)
+
+        payment_rows = list(filter(_infer_payment_row, raw_data))
+        return self._extrapolate_payments_positive(payment_rows, raw_data)
+
+    def _extrapolate_payments_positive(self, payment_rows, raw_data):
+        if payment_rows:
+            payments_positive = self._evaluate_payment_signs(payment_rows)
+        else:
+            payments_positive = self._evaluate_nonpayment_signs(raw_data)
+        if payments_positive is None:
+            raise RuntimeError(
+                "The sign of credits/debits could not be determined from the data."
+            )
+        return payments_positive
+
+    def _evaluate_payment_signs(self, payment_rows):
+        # Iterate over each payment row and collect sign information
+        amount_index = self._raw_column_indices["total"]
+        payment_signs_positive = [float(row[amount_index]) > 0 for row in payment_rows]
+        # Evaluate whether payment amounts are treated as positively valued
+        if all(payment_signs_positive):
+            payments_positive = True
+        elif not any(payment_signs_positive):
+            payments_positive = False
+        else:
+            payments_positive = None
+        return payments_positive
+
+    def _evaluate_nonpayment_signs(self, nonpayment_rows):
+        # Iterate over each non-payment and collect sign information
+        # - Assume the majority of transactions are charges if no payments are found
+        amount_index = self._raw_column_indices["total"]
+        negative_charge_count = sum(
+            float(row[amount_index]) < 0 for row in nonpayment_rows
+        )
+        # Evaluate whether assumed non-payment amounts seem positively valued
+        negative_charge_frac = negative_charge_count / len(nonpayment_rows)
+        if negative_charge_frac == 0.5:
+            payments_positive = None
+        else:
+            # Payments are assumed to be positive if most of the transactions are
+            # charges and most of the charges are negative
+            payments_positive = negative_charge_frac > 0.5
+        return payments_positive
+
+    def _process_data(self, row):
+        # Process the raw data into an output format
+        processed_row = []
+        for column_type in self.column_types:
+            value = row[self._raw_column_indices[column_type]]
+            if column_type == "transaction_date":
+                value = self._process_date_data(value)
+            elif column_type == "total":
+                value = self._process_amount_data(value)
+            processed_row.append(value)
+        return processed_row
+
+    def _process_amount_data(self, value):
+        # Charges should be positively valued, payments negatively valued
+        value = float(value)
+        return -value if self._negative_charges else value
+
+    def _process_date_data(self, value):
+        # Output dates as `datetime.date` objects
+        return parse_date(value)