tariochbctools 0.38.1__tar.gz → 1.0.0__tar.gz

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/.pre-commit-config.yaml

@@ -52,4 +52,5 @@ repos:
   rev: v1.10.1
   hooks:
   -   id: mypy
-      args: [--install-types, --non-interactive, --ignore-missing-imports]
+      args: [--install-types, --non-interactive, --ignore-missing-imports, --disallow-incomplete-defs]
+#      args: [--install-types, --non-interactive, --ignore-missing-imports, --disallow-untyped-defs]

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tariochbctools
-Version: 0.38.1
+Version: 1.0.0
 Summary: Importers, plugins and price fetchers for Beancount
 Home-page: https://github.com/tarioch/beancounttools/
 Author: Patrick Ruckstuhl
@@ -20,7 +20,9 @@ Classifier: License :: OSI Approved :: MIT License
 Description-Content-Type: text/x-rst; charset=UTF-8
 License-File: LICENSE.txt
 Requires-Dist: importlib-metadata; python_version < "3.8"
-Requires-Dist: beancount<3,>=2
+Requires-Dist: beancount>=3
+Requires-Dist: beangulp
+Requires-Dist: beanprice
 Requires-Dist: bitstampclient
 Requires-Dist: mt-940
 Requires-Dist: pyyaml
@@ -31,6 +33,7 @@ Requires-Dist: opencv-python
 Requires-Dist: blockcypher
 Requires-Dist: imap-tools
 Requires-Dist: undictify
+Requires-Dist: rsa
 Provides-Extra: testing
 Requires-Dist: pytest; extra == "testing"
 Requires-Dist: pytest-cov; extra == "testing"

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/docs/importers.rst

@@ -170,12 +170,12 @@ TrueLayer account IDs to beancount accounts. e.g.:
 If it is present, transactions for *only these accounts* will be imported.
 
 
-Nordigen
---------
+GoCardless (formerly Nordigen)
+------------------------------
 
-Import from `Nordigen <http://nordigen.com/>`__ using their api services. e.g. supports Revolut.
+Import from `GoCardless <https://gocardless.com/bank-account-data/>`__ (formerly known as `Nordigen <https://gocardless.com/en-us/g/gc-nordigen/>`__) using their api services. Supports `2500+ banks <https://gocardless.com/bank-account-data/coverage/>`__ in Europe and the UK including e.g. Revolut.
 You need to create a free account and create a token. I've included a small cli to allow to hook up
-to different banks with nordigen. If you're country is not supported you can play around with other countries
+to different banks with GoCardless. If you're country is not supported you can play around with other countries
 e.g. CH is not allowed but things like revolut still work. You can also create multiple links and they will
 all be listed in the end.
 

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/setup.cfg

@@ -29,7 +29,9 @@ package_dir =
 	=src
 install_requires = 
 	importlib-metadata; python_version<"3.8"
-	beancount>=2,<3
+	beancount>=3
+	beangulp
+	beanprice
 	bitstampclient
 	mt-940
 	pyyaml
@@ -40,6 +42,7 @@ install_requires =
 	blockcypher
 	imap-tools
 	undictify
+	rsa
 
 [options.packages.find]
 where = src

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/src/tariochbctools/importers/bcge/importer.py

@@ -1,14 +1,15 @@
 import re
+from typing import Any
 
 from tariochbctools.importers.general import mt940importer
 
 
-def strip_newline(string):
+def strip_newline(string: str) -> str:
     return string.replace("\n", "").replace("\r", "")
 
 
 class BCGEImporter(mt940importer.Importer):
-    def prepare_payee(self, trxdata):
+    def prepare_payee(self, trxdata: dict[str, Any]) -> str:
         transaction_details = strip_newline(trxdata["transaction_details"])
         payee = re.search(r"ORDP/([^/]+)", transaction_details)
         if payee is None:
@@ -16,7 +17,7 @@ class BCGEImporter(mt940importer.Importer):
         else:
             return payee.group(1)
 
-    def prepare_narration(self, trxdata):
+    def prepare_narration(self, trxdata: dict[str, Any]) -> str:
         transaction_details = strip_newline(trxdata["transaction_details"])
         extra_details = strip_newline(trxdata["extra_details"])
         beneficiary = re.search(r"/BENM/([^/]+)", transaction_details)

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/src/tariochbctools/importers/bitst/importer.py

@@ -1,36 +1,38 @@
 from datetime import date
 from os import path
+from typing import Any
 
+import beangulp
 import bitstamp.client
 import yaml
 from beancount.core import amount, data
 from beancount.core.number import MISSING, D
-from beancount.ingest import importer
 from dateutil.parser import parse
 from dateutil.relativedelta import relativedelta
 
 from tariochbctools.importers.general.priceLookup import PriceLookup
 
 
-class Importer(importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for Bitstamp."""
 
-    def identify(self, file):
-        return path.basename(file.name).endswith("bitstamp.yaml")
+    def identify(self, filepath: str) -> bool:
+        return path.basename(filepath).endswith("bitstamp.yaml")
 
-    def file_account(self, file):
+    def account(self, filepath: str) -> data.Account:
         return ""
 
-    def extract(self, file, existing_entries):
-        self.priceLookup = PriceLookup(existing_entries, "CHF")
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        self.priceLookup = PriceLookup(existing, "CHF")
 
-        config = yaml.safe_load(file.contents())
+        with open(filepath) as file:
+            config = yaml.safe_load(file)
         self.config = config
         self.client = bitstamp.client.Trading(
             username=config["username"], key=config["key"], secret=config["secret"]
         )
         self.currencies = config["currencies"]
-        self.account = config["account"]
+        self._account = config["account"]
         self.otherExpensesAccount = config["otherExpensesAccount"]
         self.capGainAccount = config["capGainAccount"]
 
@@ -46,7 +48,7 @@ class Importer(importer.ImporterProtocol):
 
         return result
 
-    def fetchSingle(self, trx):
+    def fetchSingle(self, trx: dict[str, Any]) -> data.Transaction:
         id = int(trx["id"])
         type = int(trx["type"])
         date = parse(trx["datetime"]).date()
@@ -67,12 +69,13 @@ class Importer(importer.ImporterProtocol):
 
         if type == 0:
             narration = "Deposit"
-            cost = data.Cost(
-                self.priceLookup.fetchPriceAmount(posCcy, date), "CHF", None, None
-            )
+            if posCcy:
+                cost = data.Cost(
+                    self.priceLookup.fetchPriceAmount(posCcy, date), "CHF", None, None
+                )
             postings = [
                 data.Posting(
-                    self.account + ":" + posCcy,
+                    self._account + ":" + posCcy,
                     amount.Amount(posAmt, posCcy),
                     cost,
                     None,
@@ -84,7 +87,7 @@ class Importer(importer.ImporterProtocol):
             narration = "Withdrawal"
             postings = [
                 data.Posting(
-                    self.account + ":" + negCcy,
+                    self._account + ":" + negCcy,
                     amount.Amount(negAmt, negCcy),
                     None,
                     None,
@@ -94,14 +97,15 @@ class Importer(importer.ImporterProtocol):
             ]
         elif type == 2:
             fee = D(trx["fee"])
-            if posCcy.lower() + "_" + negCcy.lower() in trx:
+            if posCcy and negCcy and posCcy.lower() + "_" + negCcy.lower() in trx:
                 feeCcy = negCcy
                 negAmt -= fee
             else:
                 feeCcy = posCcy
                 posAmt -= fee
 
-            rateFiatCcy = self.priceLookup.fetchPriceAmount(feeCcy, date)
+            if feeCcy:
+                rateFiatCcy = self.priceLookup.fetchPriceAmount(feeCcy, date)
             if feeCcy == posCcy:
                 posCcyCost = None
                 posCcyPrice = amount.Amount(rateFiatCcy, "CHF")
@@ -119,7 +123,7 @@ class Importer(importer.ImporterProtocol):
 
             postings = [
                 data.Posting(
-                    self.account + ":" + posCcy,
+                    self._account + ":" + posCcy,
                     amount.Amount(posAmt, posCcy),
                     posCcyCost,
                     posCcyPrice,
@@ -127,7 +131,7 @@ class Importer(importer.ImporterProtocol):
                     None,
                 ),
                 data.Posting(
-                    self.account + ":" + negCcy,
+                    self._account + ":" + negCcy,
                     amount.Amount(negAmt, negCcy),
                     negCcyCost,
                     negCcyPrice,

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/src/tariochbctools/importers/blockchain/importer.py

@@ -1,28 +1,29 @@
 from os import path
 
+import beangulp
 import blockcypher
 import yaml
 from beancount.core import amount, data
 from beancount.core.number import D
-from beancount.ingest import importer
 
 from tariochbctools.importers.general.priceLookup import PriceLookup
 
 
-class Importer(importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for Blockchain data."""
 
-    def identify(self, file):
-        return path.basename(file.name).endswith("blockchain.yaml")
+    def identify(self, filepath: str) -> bool:
+        return path.basename(filepath).endswith("blockchain.yaml")
 
-    def file_account(self, file):
+    def account(self, filepath: str) -> data.Entries:
         return ""
 
-    def extract(self, file, existing_entries):
-        config = yaml.safe_load(file.contents())
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        with open(filepath) as file:
+            config = yaml.safe_load(file)
         self.config = config
         baseCcy = config["base_ccy"]
-        priceLookup = PriceLookup(existing_entries, baseCcy)
+        priceLookup = PriceLookup(existing, baseCcy)
 
         entries = []
         for address in self.config["addresses"]:

tariochbctools-1.0.0/src/tariochbctools/importers/cembrastatement/importer.py

@@ -0,0 +1,118 @@
+import datetime
+import re
+from datetime import timedelta
+
+import beangulp
+import camelot
+from beancount.core import amount, data
+from beancount.core.number import D
+
+
+class Importer(beangulp.Importer):
+    """An importer for Cembra Card Statement PDF files."""
+
+    def __init__(self, filepattern: str, account: data.Account):
+        self._filepattern = filepattern
+        self._account = account
+        self.currency = "CHF"
+
+    def identify(self, filepath: str) -> bool:
+        return re.search(self._filepattern, filepath) is not None
+
+    def account(self, filepath: str) -> data.Account:
+        return self._account
+
+    def createEntry(
+        self, filepath: str, date: datetime.date, amt: data.Decimal, text: str
+    ) -> data.Transaction:
+        meta = data.new_metadata(filepath, 0)
+        return data.Transaction(
+            meta,
+            date,
+            "*",
+            "",
+            text,
+            data.EMPTY_SET,
+            data.EMPTY_SET,
+            [
+                data.Posting(self._account, amt, None, None, None, None),
+            ],
+        )
+
+    def createBalanceEntry(
+        self, filepath: str, date: datetime.date, amt: data.Decimal
+    ) -> data.Balance:
+        meta = data.new_metadata(filepath, 0)
+        return data.Balance(meta, date, self._account, amt, None, None)
+
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        entries = []
+
+        tables = camelot.read_pdf(
+            filepath, pages="2-end", flavor="stream", table_areas=["50,700,560,50"]
+        )
+        for table in tables:
+            df = table.df
+
+            # skip incompatible tables
+            if df.columns.size != 5:
+                continue
+
+            for index, row in df.iterrows():
+                trx_date, book_date, text, credit, debit = tuple(row)
+                trx_date, book_date, text, credit, debit = (
+                    trx_date.strip(),
+                    book_date.strip(),
+                    text.strip(),
+                    credit.strip(),
+                    debit.strip(),
+                )
+
+                # Transaction entry
+                try:
+                    book_date = datetime.datetime.strptime(book_date, "%d.%m.%Y").date()
+                except Exception:
+                    book_date = None
+
+                if book_date:
+                    amount = self.getAmount(debit, credit)
+
+                    if amount:
+                        entries.append(
+                            self.createEntry(filepath, book_date, amount, text)
+                        )
+                    continue
+
+                # Balance entry
+                try:
+                    m = re.search(
+                        r"Saldo per (\d\d\.\d\d\.\d\d\d\d) zu unseren Gunsten CHF",
+                        text,
+                    )
+                    if m:
+                        book_date = m.group(1)
+                        book_date = datetime.datetime.strptime(
+                            book_date, "%d.%m.%Y"
+                        ).date()
+                        # add 1 day: cembra provides balance at EOD, but beancount checks it at SOD
+                        book_date = book_date + timedelta(days=1)
+                except Exception:
+                    book_date = None
+
+                if book_date:
+                    amount = self.getAmount(debit, credit)
+
+                    if amount:
+                        entries.append(
+                            self.createBalanceEntry(filepath, book_date, amount)
+                        )
+
+        return entries
+
+    def cleanDecimal(self, formattedNumber: str) -> data.Decimal:
+        return D(formattedNumber.replace("'", ""))
+
+    def getAmount(self, debit: str, credit: str) -> data.Amount:
+        amt = -self.cleanDecimal(debit) if debit else self.cleanDecimal(credit)
+        if amt:
+            return amount.Amount(amt, self.currency)

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/src/tariochbctools/importers/general/mailAdapterImporter.py

@@ -2,24 +2,26 @@ import tempfile
 from os import path
 
 import yaml
-from beancount.ingest import cache, importer
+from beancount.core import data
+from beangulp import Importer
 from imap_tools import MailBox
 
 
-class MailAdapterImporter(importer.ImporterProtocol):
+class MailAdapterImporter(Importer):
     """An importer adapter that fetches file from mails and then calls another importer."""
 
-    def __init__(self, importers):
+    def __init__(self, importers: list[Importer]):
         self.importers = importers
 
-    def identify(self, file):
-        return "mail.yaml" == path.basename(file.name)
+    def identify(self, filepath: str) -> bool:
+        return "mail.yaml" == path.basename(filepath)
 
-    def file_account(self, file):
+    def account(self, filepath: str) -> data.Account:
         return ""
 
-    def extract(self, file, existing_entries):
-        config = yaml.safe_load(file.contents())
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        with open(filepath) as file:
+            config = yaml.safe_load(file)
 
         with MailBox(config["host"]).login(
             config["user"], config["password"], initial_folder=config["folder"]
@@ -33,13 +35,10 @@ class MailAdapterImporter(importer.ImporterProtocol):
                         with open(attFileName, "wb") as attFile:
                             attFile.write(att.payload)
                             attFile.flush()
-                            fileMemo = cache.get_file(attFileName)
 
                             for delegate in self.importers:
-                                if delegate.identify(fileMemo):
-                                    newEntries = delegate.extract(
-                                        fileMemo, existing_entries
-                                    )
+                                if delegate.identify(attFileName):
+                                    newEntries = delegate.extract(attFileName, existing)
                                     result.extend(newEntries)
                                     processed = True
 

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/src/tariochbctools/importers/general/mt940importer.py

@@ -1,29 +1,28 @@
+import re
+from typing import Any
+
+import beangulp
 import mt940
 from beancount.core import amount, data
 from beancount.core.number import D
-from beancount.ingest import importer
-from beancount.ingest.importers.mixins import identifier
 
 
-class Importer(identifier.IdentifyMixin, importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for MT940 files."""
 
-    def __init__(self, regexps, account):
-        identifier.IdentifyMixin.__init__(self, matchers=[("filename", regexps)])
-        self.account = account
-
-    def identify(self, file):
-        if file.mimetype() != "text/plain":
-            return False
+    def __init__(self, filepattern: str, account: data.Account):
+        self._filepattern = filepattern
+        self._account = account
 
-        return super().identify(file)
+    def identify(self, filepath: str) -> bool:
+        return re.search(self._filepattern, filepath) is not None
 
-    def file_account(self, file):
-        return self.account
+    def account(self, filepath: str) -> data.Account:
+        return self._account
 
-    def extract(self, file, existing_entries):
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
         entries = []
-        transactions = mt940.parse(file.contents())
+        transactions = mt940.parse(filepath)
         for trx in transactions:
             trxdata = trx.data
             ref = trxdata["bank_reference"]
@@ -31,7 +30,7 @@ class Importer(identifier.IdentifyMixin, importer.ImporterProtocol):
                 metakv = {"ref": ref}
             else:
                 metakv = None
-            meta = data.new_metadata(file.name, 0, metakv)
+            meta = data.new_metadata(filepath, 0, metakv)
             if "entry_date" in trxdata:
                 date = trxdata["entry_date"]
             else:
@@ -46,7 +45,7 @@ class Importer(identifier.IdentifyMixin, importer.ImporterProtocol):
                 data.EMPTY_SET,
                 [
                     data.Posting(
-                        self.account,
+                        self._account,
                         amount.Amount(
                             D(trxdata["amount"].amount), trxdata["amount"].currency
                         ),
@@ -61,8 +60,8 @@ class Importer(identifier.IdentifyMixin, importer.ImporterProtocol):
 
         return entries
 
-    def prepare_payee(self, trxdata):
+    def prepare_payee(self, trxdata: dict[str, Any]) -> str:
         return ""
 
-    def prepare_narration(self, trxdata):
+    def prepare_narration(self, trxdata: dict[str, Any]) -> str:
         return trxdata["transaction_details"] + " " + trxdata["extra_details"]

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/src/tariochbctools/importers/general/priceLookup.py

@@ -1,18 +1,18 @@
 from datetime import date
 
-from beancount.core import amount, prices
+from beancount.core import amount, data, prices
 from beancount.core.number import D
 
 
 class PriceLookup:
-    def __init__(self, existing_entries, baseCcy: str):
-        if existing_entries:
-            self.priceMap = prices.build_price_map(existing_entries)
+    def __init__(self, existing: data.Entries, baseCcy: str):
+        if existing:
+            self.priceMap = prices.build_price_map(existing)
         else:
             self.priceMap = None
         self.baseCcy = baseCcy
 
-    def fetchPriceAmount(self, instrument: str, date: date):
+    def fetchPriceAmount(self, instrument: str, date: date) -> data.Amount:
         if self.priceMap:
             price = prices.get_price(
                 self.priceMap, tuple([instrument, self.baseCcy]), date
@@ -21,7 +21,7 @@ class PriceLookup:
         else:
             return D(1)
 
-    def fetchPrice(self, instrument: str, date: date):
+    def fetchPrice(self, instrument: str, date: date) -> data.Amount:
         if instrument == self.baseCcy:
             return None
 

{tariochbctools-0.38.1 → tariochbctools-1.0.0}/src/tariochbctools/importers/ibkr/importer.py

@@ -2,27 +2,30 @@ import re
 from datetime import date
 from decimal import Decimal
 from os import path
+from typing import Any
 
+import beangulp
 import yaml
 from beancount.core import amount, data
 from beancount.core.number import D
-from beancount.ingest import importer
 from ibflex import Types, client, parser
 from ibflex.enums import CashAction
 
 from tariochbctools.importers.general.priceLookup import PriceLookup
 
 
-class Importer(importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for Interactive Broker using the flex query service."""
 
-    def identify(self, file):
-        return path.basename(file.name).endswith("ibkr.yaml")
+    def identify(self, filepath: str) -> bool:
+        return path.basename(filepath).endswith("ibkr.yaml")
 
-    def file_account(self, file):
+    def account(self, filepath: str) -> data.Account:
         return ""
 
-    def matches(self, trx, t, account):
+    def matches(
+        self, trx: Types.CashTransaction, t: Any, account: data.Account
+    ) -> bool:
         p = re.compile(r".* (?P<perShare>\d+\.?\d+) PER SHARE")
 
         trxPerShareGroups = p.search(trx.description)
@@ -38,13 +41,13 @@ class Importer(importer.ImporterProtocol):
             and t["account"] == account
         )
 
-    def extract(self, file, existing_entries):
-        with open(file.name, "r") as f:
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        with open(filepath, "r") as f:
             config = yaml.safe_load(f)
         token = config["token"]
         queryId = config["queryId"]
 
-        priceLookup = PriceLookup(existing_entries, config["baseCcy"])
+        priceLookup = PriceLookup(existing, config["baseCcy"])
 
         response = client.download(token, queryId)
         statement = parser.parse(response)
@@ -52,7 +55,7 @@ class Importer(importer.ImporterProtocol):
 
         result = []
         for stmt in statement.FlexStatements:
-            transactions = []
+            transactions: list = []
             account = stmt.accountId
             for trx in stmt.Trades:
                 result.append(
@@ -147,7 +150,7 @@ class Importer(importer.ImporterProtocol):
         priceLookup: PriceLookup,
         description: str,
         account: str,
-    ):
+    ) -> data.Transaction:
         narration = "Dividend: " + description
         liquidityAccount = self.getLiquidityAccount(account, currency)
         incomeAccount = self.getIncomeAccount(account)
@@ -190,7 +193,7 @@ class Importer(importer.ImporterProtocol):
     def createBuy(
         self,
         date: date,
-        account: str,
+        account: data.Account,
         asset: str,
         quantity: Decimal,
         currency: str,
@@ -199,7 +202,7 @@ class Importer(importer.ImporterProtocol):
         netCash: amount.Amount,
         baseCcy: str,
         fxRateToBase: Decimal,
-    ):
+    ) -> data.Transaction:
         narration = "Buy"
         feeAccount = self.getFeeAccount(account)
         liquidityAccount = self.getLiquidityAccount(account, currency)
@@ -238,17 +241,17 @@ class Importer(importer.ImporterProtocol):
             meta, date, "*", "", narration, data.EMPTY_SET, data.EMPTY_SET, postings
         )
 
-    def getAssetAccount(self, account: str, asset: str):
+    def getAssetAccount(self, account: str, asset: str) -> data.Account:
         return f"Assets:{account}:Investment:IB:{asset}"
 
-    def getLiquidityAccount(self, account: str, currency: str):
+    def getLiquidityAccount(self, account: str, currency: str) -> data.Account:
         return f"Assets:{account}:Liquidity:IB:{currency}"
 
-    def getReceivableAccount(self, account: str):
+    def getReceivableAccount(self, account: str) -> data.Account:
         return f"Assets:{account}:Receivable:Verrechnungssteuer"
 
-    def getIncomeAccount(self, account: str):
+    def getIncomeAccount(self, account: str) -> data.Account:
         return f"Income:{account}:Interest"
 
-    def getFeeAccount(self, account: str):
+    def getFeeAccount(self, account: str) -> data.Account:
         return f"Expenses:{account}:Fees"