wbportfolio 2.2.1__py2.py3-none-any.whl

wbportfolio/import_export/parsers/sg_lux/portfolio_cash_flow.py

@@ -0,0 +1,29 @@
+from datetime import datetime
+
+import pandas as pd
+from wbportfolio.models import ProductGroup
+
+
+def parse(import_source):
+    csv_file = import_source.file.open()
+
+    df = pd.read_csv(csv_file, encoding="latin1", usecols=[0, 2, 4, 18])
+
+    value_date = df["NAV Date"].iloc[0]
+    total_assets = df.groupby("Code").sum("Market Value FCY")
+    cash = df[df["Asset type code"] == "TRES"].groupby("Code").sum("Market Value FCY")
+
+    data = []
+
+    for code in df["Code"].unique():
+        data.append(
+            {
+                "portfolio": ProductGroup.objects.get(identifier=code).portfolio.id,
+                "value_date": datetime.strptime(value_date, "%Y/%m/%d").strftime("%Y-%m-%d"),
+                "total_assets": total_assets.T[code].iloc[0],
+                "cash": cash.T[code].iloc[0],
+                "pending": False,
+            }
+        )
+
+    return {"data": data}

wbportfolio/import_export/parsers/sg_lux/portfolio_future_cash_flow.py

@@ -0,0 +1,36 @@
+from contextlib import suppress
+from datetime import datetime
+
+import pandas as pd
+from wbportfolio.models import Portfolio
+
+
+def parse(import_source):
+    csv_file = import_source.file.open()
+
+    df = pd.read_csv(csv_file)
+
+    def get_portfolio_id(row) -> int | None:
+        with suppress(Portfolio.DoesNotExist):
+            return Portfolio.objects.get(instruments__children__isin__in=[row["Isin"]]).pk
+
+    df = df[~df["Isin"].isnull()]
+    df["Trade date"] = df["Trade date"].ffill()
+    df["portfolio"] = df.apply(get_portfolio_id, axis=1)
+    df = df[["Trade date", "portfolio", "Net Ref ccy"]]
+    df = df.groupby(["Trade date", "portfolio"]).sum().to_dict()
+
+    data = []
+    for keys, cash_flow_forecast in df["Net Ref ccy"].items():
+        data.append(
+            {
+                "value_date": datetime.strptime(keys[0], "%d/%m/%Y").strftime("%Y-%m-%d"),
+                "portfolio": keys[1],
+                "cash_flow_forecast": cash_flow_forecast,
+                "pending": True,
+            }
+        )
+
+    return {
+        "data": data,
+    }

wbportfolio/import_export/parsers/sg_lux/registers.py

@@ -0,0 +1,210 @@
+import csv
+from io import StringIO
+
+import numpy as np
+import pandas as pd
+from wbcore.contrib.geography.models import Geography
+from wbportfolio.models import Register
+from xlrd import xldate_as_datetime
+
+from .sylk import SYLK
+
+
+def convert_string_to_number(string):
+    try:
+        return float(string.replace(" ", "").replace(",", ""))
+    except ValueError:
+        return 0.0
+
+
+def parse(import_source):
+    data = list()
+
+    sylk_handler = SYLK()
+    for line in [_line.decode("cp1252") for _line in import_source.file.open("rb").readlines()]:
+        sylk_handler.parseline(line)
+
+    buffer = StringIO()
+    csvwriter = csv.writer(buffer, quotechar="'", delimiter=";", lineterminator="\n", quoting=csv.QUOTE_ALL)
+    for line in sylk_handler.stream_rows():
+        csvwriter.writerow(line)
+
+    buffer.seek(0)
+    content = buffer.read().replace('""', "")
+    df = pd.read_csv(
+        StringIO(content),
+        sep=";",
+        quotechar="'",
+        usecols=[
+            2,
+            3,
+            4,
+            5,
+            8,
+            9,
+            10,
+            12,
+            13,
+            14,
+            15,
+            16,
+            17,
+            18,
+            19,
+            20,
+            21,
+            22,
+            23,
+            24,
+            25,
+            26,
+            27,
+            28,
+            29,
+            30,
+            32,
+            33,
+            34,
+            36,
+            42,
+            43,
+            46,
+            47,
+            48,
+            49,
+            50,
+            55,
+            56,
+            57,
+            58,
+        ],
+    )
+
+    # Convert timestamps to json conform date strings
+    df["ESTABLISHMENT_DATE"] = df["ESTABLISHMENT_DATE"].apply(lambda x: xldate_as_datetime(x, datemode=0).date())
+    df["ESTABLISHMENT_DATE"] = df["ESTABLISHMENT_DATE"].apply(lambda x: x.strftime("%Y-%m-%d"))
+
+    df = df.replace(np.nan, "", regex=True)
+    # df = df.astype(str)
+    df["GLOBAL_REGISTER_NUMBER"] = df["GLOBAL_REGISTER_NUMBER"].apply(lambda x: str(int(x)) if x else "")
+
+    # Merge TextFields
+    df["custodian_address"] = df[["ADRESS_LINE_1_4", "ADRESS_LINE_2_4", "ADRESS_LINE_3_4", "ADRESS_LINE_4_4"]].apply(
+        lambda x: "<br />".join(x.dropna().astype(str)), axis=1
+    )
+
+    def concat_status(x):
+        status = []
+
+        if code := x["BLOCK_CODE"]:
+            try:
+                status.append(f"{int(code):04}")
+            except ValueError:
+                status.append(code)
+        if desc := x["BLOCK_CODE_DESCRIPTION"]:
+            status.append(desc)
+
+        for n in range(1, 5):
+            if code := x[f"BLOCK_CODE_{n}"]:
+                try:
+                    status.append(f"{int(code):04}")
+                except ValueError:
+                    status.append(code)
+            if desc := x[f"DESCRIPTION_BC{n}"]:
+                status.append(desc)
+
+        return "<br />".join(status)
+
+    df["status_message"] = df.apply(concat_status, axis=1)
+
+    # Map Status
+    def apply_status(x):
+        if x.get("status_message", None) and x["ACTIVE_INACTIVE"] != "INACTIVE":
+            return Register.RegisterStatus.WARNING
+
+        mapping = {
+            "ACTIVE": Register.RegisterStatus.ACTIVE,
+            "INACTIVE": Register.RegisterStatus.INACTIVE,
+        }
+        return mapping[x["ACTIVE_INACTIVE"]]
+
+    df["status"] = df.apply(apply_status, axis=1)
+
+    # Remove not used columns
+    del df["ADRESS_LINE_1_4"]
+    del df["ADRESS_LINE_2_4"]
+    del df["ADRESS_LINE_3_4"]
+    del df["ADRESS_LINE_4_4"]
+    del df["BLOCK_CODE"]
+    del df["BLOCK_CODE_DESCRIPTION"]
+    del df["BLOCK_CODE_1"]
+    del df["DESCRIPTION_BC1"]
+    del df["BLOCK_CODE_2"]
+    del df["DESCRIPTION_BC2"]
+    del df["BLOCK_CODE_3"]
+    del df["DESCRIPTION_BC3"]
+    del df["BLOCK_CODE_4"]
+    del df["DESCRIPTION_BC4"]
+    del df["ACTIVE_INACTIVE"]
+
+    # Map Countries
+    country_code_mapping = {
+        country["code_2"]: country["id"] for country in Geography.countries.all().values("id", "code_2")
+    }
+    country_code_mapping["XX"] = country_code_mapping["ZZ"]
+    country_title_mapping = {
+        country["name"].lower(): country["id"] for country in Geography.countries.all().values("id", "name")
+    }
+    country_title_mapping["great-britain"] = Geography.countries.get(code_2="GB").id
+    country_title_mapping["man (isle of)"] = Geography.countries.get(code_2="IM").id
+
+    df["CITIZENSHIP"] = df["CITIZENSHIP"].apply(lambda x: country_code_mapping[x])
+    df["RESIDENCE"] = df["RESIDENCE"].apply(lambda x: country_code_mapping[x])
+    df["CPAYS"] = df["CPAYS"].apply(lambda x: country_code_mapping[x])
+    df["COUNTRY"] = df["COUNTRY"].apply(lambda x: country_title_mapping[x.lower()])
+    df["CITY"] = df["CITY"].str.title()
+    # Map Investor Type
+    investor_mapping = {
+        "0006": Register.RegisterInvestorType.BANK,
+        "0011": Register.RegisterInvestorType.NOMINEE,
+        "0003": Register.RegisterInvestorType.GLOBAL,
+        "0007": Register.RegisterInvestorType.NON_FINANCIAL_ENTITY,
+    }
+    df["INVESTOR_TYPE_CODE"] = df["INVESTOR_TYPE_CODE"].apply(lambda x: investor_mapping[f"{x:04}"])
+
+    # Rename Columns
+    df.columns = [
+        "register_reference",
+        "opened",
+        "register_name_1",
+        "register_name_2",
+        "custodian_reference",
+        "custodian_name_1",
+        "custodian_name_2",
+        "custodian_postcode",
+        "custodian_city",
+        "custodian_country",
+        "investor_type",
+        "global_register_reference",
+        "citizenship",
+        "residence",
+        "external_register_reference",
+        "sales_reference",
+        "dealer_reference",
+        "outlet_reference",
+        "outlet_name",
+        "outlet_address",
+        "outlet_postcode",
+        "outlet_country",
+        "opened_reference_1",
+        "opened_reference_2",
+        "updated_reference_1",
+        "updated_reference_1",
+        "custodian_address",
+        "status_message",
+        "status",
+    ]
+    # Convert df to list of dicts
+    data = list(df.T.to_dict().values())
+
+    return {"data": data}

wbportfolio/import_export/parsers/sg_lux/sylk.py

@@ -0,0 +1,248 @@
+# Author: Skip Montanaro (skip@mojam.com)
+# Version: 0.1
+# https://github.com/smontanaro/python-bits/blob/master/sylk.py
+
+"""Read SYLK files
+
+Caveat emptor!  This module has only been tested with SYLK files generated
+by AppleWorks 5.0!  It almost certainly needs work to be able to process
+files generated by other spreadsheets.
+"""
+
+import re
+
+
+class Table:
+    """
+    Table of rows used to store sylk parsed datas
+
+    rows are compound of a various amount of datas (different length)
+    """
+
+    def __init__(self):
+        self.rows = []
+
+    def __setitem__(self, xy_tuple, val):
+        """
+        Setitem method to set datas in this grid
+
+        :param tuple xy_tuple: The x,y coordinates as 2-uple
+        :param str val: The value to store in the grid
+
+        Handle the completion of a row to ensure the grid's row is wide enough
+        to set the value
+        """
+        (x, y) = xy_tuple
+        ox = x - 1
+        oy = y - 1
+        # Ensure the table is long enough to store the val
+        while len(self.rows) < y:
+            new_row = [" "] * x
+            self.rows.append(new_row)
+
+        row = self.rows[oy]
+        if val not in ("", " "):
+            # Ensure the row is large enough to store the val
+            if len(row) < x:
+                self._extend_row(row, x - len(row))
+            self.rows[oy][ox] = val
+
+    def _extend_row(self, row, missing):
+        """
+        Extend a row adding void values
+
+        :param list row: a list of datas
+        :param int missing: The number of void items to add
+        """
+        row.extend([" "] * missing)
+        return row
+
+    def __iter__(self):
+        for row in self.rows:
+            yield row
+
+
+class SYLK:
+    """class to read SYLK files and dump to CSV"""
+
+    # when time began
+    # different computers use different core dates and store dates as offsets
+    # this makes SYLK inherently unportable, but we fudge that by
+    # using the ID field to guess at the creating platform
+    # note that PCs apparently can't properly decode SYLK files generated
+    # on Macs using Appleworks/Clarisworks because they don't take this into
+    # account
+    unixepoch = (1970, 1, 1, 0, 0, 0, 0, 0, 0)
+    macepoch = (1904, 1, 1, 0, 0, 0, 0, 0, 0)
+    # this is pure fiction...
+    pcepoch = (1900, 1, 1, 0, 0, 0, 0, 0, 0)
+
+    # map SYLK format strings into data types
+    knownformats = {
+        "General": "string",
+        "0": "int",
+        "0.00": "float",
+        "#,##0": "int",
+        "#,##0.00": "float",
+        r'"$"#,##0\ ;;\("$"#,##0\,': "float",
+        r'"$"#,##0.00\ ;;\("$"#,##0.00\,': "float",
+        "0%": "float",
+        "0.00%": "float",
+        "0.00E+00": "float",
+        "m/d/yy": "date",
+        "d-mmm-yy": "date",
+        "d-mmm": "date",
+        "mmm-yy": "date",
+        "h:mm AM/PM": "time",
+        "h:mm:ss AM/PM": "time",
+        "h:mm": "time",
+        "h:mm:ss": "time",
+        "hh:mm AM/PM": "time",
+        "hh:mm:ss AM/PM": "time",
+        "m/d/yy h:mm": "datetime",
+        "m-dd-yy": "date",
+        "m-dd": "date",
+        '"$"#,##0 ;;[Red]("$"#,##0,': "float",
+        '"$"#,##0.00 ;;[Red]("$"#,##0.00,': "float",
+        "mmm d, yyyy": "date",
+        "mmmm d, yyyy": "date",
+        "ddd, mmm d, yyyy": "date",
+        "dddd, mmmm d, yyyy": "date",
+        "d, mmmm yyyy": "date",
+    }
+
+    date_output = "%d/%m/%Y"
+
+    def __init__(self):
+        self.datebase = self.unixepoch
+        self.printformats = []
+        self.currentformat = self.currenttype = ""
+        self.curx = self.cury = 0
+        self.data = Table()
+        self.unknown = {}
+
+    def escape(self, s):
+        """
+        Escape a string
+        """
+        if s[0:1] == '"':
+            return '"' + re.sub('"', '\\"\\"', s[1:-1]) + '"'
+        return s
+
+    def parse(self, stream):
+        """
+        Parse the given stream
+        """
+        lines = re.sub("[\r\n]+", "\n", stream.read()).split("\n")
+        for line in lines:
+            self.parseline(line)
+
+    def stream_rows(self):
+        """
+        Stream the rows (to be used to write a csv file for example)
+        """
+        return self.data
+
+    def addunknown(self, fld, subfld):
+        self.unknown[fld] = self.unknown.get(fld, {})
+        self.unknown[fld][subfld] = 1
+
+    def writeunknown(self, stream):
+        if self.unknown:
+            stream.write("Unrecognized fields (subfields):\n")
+            for key in list(self.unknown.keys()):
+                stream.write("%s (%s)\n" % (key, repr(list(self.unknown[key].keys()))))
+        else:
+            stream.write("No unrecognized fields\n")
+        stream.flush()
+
+    def _id_field(self, fields):
+        if fields[1][:6] in ("PClari", "PApple", "P Sage"):
+            self.datebase = self.macepoch
+
+    def _f_field(self, fields):
+        for f in fields[1:]:
+            ftd = f[0]
+            val = f[1:]
+            if ftd == "X":
+                self.curx = int(val)
+            elif ftd == "Y":
+                self.cury = int(val)
+            elif ftd == "P":
+                # references print format for the next cell
+                self.currentformat, self.currenttype = self.printformats[int(val)]
+            else:
+                self.addunknown("F", ftd)
+
+    def _c_field(self, fields):
+        for f in fields[1:]:
+            ftd = f[0]
+            val = f[1:]
+            if ftd == "X":
+                self.curx = int(val)
+
+            elif ftd == "Y":
+                self.cury = int(val)
+
+            elif ftd == "K":
+                val = eval(self.escape(val))
+                # if type(val) == int:
+                #     if self.currenttype == "date":
+                #         # value is offset in days from datebase
+                #         date = time.localtime(
+                #             time.mktime(self.datebase) +
+                #             float(val) * 24 * 60 * 60
+                #         )
+
+                #         val = time.strftime(self.date_output, date)
+                self.data[(self.curx, self.cury)] = "%s" % val
+
+            else:
+                self.addunknown("C", ftd)
+
+    def _p_fields(self, fields):
+        if fields[1][0] == "P":
+            # print formats imply data types?
+            format = fields[1][1:].replace("\\", "")
+            if format in self.knownformats:
+                self.printformats.append((format, self.knownformats[format]))
+            else:
+                # hack to guess type...
+                hasY = "y" in format
+                hasD = "d" in format
+                hasH = "h" in format
+                hasZ = "0" in format
+                hasP = "." in format
+                if (hasD or hasY) and hasH:
+                    dtype = "datetime"
+                elif hasD or hasY:
+                    dtype = "date"
+                elif hasH:
+                    dtype = "time"
+                elif hasP and hasZ:
+                    dtype = "float"
+                elif hasZ:
+                    dtype = "int"
+                else:
+                    dtype = "string"
+                self.printformats.append((format, dtype))
+        else:
+            self.addunknown("P", fields[1][0])
+
+    def parseline(self, line):
+        fields = re.split("(?i);(?=[a-z])", line)
+        if fields[0] == "ID":
+            self._id_field(fields)
+        if fields[0] == "F":
+            self._f_field(fields)
+        elif fields[0] == "C":
+            self._c_field(fields)
+        elif fields[0] == "P":
+            self._p_fields(fields)
+        else:
+            fld = fields[0]
+            for f in fields[1:]:
+                self.addunknown(fld, f)
+
+    def __iter__(self):
+        return self.data.__iter__()

wbportfolio/import_export/parsers/sg_lux/utils.py

@@ -0,0 +1,36 @@
+import re
+from typing import Any, Dict
+
+from wbportfolio.models import Product, Register, Trade
+
+
+def get_portfolio_id(x):
+    product = Product.objects.get(id=x)
+    portfolio = product.primary_portfolio
+    return portfolio.id
+
+
+def parse_transaction_reference(trans_ref: str) -> tuple[str, str, bool]:
+    pattern = r"([0-9]{7})([0-9]{8})([CD]{1})"
+    transaction_id, outlet_id, credit_debit = re.findall(pattern, trans_ref)[0]
+    return transaction_id, outlet_id, credit_debit == "C"
+
+
+def assemble_transaction_reference(data: Dict[str, Any]) -> str:
+    transaction_ref = data["external_identifier2"]
+    register_reference = data["register__register_reference"]
+    if data.get("TRANSFER_REGISTER", None):
+        register_reference = data["TRANSFER_REGISTER"]
+
+    outlet = Register.objects.get(register_reference=register_reference).outlet_reference
+    credit_debit = "C" if data["shares"] > 0 else "D"
+
+    return f"{int(transaction_ref):07}{int(outlet):08}{credit_debit}"
+
+
+def create_transaction_reference(customer_trade: "Trade") -> str:
+    transaction_id = customer_trade.external_identifier2
+    outlet_id = customer_trade.register.clearing_reference
+    credit_debit = "C" if customer_trade.initial_shares > 0 else "D"
+
+    return f"{transaction_id:07}{outlet_id:08}{credit_debit}"

wbportfolio/import_export/parsers/sg_lux/valuation.py

@@ -0,0 +1,53 @@
+import codecs
+import csv
+import datetime
+import re
+
+from wbportfolio.import_export.utils import convert_string_to_number
+from wbportfolio.models import Product
+
+
+def file_name_parse(file_name):
+    dates = re.findall(r"([0-9]{4}-[0-9]{2}-[0-9]{2})", file_name)
+    isin = re.findall(r"\.([a-zA-Z0-9]*)_", file_name)
+
+    assert len(dates) == 2, "Not 2 dates found in the filename"
+    assert len(isin) == 1, "Not exactly 1 isin found in the filename"
+
+    return {
+        "isin": isin[0],
+        "valuation_date": datetime.datetime.strptime(dates[0], "%Y-%m-%d").date(),
+        "generation_date": datetime.datetime.strptime(dates[1], "%Y-%m-%d").date(),
+    }
+
+
+def parse(import_source):
+    # Load file into a CSV DictReader
+    csv_file = import_source.file.open()
+    # csv_file_type = type(csv_file.read())
+
+    # # If the file is a byte string, then we need to convert it.
+    # if csv_file_type is bytes:
+    csv_file = codecs.iterdecode(csv_file, "latin1")
+
+    # # Read file into a CSV Dict Reader
+    csv_reader = csv.DictReader(csv_file, delimiter=",")
+
+    # # Iterate through the CSV File and parse the data into a list
+    data = list()
+
+    for valuation in csv_reader:
+        valuation_date = datetime.datetime.strptime(valuation["NAV Date"], "%Y/%m/%d").date()
+        product = Product.objects.get(isin=valuation["ISIN Code"])
+        if valuation_date.weekday() not in [5, 6] and product.currency.key == valuation["Ccy"]:
+            data.append(
+                {
+                    "instrument": {"instrument_type": "product", "id": product.id},
+                    "date": valuation_date.strftime("%Y-%m-%d"),
+                    "net_value": round(convert_string_to_number(valuation["NAV per share"]), 6),
+                    "calculated": False,
+                }
+            )
+
+    csv_file.close()
+    return {"data": data}

wbportfolio/import_export/parsers/societe_generale/customer_trade.py

@@ -0,0 +1,54 @@
+import datetime
+import re
+from io import BytesIO
+
+import pandas as pd
+from wbportfolio.models import Product, Trade
+
+
+def parse(import_source):
+    df_dict = pd.read_excel(BytesIO(import_source.file.read()), engine="openpyxl", sheet_name=None)
+
+    data = list()
+    products = list()
+    max_date = datetime.date(1900, 1, 1)
+
+    for sheet_name, df in df_dict.items():
+        if "prices" not in sheet_name:
+            isin = re.findall("([A-Z]{2}[A-Z0-9]{9}[0-9]{1})", sheet_name)
+            product = Product.objects.get(isin=isin[0])
+            products.append(product.id)
+            df = df.rename(
+                columns={
+                    "Trade Date": "transaction_date",
+                    "Price": "price",
+                    "Nominal": "nominal",
+                    "Market CP": "bank",
+                    "Way": "way",
+                }
+            )
+            df["transaction_date"] = pd.to_datetime(
+                df["transaction_date"],
+            )
+            for trade in df.to_dict("records"):
+                max_date = max(trade["transaction_date"].date(), max_date)
+
+                shares = trade["nominal"] if trade["way"] == "S" else trade["nominal"] * -1
+                shares = shares / product.share_price
+                portfolio = product.primary_portfolio
+                data.append(
+                    {
+                        "underlying_instrument": {"id": product.id, "instrument_type": "product"},
+                        "currency__key": product.currency.key,
+                        "portfolio": portfolio.id,
+                        "transaction_date": trade["transaction_date"].strftime("%Y-%m-%d"),
+                        "shares": shares,
+                        "transaction_subtype": Trade.Type.REDEMPTION if shares < 0 else Trade.Type.SUBSCRIPTION,
+                        "bank": trade["bank"],
+                        "price": round(trade["price"] / 10, 6),
+                    }
+                )
+    return {
+        "data": data,
+        "history": {"underlying_instruments": products, "transaction_date": max_date.strftime("%Y-%m-%d")},
+    }