wbportfolio 1.49.8__py2.py3-none-any.whl → 1.49.10__py2.py3-none-any.whl

wbportfolio/import_export/parsers/natixis/fees.py

@@ -7,7 +7,7 @@ FIELD_MAP = {
     "Date": "transaction_date",
     "Manag. Fees Natixis": "ISSUER",
     "Manag. Fees Client": "MANAGEMENT",
-    "Perf fees amount": "PERFORMANCE"
+    "Perf fees amount": "PERFORMANCE",
     # "Currency": "currency"
 }
 

wbportfolio/import_export/parsers/sg_lux/customer_trade_slk.py

@@ -10,10 +10,27 @@ from wbportfolio.models import Product, Trade
 from .sylk import SYLK
 from .utils import get_portfolio_id
 
+MAP = {
+    "register_id": "register__register_reference",
+    "register_name": "bank",
+    "isin": "underlying_instrument",
+    "trans_date": "transaction_date",
+    "settl_date": "value_date",
+    "trans_ref": "external_id",
+    "trans_price": "price",
+    "quantity": "shares",
+    "note": "comment",
+}
+
 
 def parse(import_source):
     data = list()
     sylk_handler = SYLK()
+    merger_isin_map = import_source.source.import_parameters.get("custom_merged_isin_map", {"XX": "LU"})
+    merger_export_parts_trans_desc = import_source.source.import_parameters.get(
+        "merger_export_parts_trans_desc", "Export Parts"
+    )
+
     for line in [_line.decode("cp1252") for _line in import_source.file.open("rb").readlines()]:
         sylk_handler.parseline(line)
 
@@ -24,11 +41,10 @@ def parse(import_source):
 
     buffer.seek(0)
     content = buffer.read().replace('""', "")
-    df = pd.read_csv(StringIO(content), sep=";", quotechar="'", usecols=[1, 3, 4, 7, 10, 15, 16, 17, 18, 20, 21])
+    df = pd.read_csv(StringIO(content), sep=";", quotechar="'")
     if not df.empty:
         # Filter out all non transaction rows and remove record_desc col
         df = df[df["record_desc"].str.strip() == "TRANSACTION"]
-        del df["record_desc"]
 
         # Convert timestamps to json conform date strings
         df["trans_date"] = df["trans_date"].apply(lambda x: xldate_as_datetime(x, datemode=0).date())
@@ -39,9 +55,17 @@ def parse(import_source):
         # Replace all nan values with empty str
         df[["register_firstname", "cust_ref"]] = df[["register_firstname", "cust_ref"]].replace(np.nan, "", regex=True)
 
-        # Merge register_firstname and cust_ref and then remove both cols
+        # Merge register_firstname and cust_ref
         df["note"] = df["register_firstname"] + df["cust_ref"]
-        del df["register_firstname"]
+        # the trade marked with "Export Parts" and where isin is marked for merge need to be filter out as they are introduced by the merger
+        df = df[
+            ~(
+                (df["trans_desc"] == merger_export_parts_trans_desc)
+                & (df["isin"].str.contains("|".join(merger_isin_map.keys())))
+            )
+        ]
+        for pat, repl in merger_isin_map.items():
+            df["isin"] = df["isin"].str.replace(pat, repl)
 
         # Create Product Mapping and apply to df
         product_mapping = {
@@ -50,21 +74,10 @@ def parse(import_source):
         }
         df["isin"] = df["isin"].apply(lambda x: product_mapping[x])
 
-        del df["cust_ref"]
-
         # Rename Columns
+        df = df.rename(columns=MAP)
+        df = df[df.columns.intersection(MAP.values())]
 
-        df.columns = [
-            "register__register_reference",
-            "bank",
-            "underlying_instrument",
-            "transaction_date",
-            "value_date",
-            "external_id",
-            "price",
-            "shares",
-            "comment",
-        ]
         df["transaction_subtype"] = df.shares.apply(
             lambda x: Trade.Type.REDEMPTION.value if x < 0 else Trade.Type.SUBSCRIPTION.value
         )
@@ -74,6 +87,7 @@ def parse(import_source):
         )
         df["pending"] = False
         # Convert df to list of dicts
+
         data = df.to_dict("records")
 
     return {

wbportfolio/models/asset.py

@@ -1,6 +1,8 @@
+from collections import defaultdict
 from contextlib import suppress
 from datetime import date
 from decimal import Decimal, InvalidOperation
+from typing import TYPE_CHECKING
 
 from django.contrib import admin
 from django.db import models
@@ -24,6 +26,7 @@ from django.db.models.functions import Coalesce
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.utils.functional import cached_property
+from pandas._libs.tslibs.offsets import BDay
 from wbcore.contrib.currency.models import Currency, CurrencyFXRates
 from wbcore.contrib.io.mixins import ImportMixin
 from wbcore.signals import pre_merge
@@ -51,6 +54,149 @@ MINUTE = 60
 HOUR = MINUTE * 60
 DAY = HOUR * 24
 
+if TYPE_CHECKING:
+    from wbportfolio.models.portfolio import Portfolio
+
+
+class AssetPositionIterator:
+    """
+    Efficiently converts position data into AssetPosition models with batch operations
+    and proper dependency management.
+
+    Features:
+    - Bulk database fetching for performance
+    - Thread-safe operations
+    - Clear type hints
+    - Memory-efficient storage
+    """
+
+    positions: dict[tuple[date, int, int | None], "AssetPosition"]
+
+    _prices: dict[date, dict[int, float]]
+    _weights: dict[date, dict[int, float]]
+    _fx_rates: dict[date, dict[Currency, CurrencyFXRates]]
+    _instruments: dict[int, Instrument]
+
+    def __init__(
+        self,
+        portfolio: "Portfolio",
+        prices: dict[date, dict[int, float]] | None = None,
+        infer_underlying_quote_price: bool = False,
+    ):
+        self.portfolio = portfolio
+        self.infer_underlying_quote_price = infer_underlying_quote_price
+        # Initialize data stores with type hints
+        self._instruments = {}
+        self._fx_rates = defaultdict(dict)
+        self._weights = dict()
+        self._prices = prices or defaultdict(dict)
+        self.positions = dict()
+
+    def _get_instrument(self, instrument_id: int) -> Instrument:
+        try:
+            return self._instruments[instrument_id]
+        except KeyError:
+            instrument = Instrument.objects.get(id=instrument_id)
+            self._instruments[instrument_id] = instrument
+            return instrument
+
+    def _get_fx_rate(self, val_date: date, currency: Currency) -> CurrencyFXRates:
+        try:
+            return self._fx_rates[val_date][currency]
+        except KeyError:
+            fx_rate = CurrencyFXRates.objects.get_or_create(
+                currency=currency, date=val_date, defaults={"value": Decimal(0)}
+            )[0]  # we create a fx rate anyway to not fail the position. The fx rate expect to be there later on
+            self._fx_rates[val_date][currency] = fx_rate
+            return fx_rate
+
+    def _get_price(self, val_date: date, instrument: Instrument) -> float | None:
+        try:
+            return self._prices[val_date][instrument.id]
+        except KeyError:
+            return None
+
+    def _dict_to_model(self, val_date: date, instrument_id: int, weighting: float) -> "AssetPosition":
+        underlying_quote = self._get_instrument(instrument_id)
+        currency_fx_rate_portfolio_to_usd = self._get_fx_rate(val_date, self.portfolio.currency)
+        currency_fx_rate_instrument_to_usd = self._get_fx_rate(val_date, underlying_quote.currency)
+        price = self._get_price(val_date, underlying_quote)
+
+        position = AssetPosition(
+            underlying_quote=underlying_quote,
+            weighting=weighting,
+            date=val_date,
+            asset_valuation_date=val_date,
+            is_estimated=True,
+            portfolio=self.portfolio,
+            currency=underlying_quote.currency,
+            initial_price=price,
+            currency_fx_rate_portfolio_to_usd=currency_fx_rate_portfolio_to_usd,
+            currency_fx_rate_instrument_to_usd=currency_fx_rate_instrument_to_usd,
+            initial_currency_fx_rate=None,
+            underlying_quote_price=None,
+            underlying_instrument=None,
+        )
+        position.pre_save(
+            infer_underlying_quote_price=self.infer_underlying_quote_price
+        )  # inferring underlying quote price is potentially very slow for big dataset of positions, it's not very needed for model portfolio so we disable it
+        return position
+
+    def add(
+        self,
+        positions: list["AssetPosition"] | tuple[date, dict[int, float]],
+    ):
+        """
+        Add multiple positions efficiently with batch processing
+
+        Args:
+            positions: Iterable of AssetPosition instances or dictionary of weight {instrument_id: weight} that needs to be converted into AssetPosition
+        """
+        if isinstance(positions, tuple):
+            val_date = positions[0]
+            positions = [(val_date, i, w) for i, w in positions[1].items()]  # unflatten data to make it iterable
+        for position in positions:
+            if not isinstance(position, AssetPosition):
+                position = self._dict_to_model(*position)
+
+            # Generate unique composite key
+            key = (
+                position.date,
+                position.underlying_quote.id,
+                position.portfolio_created.id if position.portfolio_created else None,
+            )
+            # Merge duplicate positions
+            if existing_position := self.positions.get(key):
+                position.weighting += existing_position.weighting
+                position.initial_shares += existing_position.initial_shares
+            # ensure the position portfolio is the iterator portfolio (could be different when computing look-through for instance)
+            position.portfolio = self.portfolio
+            if position.initial_price is not None:
+                self.positions[key] = position
+                try:
+                    self._weights[position.date][position.underlying_quote.id] += float(position.weighting)
+                except KeyError:
+                    self._weights[position.date] = {position.underlying_quote.id: float(position.weighting)}
+        return self
+
+    def get_dates(self) -> list[date]:
+        """Get sorted list of unique dates"""
+        return list(self._weights.keys())
+
+    def get_weights(self) -> dict[date, dict[int, float]]:
+        """Get weight structure with instrument IDs as keys"""
+        return self._weights
+
+    def __iter__(self):
+        # return an iterable excluding the position without a valid weighting
+        yield from filter(lambda a: a.weighting, self.positions.values())
+
+    def __getitem__(self, item: tuple[date, Instrument]) -> float:
+        return self._weights[item[0]][item[1].id]
+
+    def __bool__(self) -> bool:
+        return len(self.positions.keys()) > 0
+
 
 class AssetPositionDefaultQueryset(QuerySet):
     def annotate_classification_for_group(
@@ -498,6 +644,17 @@ class AssetPosition(ImportMixin, models.Model):
                 except InvalidOperation:
                     self.initial_currency_fx_rate = Decimal(0.0)
 
+        # we set the initial shares from the previous position shares number if portfolio allows it
+        if self.initial_shares is None and not self.portfolio.only_weighting:
+            with suppress(AssetPosition.DoesNotExist):
+                previous_pos = AssetPosition.objects.get(
+                    date=(self.date - BDay(1)).date(),
+                    underlying_quote=self.underlying_quote,
+                    portfolio=self.portfolio,
+                    portfolio_created=self.portfolio_created,
+                )
+                self.initial_shares = previous_pos.initial_shares
+
     def save(self, *args, create_underlying_quote_price_if_missing: bool = False, **kwargs):
         self.pre_save(create_underlying_quote_price_if_missing=create_underlying_quote_price_if_missing)
         super().save(*args, **kwargs)