wbportfolio 1.49.9__py2.py3-none-any.whl → 1.49.11__py2.py3-none-any.whl

wbportfolio/models/portfolio.py

@@ -1,10 +1,9 @@
 import logging
-from collections import defaultdict
 from contextlib import suppress
 from datetime import date, timedelta
 from decimal import Decimal
 from math import isclose
-from typing import Any, Iterable
+from typing import TYPE_CHECKING, Any, Iterable
 
 import numpy as np
 import pandas as pd
@@ -19,6 +18,7 @@ from django.db.models import (
     Q,
     QuerySet,
     Sum,
+    Value,
 )
 from django.db.models.signals import post_save
 from django.dispatch import receiver
@@ -36,7 +36,7 @@ from wbfdm.models import Instrument, InstrumentType
 from wbfdm.models.instruments.instrument_prices import InstrumentPrice
 from wbfdm.signals import investable_universe_updated
 
-from wbportfolio.models.asset import AssetPosition
+from wbportfolio.models.asset import AssetPosition, AssetPositionIterator
 from wbportfolio.models.indexes import Index
 from wbportfolio.models.portfolio_relationship import (
     InstrumentPortfolioThroughModel,
@@ -50,68 +50,73 @@ from . import ProductGroup
 from .exceptions import InvalidAnalyticPortfolio
 
 logger = logging.getLogger("pms")
-
-
-class PositionDictConverter:
-    """Utility class to load a dictionary of dictionary (dictionary of instrument_id and float per date) into a iterable of assetposition"""
-
-    def __init__(self, portfolio, prices: pd.DataFrame, infer_underlying_quote_price=False):
-        self.portfolio = portfolio
-        self.instruments = {}
-        self.fx_rates = defaultdict(dict)
-        self.prices = prices
-        self.infer_underlying_quote_price = infer_underlying_quote_price
-
-    def _get_instrument(self, instrument_id: int) -> Instrument:
-        try:
-            return self.instruments[instrument_id]
-        except KeyError:
-            instrument = Instrument.objects.get(id=instrument_id)
-            self.instruments[instrument_id] = instrument
-            return instrument
-
-    def _get_fx_rate(self, val_date: date, currency) -> CurrencyFXRates:
-        try:
-            return self.fx_rates[currency][val_date]
-        except KeyError:
-            fx_rate = CurrencyFXRates.objects.get_or_create(
-                currency=currency, date=val_date, defaults={"value": Decimal(0)}
-            )[0]  # we create a fx rate anyway to not fail the position. The fx rate expect to be there later on
-            self.fx_rates[currency][val_date] = fx_rate
-            return fx_rate
-
-    def dict_to_model(self, val_date: date, instrument_id: int, weighting: float) -> AssetPosition | None:
-        underlying_quote = self._get_instrument(instrument_id)
-        currency_fx_rate_portfolio_to_usd = self._get_fx_rate(val_date, self.portfolio.currency)
-        currency_fx_rate_instrument_to_usd = self._get_fx_rate(val_date, underlying_quote.currency)
-        initial_price = self.prices.loc[pd.Timestamp(val_date), instrument_id]
-        # if we cannot find a price for the instrument, we return an empty position
-        if initial_price is not None:
-            position = AssetPosition(
-                underlying_quote=underlying_quote,
-                weighting=weighting,
-                date=val_date,
-                asset_valuation_date=val_date,
-                is_estimated=True,
-                portfolio=self.portfolio,
-                currency=underlying_quote.currency,
-                initial_price=self.prices.loc[pd.Timestamp(val_date), instrument_id],
-                currency_fx_rate_portfolio_to_usd=currency_fx_rate_portfolio_to_usd,
-                currency_fx_rate_instrument_to_usd=currency_fx_rate_instrument_to_usd,
-                initial_currency_fx_rate=None,
-                underlying_quote_price=None,
-                underlying_instrument=None,
-            )
-            position.pre_save(
-                infer_underlying_quote_price=self.infer_underlying_quote_price
-            )  # inferring underlying quote price is potentially very slow for big dataset of positions, it's not very needed for model portfolio so we disable it
-            return position
-
-    def convert(self, positions: dict[date, dict[int, float]]) -> Iterable[AssetPosition]:
-        for val_date, weights in positions.items():
-            yield from filter(
-                lambda x: x, map(lambda row: self.dict_to_model(val_date, row[0], row[1]), weights.items())
-            )
+if TYPE_CHECKING:
+    from wbportfolio.models.transactions.trade_proposals import TradeProposal
+
+
+def get_prices(instrument_ids: list[int], from_date: date, to_date: date) -> dict[date, dict[int, float]]:
+    """
+    Utility to fetch raw prices
+    """
+    prices = InstrumentPrice.objects.filter(instrument__in=instrument_ids, date__gte=from_date, date__lte=to_date)
+    df = (
+        pd.DataFrame(
+            prices.filter_only_valid_prices().values_list("instrument", "net_value", "date"),
+            columns=["instrument", "net_value", "date"],
+        )
+        .pivot_table(index="date", values="net_value", columns="instrument")
+        .astype(float)
+        .sort_index()
+    )
+    ts = pd.bdate_range(df.index.min(), df.index.max(), freq="B")
+    df = df.reindex(ts)
+    df = df.ffill()
+    df.index = pd.to_datetime(df.index)
+    return {ts.date(): row for ts, row in df.to_dict("index").items()}
+
+
+def get_returns(
+    instrument_ids: list[int],
+    from_date: date,
+    to_date: date,
+    to_currency: Currency | None = None,
+    ffill_returns: bool = True,
+) -> pd.DataFrame:
+    """
+    Utility methods to get instrument returns for a given date range
+
+    Args:
+        from_date: date range lower bound
+        to_date: date range upper bound
+
+    Returns:
+        Return a tuple of the returns and the last prices series for conveniance
+    """
+    if to_currency:
+        fx_rate = CurrencyFXRates.get_fx_rates_subquery_for_two_currencies("date", "instrument__currency", to_currency)
+    else:
+        fx_rate = Value(Decimal(1.0))
+    prices = InstrumentPrice.objects.filter(
+        instrument__in=instrument_ids, date__gte=from_date, date__lte=to_date
+    ).annotate(fx_rate=fx_rate, price_fx_portfolio=F("net_value") * F("fx_rate"))
+    prices_df = (
+        pd.DataFrame(
+            prices.filter_only_valid_prices().values_list("instrument", "price_fx_portfolio", "date"),
+            columns=["instrument", "price_fx_portfolio", "date"],
+        )
+        .pivot_table(index="date", values="price_fx_portfolio", columns="instrument")
+        .astype(float)
+        .sort_index()
+    )
+    if prices_df.empty:
+        raise InvalidAnalyticPortfolio()
+    ts = pd.bdate_range(prices_df.index.min(), prices_df.index.max(), freq="B")
+    prices_df = prices_df.reindex(ts)
+    if ffill_returns:
+        prices_df = prices_df.ffill()
+    prices_df.index = pd.to_datetime(prices_df.index)
+    returns = prices_to_returns(prices_df, drop_inceptions_nan=False, fill_nan=ffill_returns)
+    return returns.replace([np.inf, -np.inf, np.nan], 0)
 
 
 class DefaultPortfolioQueryset(QuerySet):
@@ -185,7 +190,7 @@ class ActiveTrackedPortfolioManager(DefaultPortfolioManager):
             super()
             .get_queryset()
             .annotate(asset_exists=Exists(AssetPosition.unannotated_objects.filter(portfolio=OuterRef("pk"))))
-            .filter(asset_exists=True, is_tracked=True)
+            .filter(Q(asset_exists=True) & (Q(is_tracked=True) | Q(is_manageable=True)))
         )
 
 
@@ -337,7 +342,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
 
     @property
     def can_be_rebalanced(self):
-        return self.is_tracked and self.is_manageable and not self.is_lookthrough
+        return self.is_manageable and not self.is_lookthrough
 
     def delete(self, **kwargs):
         super().delete(**kwargs)
@@ -374,7 +379,9 @@ class Portfolio(DeleteToDisableMixin, WBModel):
             )
         )
 
-    def get_analytic_portfolio(self, val_date: date, **kwargs) -> tuple[AnalyticPortfolio, pd.DataFrame]:
+    def get_analytic_portfolio(
+        self, val_date: date, weights: dict[int, float] | None = None, **kwargs
+    ) -> AnalyticPortfolio:
         """
         Return the analytic portfolio associated with this portfolio at the given date
 
@@ -385,18 +392,18 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         Returns:
             The instantiated analytic portfolio
         """
-        weights = self.get_weights(val_date)
+        if not weights:
+            weights = self.get_weights(val_date)
         return_date = (val_date + BDay(1)).date()
-        returns, prices = self.get_returns((val_date - BDay(2)).date(), return_date, **kwargs)
+        returns = get_returns(
+            list(weights.keys()), (val_date - BDay(2)).date(), return_date, to_currency=self.currency, **kwargs
+        )
         if pd.Timestamp(return_date) not in returns.index:
             raise InvalidAnalyticPortfolio()
         returns = returns.fillna(0)  # not sure this is what we want
-        return (
-            AnalyticPortfolio(
-                X=returns,
-                weights=weights,
-            ),
-            prices,
+        return AnalyticPortfolio(
+            X=returns,
+            weights=weights,
         )
 
     def is_invested_at_date(self, val_date: date) -> bool:
@@ -682,6 +689,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         force_recompute_weighting: bool = False,
         compute_metrics: bool = False,
         evaluate_rebalancer: bool = True,
+        changed_weights: dict[int, float] | None = None,
     ):
         logger.info(f"change at date for {self} at {val_date}")
 
@@ -709,7 +717,9 @@ class Portfolio(DeleteToDisableMixin, WBModel):
                         asset.save()
 
         # We check if there is an instrument attached to the portfolio with calculated NAV and price computation method
-        self.estimate_net_asset_values((val_date + BDay(1)).date())  # updating weighting in t0 influence nav in t+1
+        self.estimate_net_asset_values(
+            (val_date + BDay(1)).date(), weights=changed_weights
+        )  # updating weighting in t0 influence nav in t+1
         if evaluate_rebalancer:
             self.evaluate_rebalancing(val_date)
 
@@ -731,13 +741,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         for rel in PortfolioPortfolioThroughModel.objects.filter(
             dependency_portfolio=self, type=PortfolioPortfolioThroughModel.Type.PRIMARY, portfolio__is_lookthrough=True
         ):
-            lookthrough_portfolio = rel.portfolio
-            portfolio_total_asset_value = (
-                self.get_total_asset_under_management(val_date) if not lookthrough_portfolio.only_weighting else None
-            )
-            lookthrough_portfolio.compute_lookthrough(
-                val_date, portfolio_total_asset_value=portfolio_total_asset_value
-            )
+            rel.portfolio.compute_lookthrough(val_date)
         for rel in PortfolioPortfolioThroughModel.objects.filter(
             dependency_portfolio=self, type=PortfolioPortfolioThroughModel.Type.MODEL
         ):
@@ -757,14 +761,14 @@ class Portfolio(DeleteToDisableMixin, WBModel):
                 logger.info(f"Evaluate Rebalancing for {self} at {next_business_date}")
                 self.automatic_rebalancer.evaluate_rebalancing(next_business_date)
 
-    def estimate_net_asset_values(self, val_date: date):
+    def estimate_net_asset_values(self, val_date: date, weights: dict[int | float] | None = None):
         for instrument in self.pms_instruments:
             if instrument.is_active_at_date(val_date) and (
                 net_asset_value_computation_method_path := instrument.net_asset_value_computation_method_path
             ):
                 logger.info(f"Estimate NAV of {val_date:%Y-%m-%d} for instrument {instrument}")
                 net_asset_value_computation_method = import_from_dotted_path(net_asset_value_computation_method_path)
-                estimated_net_asset_value = net_asset_value_computation_method(val_date, instrument)
+                estimated_net_asset_value = net_asset_value_computation_method(val_date, instrument, weights=weights)
                 if estimated_net_asset_value is not None:
                     InstrumentPrice.objects.update_or_create(
                         instrument=instrument,
@@ -780,40 +784,60 @@ class Portfolio(DeleteToDisableMixin, WBModel):
                     ):  # if price date is the latest instrument price date, we recompute the last valuation data
                         instrument.update_last_valuation_date()
 
-    def batch_portfolio(self, start_date: date, end_date: date):
-        """
-        Create the cumulative portfolios between the two given dates and stop at the first rebalancing (if any)
-
-        Returns: The trade proposal generated by the rebalancing, if any (otherwise None)
-        """
+    def drift_weights(self, start_date: date, end_date: date) -> tuple[AssetPositionIterator, "TradeProposal"]:
+        logger.info(f"drift weights for {self} from {start_date:%Y-%m-%d} to {end_date:%Y-%m-%d}")
         rebalancer = getattr(self, "automatic_rebalancer", None)
-
+        # Get initial weights
         weights = self.get_weights(start_date)  # initial weights
-        positions = dict()
-        next_trade_proposal = None
-        rebalancing_date = None
-        logger.info(f"compute next weights in batch for {self} from {start_date:%Y-%m-%d} to {end_date:%Y-%m-%d}")
-        returns, prices = self.get_returns((start_date - BDay(3)).date(), end_date, ffill_returns=True)
+        if not weights:
+            previous_date = self.assets.filter(date__lte=start_date).latest("date").date
+            drifted_positions, _ = self.drift_weights(previous_date, start_date)
+            weights = drifted_positions.get_weights()[start_date]
+
+        # Get returns and prices data for the whole date range
+        instrument_ids = list(weights.keys())
+        returns = get_returns(
+            instrument_ids,
+            (start_date - BDay(3)).date(),
+            end_date,
+            to_currency=self.currency,
+            ffill_returns=True,
+        )
+        # Get raw prices to speed up asset position creation
+        prices = get_prices(instrument_ids, (start_date - BDay(3)).date(), end_date)
+        # Instantiate the position iterator with the initial weights
+        positions = AssetPositionIterator(self, prices=prices)
+        last_trade_proposal = None
         for to_date_ts in pd.date_range(start_date + timedelta(days=1), end_date, freq="B"):
             to_date = to_date_ts.date()
+            to_is_active = self.is_active_at_date(to_date)
             logger.info(f"Processing {to_date:%Y-%m-%d}")
             if rebalancer and rebalancer.is_valid(to_date):
-                rebalancing_date = to_date
-                break
-            try:
-                last_returns = returns.loc[[to_date_ts], :]
-                analytic_portfolio = AnalyticPortfolio(weights=weights, X=last_returns)
-                weights = analytic_portfolio.get_next_weights()
-                positions[to_date] = weights
-            except KeyError:  # if no return for that date, we break and continue
+                last_trade_proposal = rebalancer.evaluate_rebalancing(to_date)
+                # if trade proposal/rebalancing is not approved, we cannot continue the drift
+                if last_trade_proposal.status != last_trade_proposal.Status.APPROVED:
+                    break
+                target_portfolio = last_trade_proposal._build_dto().convert_to_portfolio()
+                next_weights = {
+                    underlying_quote_id: float(pos.weighting)
+                    for underlying_quote_id, pos in target_portfolio.positions_map.items()
+                }
+            else:
+                try:
+                    last_returns = returns.loc[[to_date_ts], :]
+                    analytic_portfolio = AnalyticPortfolio(weights=weights, X=last_returns)
+                    next_weights = analytic_portfolio.get_next_weights()
+                except KeyError:  # if no return for that date, we break and continue
+                    next_weights = weights
+            if to_is_active:
+                positions.add((to_date, next_weights))
+            else:
+                positions.add(
+                    (to_date, {underlying_quote_id: 0.0 for underlying_quote_id in weights.keys()})
+                )  # if we have no return or portfolio is not active anymore, we return an emptied portfolio
                 break
-        positions_generator = PositionDictConverter(self, prices)
-        positions = list(positions_generator.convert(positions))
-        self.bulk_create_positions(positions, delete_leftovers=True, compute_metrics=False, evaluate_rebalancer=False)
-        if rebalancing_date:
-            next_trade_proposal = rebalancer.evaluate_rebalancing(rebalancing_date)
-
-        return next_trade_proposal
+            weights = next_weights
+        return positions, last_trade_proposal
 
     def propagate_or_update_assets(self, from_date: date, to_date: date):
         """
@@ -824,33 +848,13 @@ class Portfolio(DeleteToDisableMixin, WBModel):
             to_date:  The date to create the new portfolio at
 
         """
-        from_is_active = self.is_active_at_date(from_date)
-        to_is_active = self.is_active_at_date(to_date)
-
-        def _parse_position(asset: AssetPosition) -> AssetPosition:
-            # function to handle the position modification after instantiation
-            if from_is_active and not to_is_active:
-                asset.weighting = Decimal(0.0)
-            asset.initial_shares = AssetPosition.unannotated_objects.filter(
-                date=from_date, underlying_quote=asset.underlying_quote, portfolio=self
-            ).aggregate(sum_shares=Sum("initial_shares"))["sum_shares"]
-            return asset
-
         # we don't propagate on already imported portfolio by default
         is_target_portfolio_imported = self.assets.filter(date=to_date, is_estimated=False).exists()
         if (
-            self.is_tracked and not self.is_lookthrough and not is_target_portfolio_imported and from_is_active
+            not self.is_lookthrough and not is_target_portfolio_imported and self.is_active_at_date(from_date)
         ):  # we cannot propagate a new portfolio for untracked, or look-through or already imported or inactive portfolios
-            logger.info(f"computing next weight for {self} from {from_date:%Y-%m-%d} to {to_date:%Y-%m-%d}")
-            with suppress(InvalidAnalyticPortfolio):
-                analytic_portfolio, to_prices = self.get_analytic_portfolio(from_date)
-                if not to_prices.empty:
-                    weights = analytic_portfolio.get_next_weights()
-                    positions_generator = PositionDictConverter(
-                        self, to_prices, infer_underlying_quote_price=True
-                    ).convert({to_date: weights})
-                    positions = list(map(lambda a: _parse_position(a), positions_generator))
-                    self.bulk_create_positions(positions, delete_leftovers=True, compute_metrics=True)
+            positions, _ = self.drift_weights(from_date, to_date)
+            self.bulk_create_positions(positions, delete_leftovers=True, compute_metrics=True)
 
     def get_lookthrough_positions(
         self,
@@ -896,10 +900,8 @@ class Portfolio(DeleteToDisableMixin, WBModel):
 
                 setattr(position, "path", path)
                 position.initial_shares = None
-                if portfolio_total_asset_value:
-                    position.initial_shares = (position.weighting * portfolio_total_asset_value) / (
-                        position.price * position.currency_fx_rate
-                    )
+                if portfolio_total_asset_value and (price_fx_portfolio := position.price * position.currency_fx_rate):
+                    position.initial_shares = (position.weighting * portfolio_total_asset_value) / price_fx_portfolio
                 if child_portfolio := position.underlying_quote.primary_portfolio:
                     if with_intermediary_position:
                         yield position
@@ -922,27 +924,23 @@ class Portfolio(DeleteToDisableMixin, WBModel):
             assets = self.assets.filter(date=val_date)
         return assets
 
-    def compute_lookthrough(self, sync_date: date, portfolio_total_asset_value: Decimal | None = None):
-        if not self.primary_portfolio or not self.is_lookthrough:
-            raise ValueError(
-                "Lookthrough position can only be computed on lookthrough portfolio with a primary portfolio"
-            )
-        logger.info(f"Compute Look-Through for {self} at {sync_date}")
-        positions = self.primary_portfolio.get_lookthrough_positions(sync_date, portfolio_total_asset_value)
-        self.bulk_create_positions(list(positions), delete_leftovers=True, compute_metrics=True)
-
-    def batch_recompute_lookthrough(self, from_date: date, to_date: date):
+    def compute_lookthrough(self, from_date: date, to_date: date | None = None):
         if not self.primary_portfolio or not self.is_lookthrough:
             raise ValueError(
                 "Lookthrough position can only be computed on lookthrough portfolio with a primary portfolio"
             )
-        positions = []
-        for val_date in pd.date_range(from_date, to_date, freq="B").date:
+        positions = AssetPositionIterator(self)
+        if not to_date:
+            to_date = from_date
+        for from_date in pd.date_range(from_date, to_date, freq="B").date:
+            logger.info(f"Compute Look-Through for {self} at {from_date}")
             portfolio_total_asset_value = (
-                self.primary_portfolio.get_total_asset_under_management(val_date) if not self.only_weighting else None
+                self.primary_portfolio.get_total_asset_under_management(from_date) if not self.only_weighting else None
             )
-            positions.extend(self.primary_portfolio.get_lookthrough_positions(val_date, portfolio_total_asset_value))
-        self.bulk_create_positions(list(positions), delete_leftovers=True, compute_metrics=False)
+            positions.add(
+                list(self.primary_portfolio.get_lookthrough_positions(from_date, portfolio_total_asset_value)),
+            )
+        self.bulk_create_positions(positions, delete_leftovers=True, compute_metrics=True)
 
     def update_preferred_classification_per_instrument(self):
         # Function to automatically assign Preferred instrument based on the assets' underlying instruments of the
@@ -999,47 +997,50 @@ class Portfolio(DeleteToDisableMixin, WBModel):
     def get_representation_label_key(cls):
         return "{{name}}"
 
-    def bulk_create_positions(self, positions: list[AssetPosition], delete_leftovers: bool = False, **kwargs):
+    def bulk_create_positions(
+        self, positions: AssetPositionIterator, delete_leftovers: bool = False, force_save: bool = False, **kwargs
+    ):
         if positions:
-            update_dates = set()
-            for position in positions:
-                position.portfolio = self
-                update_dates.add(position.date)
-
             # we need to delete the existing estimated portfolio because otherwise we risk to have existing and not
             # overlapping positions remaining (as they will not be updating by the bulk create). E.g. when someone
             # change completely the trades of a portfolio model and drift it.
-            self.assets.filter(date__in=update_dates, is_estimated=True).delete()
-            leftover_positions_ids = list(
-                self.assets.filter(date__in=update_dates).values_list("id", flat=True)
-            )  # we need to get the ids otherwise the queryset is reevaluated later
-            logger.info(f"bulk saving {len(positions)} positions ({len(leftover_positions_ids)} leftovers) ...")
-            objs = AssetPosition.unannotated_objects.bulk_create(
-                positions,
-                update_fields=[
-                    "weighting",
-                    "initial_price",
-                    "initial_currency_fx_rate",
-                    "initial_shares",
-                    "currency_fx_rate_instrument_to_usd",
-                    "currency_fx_rate_portfolio_to_usd",
-                    "underlying_quote_price",
-                    "is_estimated",
-                    "portfolio",
-                    "portfolio_created",
-                    "underlying_instrument",
-                ],
-                unique_fields=["portfolio", "date", "underlying_quote", "portfolio_created"],
-                update_conflicts=True,
-                batch_size=10000,
-            )
-            if delete_leftovers:
-                objs_ids = list(map(lambda x: x.id, objs))
-                leftover_positions_ids = list(filter(lambda i: i not in objs_ids, leftover_positions_ids))
-                logger.info(f"deleting {len(leftover_positions_ids)} leftover positions..")
-                AssetPosition.objects.filter(id__in=leftover_positions_ids).delete()
-            for update_date in sorted(update_dates):
-                self.change_at_date(update_date, **kwargs)
+
+            dates = positions.get_dates()
+            self.assets.filter(date__in=dates, is_estimated=True).delete()
+
+            if self.is_tracked or force_save:  # if the portfolio is not "tracked", we do no drift weights
+                leftover_positions_ids = list(
+                    self.assets.filter(date__in=dates).values_list("id", flat=True)
+                )  # we need to get the ids otherwise the queryset is reevaluated later
+                positions_list = list(positions)
+                logger.info(
+                    f"bulk saving {len(positions_list)} positions ({len(leftover_positions_ids)} leftovers) ..."
+                )
+                objs = AssetPosition.unannotated_objects.bulk_create(
+                    positions_list,
+                    update_fields=[
+                        "weighting",
+                        "initial_price",
+                        "initial_currency_fx_rate",
+                        "initial_shares",
+                        "currency_fx_rate_instrument_to_usd",
+                        "currency_fx_rate_portfolio_to_usd",
+                        "underlying_quote_price",
+                        "portfolio",
+                        "portfolio_created",
+                        "underlying_instrument",
+                    ],
+                    unique_fields=["portfolio", "date", "underlying_quote", "portfolio_created"],
+                    update_conflicts=True,
+                    batch_size=10000,
+                )
+                if delete_leftovers:
+                    objs_ids = list(map(lambda x: x.id, objs))
+                    leftover_positions_ids = list(filter(lambda i: i not in objs_ids, leftover_positions_ids))
+                    logger.info(f"deleting {len(leftover_positions_ids)} leftover positions..")
+                    AssetPosition.objects.filter(id__in=leftover_positions_ids).delete()
+            for update_date, changed_weights in positions.get_weights().items():
+                self.change_at_date(update_date, changed_weights=changed_weights, **kwargs)
 
     @classmethod
     def _get_or_create_portfolio(cls, instrument_handler, portfolio_data):
@@ -1053,54 +1054,6 @@ class Portfolio(DeleteToDisableMixin, WBModel):
     def check_share_diff(self, val_date: date) -> bool:
         return self.assets.filter(Q(date=val_date) & ~Q(initial_shares=F("initial_shares_at_custodian"))).exists()
 
-    def get_returns(
-        self,
-        from_date: date,
-        to_date: date,
-        ffill_returns: bool = True,
-        convert_to_portfolio_currency: bool = True,
-    ) -> tuple[pd.DataFrame, pd.DataFrame]:
-        """
-        Utility methods to get instrument returns for a given date range
-
-        Args:
-            from_date: date range lower bound
-            to_date: date range upper bound
-
-        Returns:
-            Return a tuple of the returns and the last prices series for conveniance
-        """
-        instruments = list(
-            self.assets.filter(date__gte=from_date, date__lte=to_date)
-            .values_list("underlying_quote", flat=True)
-            .distinct("underlying_quote")
-        )
-        prices = InstrumentPrice.objects.filter(
-            instrument__in=instruments, date__gte=from_date, date__lte=to_date
-        ).annotate(
-            fx_rate=CurrencyFXRates.get_fx_rates_subquery_for_two_currencies(
-                "date", "instrument__currency", self.currency
-            ),
-            price_fx_portfolio=F("net_value") * F("fx_rate") if convert_to_portfolio_currency else F("net_value"),
-        )
-        prices_df = (
-            pd.DataFrame(
-                prices.filter_only_valid_prices().values_list("instrument", "price_fx_portfolio", "date"),
-                columns=["instrument", "price_fx_portfolio", "date"],
-            )
-            .pivot_table(index="date", values="price_fx_portfolio", columns="instrument")
-            .astype(float)
-            .sort_index()
-        )
-        if prices_df.empty:
-            raise InvalidAnalyticPortfolio()
-        ts = pd.bdate_range(prices_df.index.min(), prices_df.index.max(), freq="B")
-        prices_df = prices_df.reindex(ts)
-        if ffill_returns:
-            prices_df = prices_df.ffill()
-        returns = prices_to_returns(prices_df, drop_inceptions_nan=False, fill_nan=ffill_returns)
-        return returns.replace([np.inf, -np.inf, np.nan], 0), prices_df.replace([np.inf, -np.inf, np.nan], None)
-
     @classmethod
     def get_contribution_df(cls, data, need_normalize: bool = False):
         df = pd.DataFrame(
@@ -1241,20 +1194,19 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         return portfolio
 
 
-def default_estimate_net_value(val_date: date, instrument: Instrument) -> float | None:
-    portfolio = instrument.portfolio
+def default_estimate_net_value(
+    val_date: date, instrument: Instrument, weights: dict[int, float] | None = None
+) -> float | None:
+    portfolio: Portfolio = instrument.portfolio
     previous_val_date = (val_date - BDay(1)).date()
-
-    if (
-        portfolio.assets.filter(date=previous_val_date).exists()
-        and (  # we assume that in t-1 we will have a portfolio (with at least estimate position). If we use the latest position date before val_date, we run into the problem of being able to compute nav at every date
-            last_price := instrument.get_latest_price(previous_val_date)
-        )
-    ):
+    if not weights:
+        weights = portfolio.get_weights(previous_val_date)
+    # we assume that in t-1 we will have a portfolio (with at least estimate position). If we use the latest position date before val_date, we run into the problem of being able to compute nav at every date
+    if weights and (last_price := instrument.get_latest_price(previous_val_date)):
         with suppress(
             IndexError, InvalidAnalyticPortfolio
         ):  # we silent any indexerror introduced by no returns for the past days
-            analytic_portfolio, _ = portfolio.get_analytic_portfolio(previous_val_date)
+            analytic_portfolio = portfolio.get_analytic_portfolio(previous_val_date, weights=weights)
             return analytic_portfolio.get_estimate_net_value(float(last_price.net_value))
 
 
@@ -1268,9 +1220,7 @@ def post_portfolio_relationship_creation(sender, instance, created, raw, **kwarg
     ):
         with suppress(AssetPosition.DoesNotExist):
             earliest_primary_position_date = instance.dependency_portfolio.assets.earliest("date").date
-            batch_recompute_lookthrough_as_task.delay(
-                instance.portfolio.id, earliest_primary_position_date, date.today()
-            )
+            compute_lookthrough_as_task.delay(instance.portfolio.id, earliest_primary_position_date, date.today())
 
 
 @shared_task(queue="portfolio")
@@ -1280,9 +1230,9 @@ def trigger_portfolio_change_as_task(portfolio_id, val_date, **kwargs):
 
 
 @shared_task(queue="portfolio")
-def batch_recompute_lookthrough_as_task(portfolio_id: int, start: date, end: date):
+def compute_lookthrough_as_task(portfolio_id: int, start: date, end: date):
     portfolio = Portfolio.objects.get(id=portfolio_id)
-    portfolio.batch_recompute_lookthrough(start, end)
+    portfolio.compute_lookthrough(start, to_date=end)
 
 
 @receiver(investable_universe_updated, sender="wbfdm.Instrument")