wbportfolio 1.52.0__py2.py3-none-any.whl → 1.59.4__py2.py3-none-any.whl

wbportfolio/models/portfolio.py

@@ -1,42 +1,36 @@
 import logging
+from collections import defaultdict
 from contextlib import suppress
 from datetime import date, timedelta
 from decimal import Decimal
-from math import isclose
-from typing import TYPE_CHECKING, Any, Iterable
+from typing import TYPE_CHECKING, Any, Generator, Iterable
 
 import numpy as np
 import pandas as pd
 from celery import shared_task
-from django.contrib.contenttypes.models import ContentType
 from django.contrib.postgres.fields import DateRangeField
+from django.core.exceptions import ObjectDoesNotExist
+from django.core.validators import MaxValueValidator, MinValueValidator
 from django.db import models
-from django.db.models import (
-    Exists,
-    F,
-    OuterRef,
-    Q,
-    QuerySet,
-    Sum,
-    Value,
-)
+from django.db.models import Exists, F, OuterRef, Q, QuerySet, Sum
 from django.db.models.signals import post_save
 from django.dispatch import receiver
 from django.utils import timezone
 from django.utils.functional import cached_property
 from pandas._libs.tslibs.offsets import BDay
-from skfolio.preprocessing import prices_to_returns
-from wbcore.contrib.currency.models import Currency, CurrencyFXRates
+from wbcore.contrib.authentication.models import User
+from wbcore.contrib.currency.models import Currency
+from wbcore.contrib.notifications.dispatch import send_notification
 from wbcore.contrib.notifications.utils import create_notification_type
 from wbcore.models import WBModel
 from wbcore.utils.importlib import import_from_dotted_path
 from wbcore.utils.models import ActiveObjectManager, DeleteToDisableMixin
-from wbfdm.contrib.metric.tasks import compute_metrics_as_task
-from wbfdm.models import Instrument, InstrumentType
+from wbfdm.models import Cash, Instrument, InstrumentType
 from wbfdm.models.instruments.instrument_prices import InstrumentPrice
 from wbfdm.signals import investable_universe_updated
 
-from wbportfolio.models.asset import AssetPosition, AssetPositionIterator
+from wbportfolio.models.asset import AssetPosition
+from wbportfolio.models.builder import AssetPositionBuilder
 from wbportfolio.models.indexes import Index
 from wbportfolio.models.portfolio_relationship import (
     InstrumentPortfolioThroughModel,
@@ -45,78 +39,17 @@ from wbportfolio.models.portfolio_relationship import (
 from wbportfolio.models.products import Product
 from wbportfolio.pms.analytics.portfolio import Portfolio as AnalyticPortfolio
 from wbportfolio.pms.typing import Portfolio as PortfolioDTO
+from wbportfolio.pms.typing import Position as PositionDTO
 
-from . import ProductGroup
-from .exceptions import InvalidAnalyticPortfolio
+from ..constants import EQUITY_TYPE_KEYS
+from ..order_routing.adapters import BaseCustodianAdapter
+from . import PortfolioRole, ProductGroup
 
 logger = logging.getLogger("pms")
 if TYPE_CHECKING:
-    from wbportfolio.models.transactions.trade_proposals import TradeProposal
-
-
-def get_prices(instrument_ids: list[int], from_date: date, to_date: date) -> dict[date, dict[int, float]]:
-    """
-    Utility to fetch raw prices
-    """
-    prices = InstrumentPrice.objects.filter(instrument__in=instrument_ids, date__gte=from_date, date__lte=to_date)
-    df = (
-        pd.DataFrame(
-            prices.filter_only_valid_prices().values_list("instrument", "net_value", "date"),
-            columns=["instrument", "net_value", "date"],
-        )
-        .pivot_table(index="date", values="net_value", columns="instrument")
-        .astype(float)
-        .sort_index()
-    )
-    ts = pd.bdate_range(df.index.min(), df.index.max(), freq="B")
-    df = df.reindex(ts)
-    df = df.ffill()
-    df.index = pd.to_datetime(df.index)
-    return {ts.date(): row for ts, row in df.to_dict("index").items()}
-
-
-def get_returns(
-    instrument_ids: list[int],
-    from_date: date,
-    to_date: date,
-    to_currency: Currency | None = None,
-    ffill_returns: bool = True,
-) -> pd.DataFrame:
-    """
-    Utility methods to get instrument returns for a given date range
-
-    Args:
-        from_date: date range lower bound
-        to_date: date range upper bound
-
-    Returns:
-        Return a tuple of the returns and the last prices series for conveniance
-    """
-    if to_currency:
-        fx_rate = CurrencyFXRates.get_fx_rates_subquery_for_two_currencies("date", "instrument__currency", to_currency)
-    else:
-        fx_rate = Value(Decimal(1.0))
-    prices = InstrumentPrice.objects.filter(
-        instrument__in=instrument_ids, date__gte=from_date, date__lte=to_date
-    ).annotate(fx_rate=fx_rate, price_fx_portfolio=F("net_value") * F("fx_rate"))
-    prices_df = (
-        pd.DataFrame(
-            prices.filter_only_valid_prices().values_list("instrument", "price_fx_portfolio", "date"),
-            columns=["instrument", "price_fx_portfolio", "date"],
-        )
-        .pivot_table(index="date", values="price_fx_portfolio", columns="instrument")
-        .astype(float)
-        .sort_index()
-    )
-    if prices_df.empty:
-        raise InvalidAnalyticPortfolio()
-    ts = pd.bdate_range(prices_df.index.min(), prices_df.index.max(), freq="B")
-    prices_df = prices_df.reindex(ts)
-    if ffill_returns:
-        prices_df = prices_df.ffill()
-    prices_df.index = pd.to_datetime(prices_df.index)
-    returns = prices_to_returns(prices_df, drop_inceptions_nan=False, fill_nan=ffill_returns)
-    return returns.replace([np.inf, -np.inf, np.nan], 0)
+    pass
+
+MARKET_HOLIDAY_MAX_DURATION = 15
 
 
 class DefaultPortfolioQueryset(QuerySet):
@@ -142,7 +75,7 @@ class DefaultPortfolioQueryset(QuerySet):
         """
         A method to sort the given queryset to return undependable portfolio first. This is very useful if a routine needs to be applied sequentially on portfolios by order of dependence.
         """
-        MAX_ITERATIONS: int = (
+        max_iterations: int = (
             5  # in order to avoid circular dependency and infinite loop, we need to stop recursion at a max depth
         )
         remaining_portfolios = set(self)
@@ -155,7 +88,7 @@ class DefaultPortfolioQueryset(QuerySet):
             dependency_relationships = PortfolioPortfolioThroughModel.objects.filter(
                 portfolio=p, dependency_portfolio__in=remaining_portfolios
             )  # get dependency portfolios
-            if iterator_counter >= MAX_ITERATIONS or (
+            if iterator_counter >= max_iterations or (
                 not dependency_relationships.exists() and not bool(parent_portfolios)
             ):  # if not dependency portfolio or parent portfolio that remained, then we yield
                 remaining_portfolios.remove(p)
@@ -196,28 +129,32 @@ class ActiveTrackedPortfolioManager(DefaultPortfolioManager):
 
 class PortfolioPortfolioThroughModel(models.Model):
     class Type(models.TextChoices):
-        PRIMARY = "PRIMARY", "Primary"
+        LOOK_THROUGH = "LOOK_THROUGH", "Look-through"
         MODEL = "MODEL", "Model"
         CUSTODIAN = "CUSTODIAN", "Custodian"
+        HIERARCHICAL = "HIERARCHICAL", "Hierarchical"
 
     portfolio = models.ForeignKey("wbportfolio.Portfolio", on_delete=models.CASCADE, related_name="dependency_through")
     dependency_portfolio = models.ForeignKey(
         "wbportfolio.Portfolio", on_delete=models.CASCADE, related_name="dependent_through"
     )
-    type = models.CharField(choices=Type.choices, default=Type.PRIMARY, verbose_name="Type")
+    type = models.CharField(choices=Type.choices, default=Type.LOOK_THROUGH, verbose_name="Type")
 
     def __str__(self):
         return f"{self.portfolio} dependant on {self.dependency_portfolio} ({self.Type[self.type].label})"
 
     class Meta:
         constraints = [
-            models.UniqueConstraint(fields=["portfolio", "type"], name="unique_primary", condition=Q(type="PRIMARY")),
+            models.UniqueConstraint(
+                fields=["portfolio", "type"], name="unique_lookthrough", condition=Q(type="LOOK_THROUGH")
+            ),
             models.UniqueConstraint(fields=["portfolio", "type"], name="unique_model", condition=Q(type="MODEL")),
         ]
 
 
 class Portfolio(DeleteToDisableMixin, WBModel):
     assets: models.QuerySet[AssetPosition]
+    builder: AssetPositionBuilder
 
     name = models.CharField(
         max_length=255,
@@ -275,7 +212,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
 
     is_manageable = models.BooleanField(
         default=False,
-        help_text="True if the portfolio can be manually modified (e.g. Trade proposal be submitted or total weight recomputed)",
+        help_text="True if the portfolio can be manually modified (e.g. Order Proposal be submitted or total weight recomputed)",
     )
     is_tracked = models.BooleanField(
         default=True,
@@ -303,14 +240,37 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         blank=True,
     )
 
+    # OMS default parameters. Used to seed order proposal default value upon creation
+    default_order_proposal_min_order_value = models.IntegerField(
+        default=0, verbose_name="Default Order Proposal Minimum Order Value"
+    )
+    default_order_proposal_min_weighting = models.DecimalField(
+        max_digits=9,
+        decimal_places=8,
+        default=Decimal(0),
+        verbose_name="Default Order Proposal Minimum Weight",
+        validators=[MinValueValidator(Decimal("0")), MaxValueValidator(Decimal("1"))],
+    )
+    default_order_proposal_total_cash_weight = models.DecimalField(
+        default=Decimal("0"),
+        decimal_places=4,
+        max_digits=5,
+        verbose_name="Default Order Proposal Total Cash Weight",
+        validators=[MinValueValidator(Decimal("0")), MaxValueValidator(Decimal("1"))],
+    )
+
     objects = DefaultPortfolioManager()
     tracked_objects = ActiveTrackedPortfolioManager()
 
+    def __init__(self, *args, **kwargs):
+        self.builder = AssetPositionBuilder(self)
+        super().__init__(*args, **kwargs)
+
     @property
     def primary_portfolio(self):
         with suppress(PortfolioPortfolioThroughModel.DoesNotExist):
             return PortfolioPortfolioThroughModel.objects.get(
-                portfolio=self, type=PortfolioPortfolioThroughModel.Type.PRIMARY
+                portfolio=self, type=PortfolioPortfolioThroughModel.Type.LOOK_THROUGH
             ).dependency_portfolio
 
     @property
@@ -329,6 +289,13 @@ class Portfolio(DeleteToDisableMixin, WBModel):
                 dependency_portfolio__is_composition=True,
             ).dependency_portfolio
 
+    @property
+    def is_model(self) -> bool:
+        return PortfolioPortfolioThroughModel.objects.filter(
+            type=PortfolioPortfolioThroughModel.Type.MODEL,
+            dependency_portfolio=self,
+        ).exists()
+
     @property
     def imported_assets(self):
         return self.assets.filter(is_estimated=False)
@@ -340,6 +307,30 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         instruments.extend([i for i in Index.objects.filter(portfolios=self)])
         return instruments
 
+    @property
+    def cash_component(self) -> Cash:
+        return Cash.objects.get_or_create(
+            currency=self.currency, defaults={"is_cash": True, "name": self.currency.title}
+        )[0]
+
+    def get_authenticated_custodian_adapter(self, **kwargs) -> BaseCustodianAdapter | None:
+        supported_instruments_for_routing = list(
+            filter(lambda o: o.order_routing_custodian_adapter, self.pms_instruments)
+        )
+        if not supported_instruments_for_routing:
+            raise ValueError("No custodian adapter for this portfolio")
+
+        pms_instrument = supported_instruments_for_routing[
+            0
+        ]  # for simplicity we support only one instrument per portfolio that is allowed to support order routing
+        adapter = import_from_dotted_path(pms_instrument.order_routing_custodian_adapter)(
+            isin=pms_instrument.isin, identifier=pms_instrument.identifier, **kwargs
+        )
+        adapter.authenticate()
+        if not adapter.is_valid():
+            raise ValueError("This portfolio is not valid for rebalancing")
+        return adapter
+
     @property
     def can_be_rebalanced(self):
         return self.is_manageable and not self.is_lookthrough
@@ -355,9 +346,17 @@ class Portfolio(DeleteToDisableMixin, WBModel):
 
     def _build_dto(self, val_date: date, **extra_kwargs) -> PortfolioDTO:
         "returns the dto representation of this portfolio at the specified date"
-        return PortfolioDTO(
-            tuple([pos._build_dto() for pos in self.assets.filter(date=val_date, **extra_kwargs)]),
-        )
+        assets = self.assets.filter(date=val_date, **extra_kwargs)
+        try:
+            last_returns, _ = self.get_analytic_portfolio(val_date, use_dl=True).get_contributions()
+            last_returns = last_returns.to_dict()
+        except ValueError:
+            last_returns = {}
+        positions = []
+        for asset in assets:
+            positions.append(asset._build_dto(daily_return=last_returns.get(asset.underlying_quote.id, Decimal("0"))))
+
+        return PortfolioDTO(positions)
 
     def get_weights(self, val_date: date) -> dict[int, float]:
         """
@@ -380,7 +379,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         )
 
     def get_analytic_portfolio(
-        self, val_date: date, weights: dict[int, float] | None = None, **kwargs
+        self, val_date: date, weights: dict[int, float] | None = None, use_dl: bool = True, **kwargs
     ) -> AnalyticPortfolio:
         """
         Return the analytic portfolio associated with this portfolio at the given date
@@ -395,11 +394,10 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         if not weights:
             weights = self.get_weights(val_date)
         return_date = (val_date + BDay(1)).date()
-        returns = get_returns(
-            list(weights.keys()), (val_date - BDay(2)).date(), return_date, to_currency=self.currency, **kwargs
-        )
+        returns = self.load_builder_returns(val_date, return_date, use_dl=use_dl).copy()
         if pd.Timestamp(return_date) not in returns.index:
-            raise InvalidAnalyticPortfolio()
+            raise ValueError()
+        returns = returns.loc[:return_date, :]
         returns = returns.fillna(0)  # not sure this is what we want
         return AnalyticPortfolio(
             X=returns,
@@ -430,12 +428,22 @@ class Portfolio(DeleteToDisableMixin, WBModel):
                 True,
             ),
             create_notification_type(
-                "wbportfolio.portfolio.replay_done",
-                "Portfolio Replay finished",
-                "Sends a notification when a the requested trade proposal replay is done",
+                "wbportfolio.portfolio.action_done",
+                "Portfolio Action finished",
+                "Sends a notification when a the requested portfolio action is done (e.g. replay, quote adjustment...)",
+                True,
+                True,
+                True,
+                is_lock=True,
+            ),
+            create_notification_type(
+                "wbportfolio.portfolio.warning",
+                "PMS Warning",
+                "Sends a notification to warn portfolio manager or administrator regarding issue that needs action.",
                 True,
                 True,
                 True,
+                is_lock=True,
             ),
         ]
 
@@ -565,7 +573,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
             val_date=val_date,
             exclude_cash=True,
             exclude_index=True,
-            extra_filter_parameters={"underlying_instrument__instrument_type": InstrumentType.EQUITY},
+            extra_filter_parameters={"underlying_instrument__instrument_type__key__in": EQUITY_TYPE_KEYS},
             **kwargs,
         )
         if not df.empty:
@@ -578,7 +586,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
             val_date=val_date,
             exclude_cash=True,
             exclude_index=True,
-            extra_filter_parameters={"underlying_instrument__instrument_type": InstrumentType.EQUITY},
+            extra_filter_parameters={"underlying_instrument__instrument_type__key__in": EQUITY_TYPE_KEYS},
             **kwargs,
         )
         if not df.empty:
@@ -650,7 +658,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
     ) -> pd.DataFrame:
         qs = self._get_assets(with_cash=with_cash).filter(date__gte=start, date__lte=end)
         if only_equity:
-            qs = qs.filter(underlying_instrument__instrument_type=InstrumentType.EQUITY)
+            qs = qs.filter(underlying_instrument__instrument_type__key__in=EQUITY_TYPE_KEYS)
         qs = qs.annotate_hedged_currency_fx_rate(hedged_currency)
         df = Portfolio.get_contribution_df(
             qs.select_related("underlying_instrument").values_list(
@@ -690,42 +698,49 @@ class Portfolio(DeleteToDisableMixin, WBModel):
             if portfolio := asset.underlying_instrument.portfolio:
                 yield portfolio, asset.weighting
 
+    def get_next_rebalancing_date(self, start_date: date) -> date | None:
+        if automatic_rebalancer := getattr(self, "automatic_rebalancer", None):
+            return automatic_rebalancer.get_next_rebalancing_date(start_date)
+
+    def fix_quantization(self, val_date: date):
+        assets = self.assets.filter(date=val_date)
+        total_weighting = assets.aggregate(s=Sum("weighting"))["s"]
+        if total_weighting and (quantization_error := Decimal("1") - total_weighting):
+            cash = self.cash_component
+            try:
+                cash_pos = assets.get(underlying_quote=cash)
+                cash_pos.weighting += quantization_error
+            except AssetPosition.DoesNotExist:
+                cash_pos = AssetPosition(
+                    portfolio=self,
+                    underlying_quote=cash,
+                    weighting=quantization_error,
+                    initial_price=Decimal("1"),
+                    date=val_date,
+                    is_estimated=True,
+                )
+            cash_pos.save(create_underlying_quote_price_if_missing=True)
+
     def change_at_date(
         self,
         val_date: date,
-        recompute_weighting: bool = False,
-        force_recompute_weighting: bool = False,
+        fix_quantization: bool = False,
         evaluate_rebalancer: bool = True,
-        changed_weights: dict[int, float] | None = None,
+        changed_portfolio: AnalyticPortfolio | None = None,
+        broadcast_changes_at_date: bool = True,
+        **kwargs,
     ):
+        if not self.is_tracked:
+            return
         logger.info(f"change at date for {self} at {val_date}")
 
-        if recompute_weighting:
-            # We normalize weight across the portfolio for a given date
-            qs = self.assets.filter(date=val_date).filter(
-                Q(total_value_fx_portfolio__isnull=False) | Q(weighting__isnull=False)
-            )
-            if (self.is_lookthrough or self.is_manageable or force_recompute_weighting) and qs.exists():
-                total_weighting = qs.aggregate(s=Sum("weighting"))["s"]
-                # We check if this actually necessary
-                # (i.e. if the weight is already summed to 100%, it is already normalized)
-                if (
-                    not total_weighting
-                    or not isclose(total_weighting, Decimal(1.0), abs_tol=0.001)
-                    or force_recompute_weighting
-                ):
-                    total_value = qs.aggregate(s=Sum("total_value_fx_portfolio"))["s"]
-                    # TODO we change this because postgres doesn't support join statement in update (and total_value_fx_portfolio is a joined annoted field)
-                    for asset in qs:
-                        if total_value:
-                            asset.weighting = asset._total_value_fx_portfolio / total_value
-                        elif total_weighting:
-                            asset.weighting = asset.weighting / total_weighting
-                        asset.save()
+        if fix_quantization:
+            # We assume all ptf total weight is 100% but quantization error can occur. In that case, we create a cash component and add the weight there.
+            self.fix_quantization(val_date)
 
         # We check if there is an instrument attached to the portfolio with calculated NAV and price computation method
         self.estimate_net_asset_values(
-            (val_date + BDay(1)).date(), weights=changed_weights
+            (val_date + BDay(1)).date(), analytic_portfolio=changed_portfolio
         )  # updating weighting in t0 influence nav in t+1
         if evaluate_rebalancer:
             self.evaluate_rebalancing(val_date)
@@ -737,25 +752,44 @@ class Portfolio(DeleteToDisableMixin, WBModel):
             if not self.initial_position_date or self.initial_position_date > val_date:
                 self.initial_position_date = val_date
         self.save()
+        if broadcast_changes_at_date:
+            self.handle_controlling_portfolio_change_at_date(
+                val_date,
+                fix_quantization=fix_quantization,
+                changed_portfolio=changed_portfolio,
+                **kwargs,
+            )
 
-        self.handle_controlling_portfolio_change_at_date(val_date)
-
-    def handle_controlling_portfolio_change_at_date(self, val_date: date):
-        for rel in PortfolioPortfolioThroughModel.objects.filter(
-            dependency_portfolio=self, type=PortfolioPortfolioThroughModel.Type.PRIMARY, portfolio__is_lookthrough=True
-        ):
-            rel.portfolio.compute_lookthrough(val_date)
+    def handle_controlling_portfolio_change_at_date(self, val_date: date, **kwargs):
+        if self.is_tracked:
+            for rel in PortfolioPortfolioThroughModel.objects.filter(
+                dependency_portfolio=self,
+                type=PortfolioPortfolioThroughModel.Type.LOOK_THROUGH,
+                portfolio__is_lookthrough=True,
+            ):
+                rel.portfolio.compute_lookthrough(val_date)
+            for rel in PortfolioPortfolioThroughModel.objects.filter(
+                dependency_portfolio=self, type=PortfolioPortfolioThroughModel.Type.MODEL
+            ):
+                rel.portfolio.evaluate_rebalancing(val_date)
+            for dependent_portfolio in self.get_child_portfolios(val_date):
+                # dependent_portfolio.change_at_date(val_date, **kwargs)
+                dependent_portfolio.handle_controlling_portfolio_change_at_date(val_date, **kwargs)
+
+    def get_model_portfolio_relationships(
+        self, val_date: date
+    ) -> Generator[PortfolioPortfolioThroughModel, None, None]:
         for rel in PortfolioPortfolioThroughModel.objects.filter(
             dependency_portfolio=self, type=PortfolioPortfolioThroughModel.Type.MODEL
         ):
-            rel.portfolio.evaluate_rebalancing(val_date)
+            if rel.portfolio.is_active_at_date(val_date):
+                yield rel
         for dependent_portfolio in self.get_child_portfolios(val_date):
-            dependent_portfolio.change_at_date(val_date)
-            dependent_portfolio.handle_controlling_portfolio_change_at_date(val_date)
+            yield from dependent_portfolio.get_model_portfolio_relationships(val_date)
 
     def evaluate_rebalancing(self, val_date: date):
         if hasattr(self, "automatic_rebalancer"):
-            # if the portfolio has an automatic rebalancer and the next business day is suitable with the rebalancer, we create a trade proposal automatically
+            # if the portfolio has an automatic rebalancer and the next business day is suitable with the rebalancer, we create a order proposal automatically
             next_business_date = (val_date + BDay(1)).date()
             if self.automatic_rebalancer.is_valid(val_date):  # we evaluate the rebalancer in t0 and t+1
                 logger.info(f"Evaluate Rebalancing for {self} at {val_date}")
@@ -764,83 +798,102 @@ class Portfolio(DeleteToDisableMixin, WBModel):
                 logger.info(f"Evaluate Rebalancing for {self} at {next_business_date}")
                 self.automatic_rebalancer.evaluate_rebalancing(next_business_date)
 
-    def estimate_net_asset_values(self, val_date: date, weights: dict[int | float] | None = None):
-        for instrument in self.pms_instruments:
-            if instrument.is_active_at_date(val_date) and (
-                net_asset_value_computation_method_path := instrument.net_asset_value_computation_method_path
-            ):
-                logger.info(f"Estimate NAV of {val_date:%Y-%m-%d} for instrument {instrument}")
-                net_asset_value_computation_method = import_from_dotted_path(net_asset_value_computation_method_path)
-                estimated_net_asset_value = net_asset_value_computation_method(val_date, instrument, weights=weights)
-                if estimated_net_asset_value is not None:
-                    InstrumentPrice.objects.update_or_create(
-                        instrument=instrument,
-                        date=val_date,
-                        calculated=True,
-                        defaults={
-                            "gross_value": estimated_net_asset_value,
-                            "net_value": estimated_net_asset_value,
-                        },
+    def estimate_net_asset_values(self, val_date: date, analytic_portfolio: AnalyticPortfolio | None = None):
+        effective_portfolio_date = (val_date - BDay(1)).date()
+        with suppress(ValueError):
+            if not analytic_portfolio:
+                analytic_portfolio = self.get_analytic_portfolio(effective_portfolio_date, use_dl=False)
+            for instrument in self.pms_instruments:
+                # we assume that in t-1 we will have a portfolio (with at least estimate position). If we use the latest position date before val_date, we run into the problem of being able to compute nav at every date
+                last_price = instrument.get_latest_price(effective_portfolio_date)
+                if (
+                    instrument.is_active_at_date(val_date)
+                    and (net_asset_value_computation_method_path := instrument.net_asset_value_computation_method_path)
+                    and last_price
+                ):
+                    logger.info(f"Estimate NAV of {val_date:%Y-%m-%d} for instrument {instrument}")
+                    net_asset_value_computation_method = import_from_dotted_path(
+                        net_asset_value_computation_method_path
                     )
-                    if (
-                        val_date == instrument.last_price_date
-                    ):  # if price date is the latest instrument price date, we recompute the last valuation data
-                        instrument.update_last_valuation_date()
-
-    def drift_weights(self, start_date: date, end_date: date) -> tuple[AssetPositionIterator, "TradeProposal"]:
+                    estimated_net_asset_value = net_asset_value_computation_method(last_price, analytic_portfolio)
+                    if estimated_net_asset_value is not None:
+                        InstrumentPrice.objects.update_or_create(
+                            instrument=instrument,
+                            date=val_date,
+                            calculated=True,
+                            defaults={
+                                "gross_value": estimated_net_asset_value,
+                                "net_value": estimated_net_asset_value,
+                            },
+                        )
+                        if (
+                            val_date == instrument.last_price_date
+                        ):  # if price date is the latest instrument price date, we recompute the last valuation data
+                            instrument.update_last_valuation_date()
+
+    def drift_weights(
+        self, start_date: date, end_date: date, stop_at_rebalancing: bool = False
+    ) -> Generator[tuple[date, dict[int, float]], None, models.Model]:
         logger.info(f"drift weights for {self} from {start_date:%Y-%m-%d} to {end_date:%Y-%m-%d}")
+
         rebalancer = getattr(self, "automatic_rebalancer", None)
         # Get initial weights
         weights = self.get_weights(start_date)  # initial weights
         if not weights:
             previous_date = self.assets.filter(date__lte=start_date).latest("date").date
-            drifted_positions, _ = self.drift_weights(previous_date, start_date)
-            weights = drifted_positions.get_weights()[start_date]
-
-        # Get returns and prices data for the whole date range
-        instrument_ids = list(weights.keys())
-        returns = get_returns(
-            instrument_ids,
-            (start_date - BDay(3)).date(),
-            end_date,
-            to_currency=self.currency,
-            ffill_returns=True,
-        )
-        # Get raw prices to speed up asset position creation
-        prices = get_prices(instrument_ids, (start_date - BDay(3)).date(), end_date)
-        # Instantiate the position iterator with the initial weights
-        positions = AssetPositionIterator(self, prices=prices)
-        last_trade_proposal = None
+            _, weights = next(self.drift_weights(previous_date, start_date))
+        last_order_proposal = None
         for to_date_ts in pd.date_range(start_date + timedelta(days=1), end_date, freq="B"):
             to_date = to_date_ts.date()
             to_is_active = self.is_active_at_date(to_date)
             logger.info(f"Processing {to_date:%Y-%m-%d}")
-            if rebalancer and rebalancer.is_valid(to_date):
-                last_trade_proposal = rebalancer.evaluate_rebalancing(to_date)
-                # if trade proposal/rebalancing is not approved, we cannot continue the drift
-                if last_trade_proposal.status != last_trade_proposal.Status.APPROVED:
+            order_proposal = None
+            try:
+                last_returns = self.builder.returns.loc[[to_date_ts], :]
+                analytic_portfolio = AnalyticPortfolio(weights=weights, X=last_returns)
+                drifted_weights = analytic_portfolio.get_next_weights()
+            except KeyError:  # if no return for that date, we break and continue
+                break
+            try:
+                order_proposal = self.order_proposals.get(
+                    trade_date=to_date, rebalancing_model__isnull=True, status="CONFIRMED"
+                )
+            except ObjectDoesNotExist:
+                if rebalancer and rebalancer.is_valid(to_date):
+                    rebalancer.portfolio = self  # ensure reference is the same to access cached returns
+                    effective_portfolio = PortfolioDTO(
+                        positions=[
+                            PositionDTO(
+                                date=to_date,
+                                underlying_instrument=i,
+                                weighting=Decimal(w),
+                                daily_return=Decimal(last_returns.iloc[-1][i]),
+                            )
+                            for i, w in weights.items()
+                        ]
+                    )
+                    order_proposal = rebalancer.evaluate_rebalancing(to_date, effective_portfolio=effective_portfolio)
+            if order_proposal:
+                last_order_proposal = order_proposal
+                if stop_at_rebalancing:
                     break
-                target_portfolio = last_trade_proposal._build_dto().convert_to_portfolio()
                 next_weights = {
-                    underlying_quote_id: float(pos.weighting)
-                    for underlying_quote_id, pos in target_portfolio.positions_map.items()
+                    trade.underlying_instrument.id: float(trade._target_weight)
+                    for trade in order_proposal.get_orders()
                 }
+                yield to_date, next_weights
             else:
-                try:
-                    last_returns = returns.loc[[to_date_ts], :]
-                    analytic_portfolio = AnalyticPortfolio(weights=weights, X=last_returns)
-                    next_weights = analytic_portfolio.get_next_weights()
-                except KeyError:  # if no return for that date, we break and continue
-                    next_weights = weights
-            if to_is_active:
-                positions.add((to_date, next_weights))
-            else:
-                positions.add(
-                    (to_date, {underlying_quote_id: 0.0 for underlying_quote_id in weights.keys()})
-                )  # if we have no return or portfolio is not active anymore, we return an emptied portfolio
-                break
+                next_weights = drifted_weights
+                if to_is_active:
+                    yield to_date, next_weights
+                else:
+                    yield (
+                        to_date,
+                        {underlying_quote_id: 0.0 for underlying_quote_id in weights.keys()},
+                    )  # if we have no return or portfolio is not active anymore, we return an emptied portfolio
+                    break
             weights = next_weights
-        return positions, last_trade_proposal
+        return last_order_proposal
 
     def propagate_or_update_assets(self, from_date: date, to_date: date):
         """
@@ -856,10 +909,21 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         if (
             not self.is_lookthrough and not is_target_portfolio_imported and self.is_active_at_date(from_date)
         ):  # we cannot propagate a new portfolio for untracked, or look-through or already imported or inactive portfolios
-            positions, _ = self.drift_weights(from_date, to_date)
-            self.bulk_create_positions(
-                positions, delete_leftovers=True, compute_metrics=True, evaluate_rebalancer=False
-            )
+            self.load_builder_returns(from_date, to_date)
+            for pos_date, weights in self.drift_weights(from_date, to_date):
+                self.builder.add((pos_date, weights))
+            self.builder.bulk_create_positions(delete_leftovers=True)
+            self.builder.schedule_change_at_dates()
+            self.builder.schedule_metric_computation()
+
+    def load_builder_returns(self, from_date: date, to_date: date, use_dl: bool = True) -> pd.DataFrame:
+        instruments_ids = list(self.get_weights(from_date).keys())
+        for tp in self.order_proposals.filter(trade_date__gte=from_date, trade_date__lte=to_date):
+            instruments_ids.extend(tp.orders.values_list("underlying_instrument", flat=True))
+        self.builder.load_returns(
+            set(instruments_ids), (from_date - BDay(1)).date(), (to_date + BDay(1)).date(), use_dl=use_dl
+        )
+        return self.builder.returns
 
     def get_lookthrough_positions(
         self,
@@ -903,7 +967,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
                 except IndexError:
                     position.portfolio_created = None
 
-                setattr(position, "path", path)
+                position.path = path
                 position.initial_shares = None
                 if portfolio_total_asset_value and (price_fx_portfolio := position.price * position.currency_fx_rate):
                     position.initial_shares = (position.weighting * portfolio_total_asset_value) / price_fx_portfolio
@@ -926,7 +990,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         if self.is_composition:
             assets = list(self.get_lookthrough_positions(val_date, **kwargs))
         else:
-            assets = self.assets.filter(date=val_date)
+            assets = list(self.assets.filter(date=val_date))
         return assets
 
     def compute_lookthrough(self, from_date: date, to_date: date | None = None):
@@ -934,18 +998,20 @@ class Portfolio(DeleteToDisableMixin, WBModel):
             raise ValueError(
                 "Lookthrough position can only be computed on lookthrough portfolio with a primary portfolio"
             )
-        positions = AssetPositionIterator(self)
         if not to_date:
             to_date = from_date
-        for from_date in pd.date_range(from_date, to_date, freq="B").date:
-            logger.info(f"Compute Look-Through for {self} at {from_date}")
+        for val_date in pd.date_range(from_date, to_date, freq="B").date:
+            logger.info(f"Compute Look-Through for {self} at {val_date}")
             portfolio_total_asset_value = (
-                self.primary_portfolio.get_total_asset_under_management(from_date) if not self.only_weighting else None
+                self.primary_portfolio.get_total_asset_under_management(val_date) if not self.only_weighting else None
             )
-            positions.add(
-                list(self.primary_portfolio.get_lookthrough_positions(from_date, portfolio_total_asset_value)),
+            self.builder.add(
+                list(self.primary_portfolio.get_lookthrough_positions(val_date, portfolio_total_asset_value)),
+                infer_underlying_quote_price=True,
             )
-        self.bulk_create_positions(positions, delete_leftovers=True, compute_metrics=True)
+        self.builder.bulk_create_positions(delete_leftovers=True)
+        self.builder.schedule_change_at_dates()
+        self.builder.schedule_metric_computation()
 
     def update_preferred_classification_per_instrument(self):
         # Function to automatically assign Preferred instrument based on the assets' underlying instruments of the
@@ -1002,63 +1068,6 @@ class Portfolio(DeleteToDisableMixin, WBModel):
     def get_representation_label_key(cls):
         return "{{name}}"
 
-    def bulk_create_positions(
-        self,
-        positions: AssetPositionIterator,
-        delete_leftovers: bool = False,
-        force_save: bool = False,
-        compute_metrics: bool = True,
-        **kwargs,
-    ):
-        if positions:
-            # we need to delete the existing estimated portfolio because otherwise we risk to have existing and not
-            # overlapping positions remaining (as they will not be updating by the bulk create). E.g. when someone
-            # change completely the trades of a portfolio model and drift it.
-
-            dates = positions.get_dates()
-            self.assets.filter(date__in=dates, is_estimated=True).delete()
-
-            if self.is_tracked or force_save:  # if the portfolio is not "tracked", we do no drift weights
-                leftover_positions_ids = list(
-                    self.assets.filter(date__in=dates).values_list("id", flat=True)
-                )  # we need to get the ids otherwise the queryset is reevaluated later
-                positions_list = list(positions)
-                logger.info(
-                    f"bulk saving {len(positions_list)} positions ({len(leftover_positions_ids)} leftovers) ..."
-                )
-                objs = AssetPosition.unannotated_objects.bulk_create(
-                    positions_list,
-                    update_fields=[
-                        "weighting",
-                        "initial_price",
-                        "initial_currency_fx_rate",
-                        "initial_shares",
-                        "currency_fx_rate_instrument_to_usd",
-                        "currency_fx_rate_portfolio_to_usd",
-                        "underlying_quote_price",
-                        "portfolio",
-                        "portfolio_created",
-                        "underlying_instrument",
-                    ],
-                    unique_fields=["portfolio", "date", "underlying_quote", "portfolio_created"],
-                    update_conflicts=True,
-                    batch_size=10000,
-                )
-                if delete_leftovers:
-                    objs_ids = list(map(lambda x: x.id, objs))
-                    leftover_positions_ids = list(filter(lambda i: i not in objs_ids, leftover_positions_ids))
-                    logger.info(f"deleting {len(leftover_positions_ids)} leftover positions..")
-                    AssetPosition.objects.filter(id__in=leftover_positions_ids).delete()
-            if compute_metrics and self.is_tracked:
-                for val_date in dates:
-                    compute_metrics_as_task.delay(
-                        val_date,
-                        basket_id=self.id,
-                        basket_content_type_id=ContentType.objects.get_for_model(Portfolio).id,
-                    )
-            for update_date, changed_weights in positions.get_weights().items():
-                self.change_at_date(update_date, changed_weights=changed_weights, **kwargs)
-
     @classmethod
     def _get_or_create_portfolio(cls, instrument_handler, portfolio_data):
         if isinstance(portfolio_data, int):
@@ -1198,6 +1207,7 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         index = Index.objects.create(name=self.name, currency=self.currency)
         index.portfolios.all().delete()
         InstrumentPortfolioThroughModel.objects.update_or_create(instrument=index, defaults={"portfolio": self})
+        return index
 
     @classmethod
     def create_model_portfolio(cls, name: str, currency: Currency, with_index: bool = True):
@@ -1211,20 +1221,9 @@ class Portfolio(DeleteToDisableMixin, WBModel):
         return portfolio
 
 
-def default_estimate_net_value(
-    val_date: date, instrument: Instrument, weights: dict[int, float] | None = None
-) -> float | None:
-    portfolio: Portfolio = instrument.portfolio
-    previous_val_date = (val_date - BDay(1)).date()
-    if not weights:
-        weights = portfolio.get_weights(previous_val_date)
-    # we assume that in t-1 we will have a portfolio (with at least estimate position). If we use the latest position date before val_date, we run into the problem of being able to compute nav at every date
-    if weights and (last_price := instrument.get_latest_price(previous_val_date)):
-        with suppress(
-            IndexError, InvalidAnalyticPortfolio
-        ):  # we silent any indexerror introduced by no returns for the past days
-            analytic_portfolio = portfolio.get_analytic_portfolio(previous_val_date, weights=weights)
-            return analytic_portfolio.get_estimate_net_value(float(last_price.net_value))
+def default_estimate_net_value(last_price: Decimal, analytic_portfolio: AnalyticPortfolio) -> float | None:
+    with suppress(IndexError, ValueError):  # we silent any indexerror introduced by no returns for the past days
+        return analytic_portfolio.get_estimate_net_value(float(last_price.net_value))
 
 
 @receiver(post_save, sender="wbportfolio.PortfolioPortfolioThroughModel")
@@ -1233,7 +1232,7 @@ def post_portfolio_relationship_creation(sender, instance, created, raw, **kwarg
         not raw
         and created
         and instance.portfolio.is_lookthrough
-        and instance.type == PortfolioPortfolioThroughModel.Type.PRIMARY
+        and instance.type == PortfolioPortfolioThroughModel.Type.LOOK_THROUGH
     ):
         with suppress(AssetPosition.DoesNotExist):
             earliest_primary_position_date = instance.dependency_portfolio.assets.earliest("date").date
@@ -1258,10 +1257,33 @@ def update_portfolio_after_investable_universe(*args, end_date: date | None = No
         end_date = date.today()
     end_date = (end_date + timedelta(days=1) - BDay(1)).date()  # shift in case of business day
     from_date = (end_date - BDay(1)).date()
+    excluded_positions = defaultdict(list)
     for portfolio in Portfolio.tracked_objects.all().to_dependency_iterator(from_date):
         if not portfolio.is_lookthrough:
             try:
                 portfolio.propagate_or_update_assets(from_date, end_date)
+                for positions in portfolio.builder.excluded_positions.values():
+                    for pos in positions:
+                        excluded_positions[pos.underlying_quote].append(portfolio)
+                portfolio.builder.clear()
             except Exception as e:
                 logger.error(f"Exception while propagating portfolio assets {portfolio}: {e}")
         portfolio.estimate_net_asset_values(end_date)
+    # if there were excluded positions, we compiled a itemized list of quote per portfolio that got excluded and warn the current portfolio manager
+    if excluded_positions:
+        body = (
+            "<p>While drifting the portfolios, the following quotes got excluded because of missing prices: </p><ul>"
+        )
+        for quote, portfolios in excluded_positions.items():
+            body += f"<li>{quote}</li><p>Impacted portfolios: </p><ul>"
+            for portfolio in portfolios:
+                body += f"<li>{portfolio}</li>"
+            body += "</ul>"
+        body += "</ul> <p>Note: If the quote has simply changed its primary exchange, please use the adjustment tool provided. Otherwise, please contact a system administrator.</p>"
+        for user in User.objects.filter(profile__in=PortfolioRole.portfolio_managers(), is_active=True):
+            send_notification(
+                code="wbportfolio.portfolio.warning",
+                title="Positions were automatically excluded",
+                body=body,
+                user=user,
+            )