finbourne-sdk-utils 0.0.24__py3-none-any.whl

finbourne_sdk_utils/cocoon/cocoon.py

@@ -0,0 +1,1862 @@
+import asyncio
+import concurrent
+import uuid
+
+import lusid
+import pandas as pd
+import json
+
+from typing import List, Tuple
+
+from finbourne_sdk_utils import cocoon
+from finbourne_sdk_utils.cocoon.async_tools import run_in_executor, ThreadPool
+from finbourne_sdk_utils.cocoon.dateorcutlabel import DateOrCutLabel
+from finbourne_sdk_utils.cocoon.utilities import (
+    checkargs,
+    strip_whitespace,
+    group_request_into_one,
+    extract_unique_portfolio_codes,
+    extract_unique_portfolio_codes_effective_at_tuples,
+    get_attributes_and_types
+)
+from finbourne_sdk_utils.cocoon.validator import Validator
+from datetime import datetime
+import pytz
+import logging
+
+
+class BatchLoader:
+    """
+    This class contains all the methods used for loading data in batches. The @run_in_executor decorator makes the
+    synchronous functions awaitable
+    """
+
+    @staticmethod
+    @run_in_executor
+    def load_instrument_batch(
+            api_factory: lusid.SyncApiClientFactory, instrument_batch: list, **kwargs
+    ) -> lusid.models.UpsertInstrumentsResponse:
+        """
+        Upserts a batch of instruments to LUSID
+
+        Parameters
+        ----------
+        api_factory : lusid.SyncApiClientFactory
+            The api factory to use
+        instrument_batch : list[lusid.models.InstrumentDefinition]
+            The batch of instruments to upsert
+        **kwargs
+            arguments specific to each call e.g. effective_at for holdings, unique_identifiers
+
+        Returns
+        -------
+        lusid.models.UpsertInstrumentsResponse
+            The response from LUSID
+        """
+
+        # Ensure that the list of allowed unique identifiers exists
+        if "unique_identifiers" not in list(kwargs.keys()):
+            unique_identifiers = cocoon.instruments.get_unique_identifiers(
+                api_factory=api_factory
+            )
+        else:
+            unique_identifiers = kwargs["unique_identifiers"]
+
+        @checkargs
+        def get_alphabetically_first_identifier_key(
+                instrument: lusid.models.InstrumentDefinition, unique_identifiers: list
+        ):
+            """
+            Gets the alphabetically first occurring unique identifier on an instrument and use it as the correlation
+            id on the request
+
+            Parameters
+            ----------
+            instrument : lusid.models.InstrumentDefinition
+                The instrument to create a correlation id for
+            unique_identifiers : list[str] unique_identifiers
+                The list of allowed unique identifiers
+
+            Returns
+            -------
+            str
+                The correlation id to use on the request
+            """
+
+            unique_identifiers_populated = list(
+                set(unique_identifiers).intersection(
+                    set(list(instrument.identifiers.keys()))
+                )
+            )
+            unique_identifiers_populated.sort()
+            first_unique_identifier_alphabetically = unique_identifiers_populated[0]
+            return f"{first_unique_identifier_alphabetically}: {instrument.identifiers[first_unique_identifier_alphabetically].value}"
+
+        # If scope is not defined set to default scope
+        return api_factory.build(lusid.api.InstrumentsApi).upsert_instruments(
+            scope=kwargs["instrument_scope"],
+            request_body={
+                get_alphabetically_first_identifier_key(
+                    instrument, unique_identifiers
+                ): instrument
+                for instrument in instrument_batch
+            },
+        )
+
+    @staticmethod
+    @run_in_executor
+    def load_quote_batch(
+            api_factory: lusid.SyncApiClientFactory, quote_batch: list, **kwargs
+    ) -> lusid.models.UpsertQuotesResponse:
+        """
+        Upserts a batch of quotes into LUSID
+
+        Parameters
+        ----------
+        api_factory : lusid.SyncApiClientFactory
+            The api factory to use
+        quote_batch : list[lusid.models.UpsertQuoteRequest]
+            The batch of quotes to upsert
+        kwargs
+            scope
+
+        Returns
+        -------
+        lusid.models.UpsertQuotesResponse
+            The response from LUSID
+
+        """
+
+        if "scope" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load quotes without a scope, please ensure that a scope is provided."
+            )
+
+        return api_factory.build(lusid.api.QuotesApi).upsert_quotes(
+            scope=kwargs["scope"],
+            request_body={
+                "_".join(
+                    [
+                        quote.quote_id.quote_series_id.instrument_id,
+                        quote.quote_id.quote_series_id.instrument_id_type,
+                        str(quote.quote_id.effective_at),
+                    ]
+                ): quote
+                for quote in quote_batch
+            },
+        )
+
+    @staticmethod
+    @run_in_executor
+    def load_transaction_batch(
+            api_factory: lusid.SyncApiClientFactory, transaction_batch: list, **kwargs
+    ) -> lusid.models.UpsertPortfolioTransactionsResponse:
+        """
+        Upserts a batch of transactions into LUSID
+
+        Parameters
+        ----------
+        api_factory : lusid.SyncApiClientFactory
+            The api factory to use
+        code : str
+            The code of the TransactionPortfolio to upsert the transactions into
+        transaction_batch : list[lusid.models.TransactionRequest]
+            The batch of transactions to upsert
+        kwargs
+            code -The code of the TransactionPortfolio to upsert the transactions into
+
+        Returns
+        -------
+        lusid.models.UpsertPortfolioTransactionsResponse
+            The response from LUSID
+        """
+
+        if "scope" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load transactions without a scope, please ensure that a scope is provided."
+            )
+
+        if "code" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load transactions without a portfolio code, please ensure that a code is provided."
+            )
+
+        return api_factory.build(
+            lusid.api.TransactionPortfoliosApi
+        ).upsert_transactions(
+            scope=kwargs["scope"],
+            code=kwargs["code"],
+            transaction_request=transaction_batch,
+        )
+
+    @staticmethod
+    @run_in_executor
+    def load_transactions_with_commit_mode_batch(
+            api_factory: lusid.SyncApiClientFactory, transaction_batch: List, **kwargs
+    ) -> lusid.models.UpsertPortfolioTransactionsResponse:
+        """
+        Upserts a batch of transactions into LUSID with specified type of upsert.
+
+        Parameters
+        ----------
+        api_factory : lusid.SyncApiClientFactory
+            The api factory to use
+        code : str
+            The code of the TransactionPortfolio to upsert the transactions into
+        transaction_batch : List[lusid.models.TransactionRequest]
+            The batch of transactions to upsert
+        kwargs
+            code -The code of the TransactionPortfolio to upsert the transactions into
+            transactions_commit_mode - The type of upsert method to use (Atomic or Partial)
+
+        Returns
+        -------
+        lusid.models.BatchUpsertPortfolioTransactionsResponse
+            The response from LUSID
+        """
+
+        if "scope" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load transactions without a scope, please ensure that a scope is provided."
+            )
+
+        if "code" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load transactions without a portfolio code, please ensure that a code is provided."
+            )
+
+        if kwargs["transactions_commit_mode"] is None:
+            raise KeyError(
+                "You are trying to load transactions without a commit mode (transaction_commit_mode), please provide a commit mode (Atomic or Partial)."
+            )
+
+        if kwargs["transactions_commit_mode"].lower() not in ("atomic", "partial"):
+            raise KeyError(
+                "You are trying to load transactions without an accepted type of commit mode, please provide either Atomic or Partial to the commit mode."
+            )
+
+        request_body = {
+            f"transaction_{idx}": transaction
+            for idx, transaction in enumerate(transaction_batch)
+        }
+
+
+        return api_factory.build(
+            lusid.api.TransactionPortfoliosApi
+        ).batch_upsert_transactions(
+            scope=kwargs["scope"],
+            code=kwargs["code"],
+            success_mode=kwargs["transactions_commit_mode"],
+            request_body=request_body
+        )
+
+    @staticmethod
+    @run_in_executor
+    def load_holding_batch(
+            api_factory: lusid.SyncApiClientFactory, holding_batch: list, **kwargs
+    ) -> lusid.models.HoldingsAdjustment:
+        """
+        Upserts a batch of holdings into LUSID
+
+        Parameters
+        ----------
+        api_factory : lusid.SyncApiClientFactory
+            The api factory to use
+        holding_batch : list[lusid.models.AdjustHoldingRequest]
+            The batch of holdings
+        scope : str
+            The scope to upsert holdings into
+        code : str
+            The code of the portfolio to upsert holdings into
+        effective_at : str/Datetime/np.datetime64/np.ndarray/pd.Timestamp
+            The effective date of the holdings batch
+        kwargs
+
+        Returns
+        -------
+        lusid.models.HoldingsAdjustment
+            The response from LUSID
+
+        """
+
+        if "scope" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load transactions without a scope, please ensure that a scope is provided."
+            )
+
+        if "code" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load transactions without a portfolio code, please ensure that a code is provided."
+            )
+
+        if "effective_at" not in list(kwargs.keys()):
+            raise KeyError(
+                """There is no mapping for effective_at in the required mapping, please add it"""
+            )
+
+        # If only an adjustment has been specified
+        if (
+                "holdings_adjustment_only" in list(kwargs.keys())
+                and kwargs["holdings_adjustment_only"]
+        ):
+            return api_factory.build(
+                lusid.api.TransactionPortfoliosApi
+            ).adjust_holdings(
+                scope=kwargs["scope"],
+                code=kwargs["code"],
+                effective_at=str(DateOrCutLabel(kwargs["effective_at"])),
+                adjust_holding_request=holding_batch,
+            )
+
+        return api_factory.build(lusid.api.TransactionPortfoliosApi).set_holdings(
+            scope=kwargs["scope"],
+            code=kwargs["code"],
+            effective_at=str(DateOrCutLabel(kwargs["effective_at"])),
+            adjust_holding_request=holding_batch,
+        )
+
+    @staticmethod
+    @run_in_executor
+    def load_portfolio_batch(
+            api_factory: lusid.SyncApiClientFactory, portfolio_batch: list, **kwargs
+    ) -> lusid.models.Portfolio:
+        """
+        Upserts a batch of portfolios to LUSID
+
+        Parameters
+        ----------
+        api_factory : lusid.SyncApiClientFactory
+            the api factory to use
+        portfolio_batch : list[lusid.models.CreateTransactionPortfolioRequest]
+            The batch of portfolios to create
+        scope : str
+            The scope to create the portfolios in
+        code : str
+            The code of the portfolio to create
+        kwargs
+
+        Returns
+        -------
+        lusid.models.Portfolio
+            the response from LUSID
+        """
+
+        if "scope" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load transactions without a scope, please ensure that a scope is provided."
+            )
+
+        if "code" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load transactions without a portfolio code, please ensure that a code is provided."
+            )
+
+        try:
+            return api_factory.build(lusid.api.PortfoliosApi).get_portfolio(
+                scope=kwargs["scope"], code=kwargs["code"]
+            )
+        # Add in here upsert portfolio properties if it does exist
+        except lusid.exceptions.ApiException as e:
+            if e.status == 404:
+                return api_factory.build(
+                    lusid.api.TransactionPortfoliosApi
+                ).create_portfolio(
+                    scope=kwargs["scope"],
+                    create_transaction_portfolio_request=portfolio_batch[0],
+                )
+            else:
+                return e
+
+    @staticmethod
+    @run_in_executor
+    def load_reference_portfolio_batch(
+            api_factory: lusid.SyncApiClientFactory,
+            reference_portfolio_batch: list,
+            **kwargs,
+    ) -> lusid.models.Portfolio:
+        """
+        Upserts a batch of reference portfolios to LUSID
+
+        Parameters
+        ----------
+        api_factory : lusid.SyncApiClientFactory
+            the api factory to use
+        portfolio_batch : list[lusid.models.CreateReferencePortfolioRequest]
+            The batch of reference portfolios to create
+        scope : str
+            The scope to create the reference portfolios in
+        code : str
+            The code of the reference portfolio to create
+        kwargs
+
+        Returns
+        -------
+        lusid.models.ReferencePortfolio
+            the response from LUSID
+        """
+
+        if "scope" not in kwargs.keys():
+            raise KeyError(
+                "You are trying to load a reference portfolio without a scope, please ensure that a scope is provided."
+            )
+
+        if "code" not in kwargs.keys():
+            raise KeyError(
+                "You are trying to load a reference portfolio without a portfolio code, please ensure that a code is provided."
+            )
+
+        try:
+            return api_factory.build(lusid.api.PortfoliosApi).get_portfolio(
+                scope=kwargs["scope"], code=kwargs["code"]
+            )
+        # TODO: Add in here upsert portfolio properties if it does exist
+
+        except lusid.exceptions.ApiException as e:
+            if e.status == 404:
+                return api_factory.build(
+                    lusid.api.ReferencePortfolioApi
+                ).create_reference_portfolio(
+                    scope=kwargs["scope"],
+                    create_reference_portfolio_request=reference_portfolio_batch[0],
+                )
+            else:
+                return e
+
+    @staticmethod
+    @run_in_executor
+    def load_instrument_property_batch(
+            api_factory: lusid.SyncApiClientFactory, property_batch: list, **kwargs
+    ) -> List[lusid.models.UpsertInstrumentPropertiesResponse]:
+        """
+        Add properties to the set instruments
+
+        Parameters
+        ----------
+        api_factory : lusid.SyncApiClientFactory
+            The api factory to use
+        property_batch : list[lusid.models.UpsertInstrumentPropertyRequest]
+            Properties to add,
+            identifiers will be resolved to a LusidInstrumentId, where an identifier resolves to more
+            than one LusidInstrumentId the property will be added to all matching instruments
+        kwargs
+
+        Returns
+        -------
+        list[lusid.models.UpsertInstrumentPropertiesResponse]
+            the response from LUSID
+        """
+
+        results = []
+        for request in property_batch:
+            search_request = lusid.models.InstrumentSearchProperty(
+                key=f"instrument/default/{request.identifier_type}",
+                value=request.identifier,
+            )
+
+            # find the matching instruments
+            mastered_instruments = api_factory.build(
+                lusid.api.SearchApi
+            ).instruments_search(
+                instrument_search_property=[search_request], mastered_only=True
+            )
+
+            # flat map the results to a list of luids
+            luids = [
+                luid
+                for luids in [
+                    list(
+                        map(
+                            lambda m: m.identifiers["LusidInstrumentId"].value,
+                            mastered.mastered_instruments,
+                        )
+                    )
+                    for mastered in [matches for matches in mastered_instruments]
+                ]
+                for luid in luids
+            ]
+
+            if len(luids) == 0:
+                continue
+
+            properties_request = [
+                lusid.models.UpsertInstrumentPropertyRequest(
+                    identifier_type="LusidInstrumentId",
+                    identifier=luid,
+                    properties=request.properties,
+                )
+                for luid in luids
+            ]
+
+            results.append(
+                api_factory.build(
+                    lusid.api.InstrumentsApi
+                ).upsert_instruments_properties(properties_request)
+            )
+
+        return results
+
+    @staticmethod
+    @run_in_executor
+    def load_portfolio_group_batch(
+            api_factory: lusid.SyncApiClientFactory,
+            portfolio_group_batch: list,
+            **kwargs,
+    ) -> lusid.models.PortfolioGroup:
+        """
+        Upserts a batch of portfolios to LUSID
+
+        Parameters
+        ----------
+        api_factory : lusid.SyncApiClientFactory
+            the api factory to use
+        portfolio_group_batch : list[lusid.models.CreateTransactionPortfolioRequest]
+            The batch of portfilios to create
+        scope : str
+            The scope to create the portfolio group in
+        code : str
+            The code of the portfolio group to create
+        kwargs
+
+        Returns
+        -------
+        lusid.models.PortfolioGroup
+            The response from LUSID
+        """
+
+        updated_request = group_request_into_one(
+            portfolio_group_batch[0].__class__.__name__,
+            portfolio_group_batch,
+            ["values"],
+        )
+
+        if "scope" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load a portfolio group without a scope, please ensure that a scope is provided."
+            )
+
+        if "code" not in list(kwargs.keys()):
+            raise KeyError(
+                "You are trying to load a portfolio group without a portfolio code, please ensure that a code is provided."
+            )
+
+        try:
+
+            current_portfolio_group = api_factory.build(
+                lusid.api.PortfolioGroupsApi
+            ).get_portfolio_group(scope=kwargs["scope"], code=kwargs["code"])
+
+            #  Capture all portfolios - the ones currently in group + the new ones to be added
+            all_portfolios_to_add = (
+                    updated_request.values + current_portfolio_group.portfolios
+            )
+
+            current_portfolios_in_group = [
+                code
+                for code in updated_request.values
+                if code in current_portfolio_group.portfolios
+            ]
+
+            if len(current_portfolios_in_group) > 0:
+                for code in current_portfolios_in_group:
+                    logging.info(
+                        f"The portfolio {code.code} with scope {code.scope} is already in group {current_portfolio_group.id.code}"
+                    )
+
+            # Parse out new portfolios only
+            new_portfolios = [
+                code
+                for code in all_portfolios_to_add
+                if code not in current_portfolio_group.portfolios
+            ]
+
+            for code, scope in set(
+                    [(resource.code, resource.scope) for resource in new_portfolios]
+            ):
+
+                try:
+
+                    current_portfolio_group = api_factory.build(
+                        lusid.api.PortfolioGroupsApi
+                    ).add_portfolio_to_group(
+                        scope=kwargs["scope"],
+                        code=kwargs["code"],
+                        effective_at=datetime.now(tz=pytz.UTC).isoformat(),
+                        resource_id=lusid.models.ResourceId(scope=scope, code=code),
+                    )
+
+                except lusid.exceptions.ApiException as e:
+                    logging.error(json.loads(e.body)["title"])
+
+            return current_portfolio_group
+
+        # Add in here upsert portfolio properties if it does exist
+        except lusid.exceptions.ApiException as e:
+            if e.status == 404:
+                return api_factory.build(
+                    lusid.api.PortfolioGroupsApi
+                ).create_portfolio_group(
+                    scope=kwargs["scope"],
+                    create_portfolio_group_request=updated_request,
+                )
+            else:
+                return e
+
+
+async def _load_data(
+        api_factory: lusid.SyncApiClientFactory,
+        single_requests: list,
+        file_type: str,
+        **kwargs,
+):
+    """
+    This function calls the appropriate batch loader
+
+    Parameters
+    ----------
+    api_factory : lusid.SyncApiClientFactory
+        The api factory to use
+    single_requests
+        The list of single requests for LUSID
+    file_type : str
+        The file type e.g. instruments, portfolios etc.
+    kwargs
+        arguments specific to each call e.g. effective_at for holdings
+
+    Returns
+    -------
+    BatchLoader : StaticMethod
+        A static method on batchloader
+    """
+
+    # Dynamically call the correct async function to use based on the file type
+    identifier = uuid.uuid4()
+    logging.debug(f"Running load_{file_type}_batch({identifier})")
+    from time import time
+
+    start = time()
+    response = await getattr(BatchLoader, f"load_{file_type}_batch")(
+        api_factory,
+        single_requests,
+        # Any specific arguments e.g. 'code' for transactions, 'effective_at' for holdings is passed in via **kwargs
+        **kwargs,
+    )
+    logging.debug(f"Batch completed ({identifier}) - duration: {time() - start}")
+    return response
+
+
+def _convert_batch_to_models(
+        data_frame: pd.DataFrame,
+        mapping_required: dict,
+        mapping_optional: dict,
+        property_columns: list,
+        properties_scope: str,
+        instrument_identifier_mapping: dict,
+        file_type: str,
+        domain_lookup: dict,
+        sub_holding_keys: list,
+        sub_holding_keys_scope: str,
+        **kwargs,
+):
+    """
+    This function populates the required models from a DataFrame and loads the data into LUSID
+
+    Parameters
+    ----------
+    data_frame : pd.DataFrame
+        The DataFrame containing the data to load
+    mapping_required : dict
+        The required mapping
+    mapping_optional : dict
+        The optional mapping
+    property_columns : list
+        The property columns to add as property values
+    properties_scope : str
+        The scope to add the property values in
+    instrument_identifier_mapping : dict
+        The mapping for the identifiers
+    file_type : str
+        The file type to load
+    domain_lookup : dict
+        The domain lookup
+    sub_holding_keys : list
+        The sub holding keys to use
+    sub_holding_keys_scope : str
+        The scope to use for the sub holding keys
+    kwargs
+        Arguments specific to each call e.g. effective_at for holdings
+
+    Returns
+    -------
+    single_requests : list
+         A list of populated LUSID request models
+    """
+
+    source_columns = [
+        column.get("target", column.get("source")) for column in property_columns
+    ]
+
+    # Get the data types of the columns to be added as properties
+    property_dtypes = data_frame.loc[:, source_columns].dtypes
+
+    # Get the types of the attributes on the top level model for this request
+    open_api_types = get_attributes_and_types(getattr(
+        lusid.models, domain_lookup[file_type]["top_level_model"]
+    ))
+
+    # If there is a sub_holding_keys attribute and it has a dict type this means the sub_holding_keys
+    # need to be populated with property values
+    if (
+            "sub_holding_keys" in open_api_types.keys()
+            and "Mapping" in open_api_types["sub_holding_keys"]
+    ):
+        sub_holding_key_dtypes = data_frame.loc[:, sub_holding_keys].dtypes
+    # If not and they are provided as full keys
+    elif len(sub_holding_keys) > 0:
+        sub_holding_keys_row = cocoon.properties._infer_full_property_keys(
+            partial_keys=sub_holding_keys,
+            properties_scope=sub_holding_keys_scope,
+            domain="Transaction",
+        )
+    # If no keys
+    else:
+        sub_holding_keys_row = None
+
+    unique_identifiers = kwargs["unique_identifiers"]
+
+    # Iterate over the DataFrame creating the single requests
+    single_requests = []
+    for index, row in data_frame.iterrows():
+
+        # Create the property values for this row
+        if domain_lookup[file_type]["domain"] is None:
+            properties = None
+        else:
+            column_to_scope = {
+                column.get("target", column.get("source")): column.get(
+                    "scope", properties_scope
+                )
+                for column in property_columns
+            }
+
+            properties = cocoon.properties.create_property_values(
+                row=row,
+                column_to_scope=column_to_scope,
+                scope=properties_scope,
+                domain=domain_lookup[file_type]["domain"],
+                dtypes=property_dtypes,
+            )
+
+        # Create the sub-holding-keys for this row
+        if (
+                "sub_holding_keys" in open_api_types.keys()
+                and "Mapping" in open_api_types["sub_holding_keys"]
+        ):
+            sub_holding_keys_row = cocoon.properties.create_property_values(
+                row=row,
+                column_to_scope={},
+                scope=sub_holding_keys_scope,
+                domain="Transaction",
+                dtypes=sub_holding_key_dtypes,
+            )
+
+        # Create identifiers for this row if applicable
+        if instrument_identifier_mapping is None or not bool(
+                instrument_identifier_mapping
+        ):
+            identifiers = None
+        else:
+            identifiers = cocoon.instruments.create_identifiers(
+                index=index,
+                row=row,
+                file_type=file_type,
+                instrument_identifier_mapping=instrument_identifier_mapping,
+                unique_identifiers=unique_identifiers,
+                full_key_format=kwargs["full_key_format"],
+            )
+
+        # Construct the from the mapping, properties and identifiers the single request object and add it to the list
+        single_requests.append(
+            cocoon.utilities.populate_model(
+                model_object_name=domain_lookup[file_type]["top_level_model"],
+                required_mapping=mapping_required,
+                optional_mapping=mapping_optional,
+                row=row,
+                properties=properties,
+                identifiers=identifiers,
+                sub_holding_keys=sub_holding_keys_row,
+            )
+        )
+
+    return single_requests
+
+
+async def _construct_batches(
+        api_factory: lusid.SyncApiClientFactory,
+        data_frame: pd.DataFrame,
+        mapping_required: dict,
+        mapping_optional: dict,
+        property_columns: list,
+        properties_scope: str,
+        instrument_identifier_mapping: dict,
+        batch_size: int,
+        file_type: str,
+        domain_lookup: dict,
+        sub_holding_keys: list,
+        sub_holding_keys_scope: str,
+        return_unmatched_items: bool,
+        **kwargs,
+):
+    """
+    This constructs the batches and asynchronously sends them to be loaded into LUSID
+
+    Parameters
+    ----------
+    api_factory : lusid.SyncApiClientFactory
+        The api factory to use
+    data_frame : pd.DataFrame
+        The DataFrame containing the data to load
+    mapping_required : dict
+        The required mapping
+    mapping_optional : dict
+        The optional mapping
+    property_columns : list
+        The property columns to add as property values
+    properties_scope : str
+        The scope to add the property values in
+    instrument_identifier_mapping : dict
+        The mapping for the identifiers
+    batch_size : int
+        The batch size to use
+    file_type : str
+        The file type to load
+    domain_lookup : dict
+        The domain lookup
+    sub_holding_keys : list
+        The sub holding keys to use
+    sub_holding_keys_scope : str
+        The scope to use for the sub-holding keys
+    return_unmatched_items : bool
+        Whether items with unmatched identifiers should be returned for transaction or holding upserts
+    kwargs
+        Arguments specific to each call e.g. effective_at for holdings
+
+    Returns
+    -------
+    dict
+        Contains the success responses and the errors (where an API exception has been raised)
+    """
+
+    # Get the different behaviours required for different entities e.g quotes can be batched without worrying about portfolios
+    batching_no_portfolios = [
+        file_type
+        for file_type, settings in domain_lookup.items()
+        if not settings["portfolio_specific"]
+    ]
+    batching_with_portfolios = [
+        file_type
+        for file_type, settings in domain_lookup.items()
+        if settings["portfolio_specific"]
+    ]
+
+    if file_type in batching_no_portfolios:
+
+        # Everything can be sent up asynchronously, prepare batches based on batch size alone
+        async_batches = [
+            data_frame.iloc[i: i + batch_size]
+            for i in range(0, len(data_frame), batch_size)
+        ]
+
+        # Nest the async batches inside a single synchronous batch
+        sync_batches = [
+            {
+                "async_batches": async_batches,
+                "codes": [None] * len(async_batches),
+                "effective_at": [None] * len(async_batches),
+            }
+        ]
+
+    elif file_type in batching_with_portfolios:
+
+        if "effective_at" in domain_lookup[file_type]["required_call_attributes"]:
+
+            # Get unique effective dates
+            unique_effective_dates = list(
+                data_frame[mapping_required["effective_at"]].unique()
+            )
+
+            # Create a group for each effective date as they can not be batched asynchronously
+            effective_at_groups = [
+                data_frame.loc[
+                    data_frame[mapping_required["effective_at"]] == effective_at
+                    ]
+                for effective_at in unique_effective_dates
+            ]
+
+            # Create a synchronous batch for each effective date
+            sync_batches = [
+                {
+                    # Different portfolio codes can be batched asynchronously inside the synchronous batch
+                    "async_batches": [
+                        effective_at_group.loc[
+                            data_frame[mapping_required["code"]] == code
+                            ]
+                        for code in list(
+                            effective_at_group[mapping_required["code"]].unique()
+                        )
+                    ],
+                    "codes": list(
+                        effective_at_group[mapping_required["code"]].unique()
+                    ),
+                    "effective_at": [
+                                        list(
+                                            effective_at_group[
+                                                mapping_required["effective_at"]
+                                            ].unique()
+                                        )[0]
+                                    ]
+                                    * len(list(effective_at_group[mapping_required["code"]].unique())),
+                }
+                for effective_at_group in effective_at_groups
+            ]
+
+        else:
+
+            unique_portfolios = list(data_frame[mapping_required["code"]].unique())
+
+            # Different portfolio codes can be batched asynchronously
+            async_batches = [
+                data_frame.loc[data_frame[mapping_required["code"]] == code]
+                for code in unique_portfolios
+            ]
+
+            # Inside the synchronous batch split the values for each portfolio into appropriate batch sizes
+            sync_batches = [
+                {
+                    "async_batches": [
+                        async_batch.iloc[i: i + batch_size]
+                        for async_batch in async_batches
+                    ],
+                    "codes": [str(code) for code in unique_portfolios],
+                    "effective_at": [None] * len(async_batches),
+                }
+                for i in range(
+                    0,
+                    max([len(async_batch) for async_batch in async_batches]),
+                    batch_size,
+                )
+            ]
+
+    logging.debug("Created sync batches: ")
+    logging.debug(
+        f"Number of batches: {len(sync_batches)}, "
+        + f"Number of items in batches: {sum([len(sync_batch['async_batches']) for sync_batch in sync_batches])}"
+    )
+
+    # Asynchronously load the data into LUSID
+    responses = [
+        await asyncio.gather(
+            *[
+                _load_data(
+                    api_factory=api_factory,
+                    single_requests=_convert_batch_to_models(
+                        data_frame=async_batch,
+                        mapping_required=mapping_required,
+                        mapping_optional=mapping_optional,
+                        property_columns=property_columns,
+                        properties_scope=properties_scope,
+                        instrument_identifier_mapping=instrument_identifier_mapping,
+                        file_type=file_type,
+                        domain_lookup=domain_lookup,
+                        sub_holding_keys=sub_holding_keys,
+                        sub_holding_keys_scope=sub_holding_keys_scope,
+                        **kwargs,
+                    ),
+                    file_type=file_type,
+                    code=code,
+                    effective_at=effective_at,
+                    **kwargs,
+                )
+                for async_batch, code, effective_at in zip(
+                    sync_batch["async_batches"],
+                    sync_batch["codes"],
+                    sync_batch["effective_at"],
+                )
+                if not async_batch.empty
+            ],
+            return_exceptions=True,
+        )
+        for sync_batch in sync_batches
+    ]
+    logging.debug("Flattening responses")
+    responses_flattened = [
+        response for responses_sub in responses for response in responses_sub
+    ]
+
+    # Raise any internal exceptions rather than propagating them to the response
+    for response in responses_flattened:
+        if isinstance(response, Exception) and not isinstance(
+                response, lusid.exceptions.ApiException
+        ):
+            raise response
+
+    # Collects the exceptions as failures and successful calls as values
+    returned_response = {
+        "errors": [r for r in responses_flattened if isinstance(r, Exception)],
+        "success": [r for r in responses_flattened if not isinstance(r, Exception)],
+    }
+
+    # For successful transactions or holdings file types, optionally return unmatched identifiers with the responses
+    if check_for_unmatched_items(
+            flag=return_unmatched_items,
+            file_type=file_type,
+    ):
+        logging.debug("returning unmatched identifiers with the responses")
+        returned_response["unmatched_items"] = unmatched_items(
+            api_factory=api_factory,
+            scope=kwargs.get("scope", None),
+            data_frame=data_frame,
+            mapping_required=mapping_required,
+            file_type=file_type,
+            returned_response=returned_response,
+            sync_batches=sync_batches,
+        )
+
+    return returned_response
+
+
+def check_for_unmatched_items(flag, file_type):
+    """
+    This method contains the conditional logic to determine whether the unmatched_items validation should be run.
+    It should not be run if:
+        a) it was not requested
+        b) the upload was not for transactions or holdings
+
+    Parameters
+    ----------
+    flag : bool
+        The bool flag that indicates whether any unmatched identifiers should be checked
+    file_type : str
+        The file type of the upload
+    Returns
+    -------
+
+    A boolean value indicating whether the unmatched_items validation should be completed
+    """
+    condition_1 = flag is True
+    condition_2 = file_type in ["transaction", "holding"]
+
+    return condition_1 and condition_2
+
+
+@checkargs
+def unmatched_items(
+        api_factory: lusid.SyncApiClientFactory,
+        scope: str,
+        data_frame: pd.DataFrame,
+        mapping_required: dict,
+        file_type: str,
+        returned_response: dict,
+        sync_batches: list = None,
+):
+    """
+    This method orchestrates the identification of holdings or transactions objects that were successfully uploaded
+    but did not resolve to known identifiers (i.e. where the instrument_uid is LUID_ZZZZZZ).
+
+    If there were any errors in the attempted upload, the check will not complete and an error message will be returned
+    to let the user know that errors will need to be resolved first.
+
+    Parameters
+    ----------
+    api_factory : lusid.SyncApiClientFactory api_factory
+        The api factory to use
+    scope : str
+        The scope of the resource to load the data into
+    data_frame : pd.DataFrame
+        The DataFrame containing the data
+    mapping_required : dict
+        The required mapping
+    file_type : str
+        The type of file e.g. transactions, instruments, holdings, quotes, portfolios
+    returned_response : dict
+        The response from laod_from_data_frame
+    sync_batches : list
+        A list of the batches used to upload the data into LUSID.
+
+    Returns
+    -------
+    responses: list
+        A list of objects to be appended to the ultimate response for load_from_data_frame.
+    """
+
+    if len(returned_response["errors"]) > 0:
+        return ["Please resolve all upload errors to check for unmatched items."]
+
+    if file_type == "transaction":
+        return _unmatched_transactions(
+            api_factory=api_factory,
+            scope=scope,
+            data_frame=data_frame,
+            mapping_required=mapping_required,
+            sync_batches=sync_batches,
+        )
+    elif file_type == "holding":
+        return _unmatched_holdings(
+            api_factory=api_factory,
+            scope=scope,
+            sync_batches=sync_batches,
+        )
+
+
+def _unmatched_transactions(
+        api_factory: lusid.SyncApiClientFactory,
+        scope: str,
+        data_frame: pd.DataFrame,
+        mapping_required: dict,
+        sync_batches: list = None,
+):
+    """
+    This method identifies which instruments were not resolved with a transaction upload using load_from_data_frame.
+
+    Parameters
+    ----------
+    api_factory : lusid.SyncApiClientFactory api_factory
+        The api factory to use
+    scope : str
+        The scope of the resource to load the data into
+    data_frame : pd.DataFrame
+        The DataFrame containing the data
+    sync_batches : list
+        A list of the batches used to upload the data into LUSID.
+
+    Returns
+    -------
+    responses: list
+        A list of transaction objects to be appended to the ultimate response for load_from_data_frame.
+    """
+    # Extract a list of portfolio codes from the sync_batches
+    portfolio_codes = extract_unique_portfolio_codes(sync_batches)
+
+    # Create empty list to hold transaction ids and instruments
+    unmatched_transactions = []
+
+    # For each portfolio, request the unmatched transactions from LUSID and append to the instantiated list
+    for portfolio_code in portfolio_codes:
+        portfolio_transactions = data_frame.loc[
+            data_frame[mapping_required["code"]] == portfolio_code
+            ]
+        from_transaction_date = min(
+            portfolio_transactions[mapping_required["transaction_date"]].apply(
+                lambda x: str(DateOrCutLabel(x))
+            )
+        )
+        to_transactions_date = max(
+            portfolio_transactions[mapping_required["transaction_date"]].apply(
+                lambda x: str(DateOrCutLabel(x))
+            )
+        )
+
+        unmatched_transactions.extend(
+            return_unmatched_transactions(
+                api_factory=api_factory,
+                scope=scope,
+                code=portfolio_code,
+                from_transaction_date=from_transaction_date,
+                to_transaction_date=to_transactions_date,
+            )
+        )
+
+    # With the upload dataframe, filter out any transactions from the list that were not part of this upload and return
+    return filter_unmatched_transactions(
+        data_frame=data_frame,
+        mapping_required=mapping_required,
+        unmatched_transactions=unmatched_transactions,
+    )
+
+
+def return_unmatched_transactions(
+        api_factory: lusid.SyncApiClientFactory,
+        scope: str,
+        code: str,
+        from_transaction_date: str,
+        to_transaction_date: str,
+):
+    """
+    Call the get transactions api and only return those transactions with unresolved identifiers.
+
+    Multiple pages of responses for the get_transactions api call are handled.
+
+    Parameters
+    ----------
+    api_factory : lusid.SyncApiClientFactory api_factory
+        The api factory to use
+    scope : str
+        The scope of the resource to load the data into
+    code : str
+        The code of the portfolio containing the transactions we want to return
+
+    Returns
+    -------
+    A list of transaction objects with the structure.
+    """
+    transactions_api = api_factory.build(lusid.api.TransactionPortfoliosApi)
+    done = False
+    next_page = None
+    unmatched_transactions = []
+
+    # We need to handle the possibility of paginated results
+    while not done:
+
+        # There must be a filter included so only transactions with unmatched instruments are returned
+        kwargs = {
+            "scope": scope,
+            "code": code,
+            "from_transaction_date": from_transaction_date,
+            "to_transaction_date": to_transaction_date,
+            "filter": "instrumentUid eq'LUID_ZZZZZZZZ'",
+        }
+
+        if next_page is not None:
+            kwargs["page"] = next_page
+
+        response = transactions_api.get_transactions(**kwargs)
+
+        unmatched_transactions.extend([response for response in response.values])
+
+        next_page = response.next_page
+        done = response.next_page is None
+
+    return unmatched_transactions
+
+
+def filter_unmatched_transactions(
+        data_frame: pd.DataFrame,
+        mapping_required: dict,
+        unmatched_transactions: list,
+):
+    """
+    This method will take the full list of unmatched transactions and remove any transactions that were not
+    part of the current upload with load_from_data_frame.
+
+    Parameters
+    ----------
+    data_frame : pd.DataFrame
+        The DataFrame containing the data
+    mapping_required : dict
+        The required mapping from load_from_data_frame that helps identify the transaction_id column in the DataFrame
+    unmatched_transactions : list
+         A list of transaction objects.
+
+    Returns
+    -------
+    A filtered list of transaction objects.
+    """
+    # Create a unique list of transaction_ids from the dataframe
+    valid_txids = set(data_frame[mapping_required.get("transaction_id")])
+
+    # Iterate through the transactions and if the transaction_id is in the set, add it to the list to be returned
+    filtered_unmatched_transactions = [
+        unmatched_transaction
+        for unmatched_transaction in unmatched_transactions
+        if unmatched_transaction.transaction_id in valid_txids
+    ]
+
+    return filtered_unmatched_transactions
+
+
+def _unmatched_holdings(
+        api_factory: lusid.SyncApiClientFactory,
+        scope: str,
+        sync_batches: list = None,
+):
+    """
+    This method identifies which instruments were not resolved with a holdings upload using load_from_data_frame.
+
+    Parameters
+    ----------
+    api_factory : lusid.SyncApiClientFactory api_factory
+        The api factory to use
+    scope : str
+        The scope of the resource to load the data into
+    sync_batches : list
+        A list of the batches used to upload the data into LUSID
+
+    Returns
+    -------
+    responses: list
+        A list of holding objects to be appended to the ultimate response for load_from_data_frame.
+    """
+    # Extract a list of tuples of portfolio codes and effective at times from sync_batches
+    code_tuples = extract_unique_portfolio_codes_effective_at_tuples(sync_batches)
+
+    # Create empty list to hold holdings that have not been resolved
+    unmatched_holdings = []
+
+    # For each holding adjustment in the upload, check whether any contained unresolved instruments and append to list
+    for code_tuple in code_tuples:
+        unmatched_holdings.extend(
+            return_unmatched_holdings(
+                api_factory=api_factory,
+                scope=scope,
+                code_tuple=code_tuple,
+            )
+        )
+
+    return unmatched_holdings
+
+
+def return_unmatched_holdings(
+        api_factory: lusid.SyncApiClientFactory,
+        scope: str,
+        code_tuple: Tuple[str, str],
+):
+    """
+    Call the get holdings adjustments api and return a list of holding objects that have unresolved identifiers.
+
+    Parameters
+    ----------
+    api_factory : lusid.SyncApiClientFactory api_factory
+        The api factory to use
+    scope : str
+        The scope of the resource to load the data into
+    code_tuple : (str, str)
+        A tuple of format (portfolio_code, effective_at) which represents the combination of values
+        used when upserting holdings during load_from_data_frame for file_type 'holding'
+
+    Returns
+    -------
+    A list of holding objects.
+
+    """
+    transactions_api = api_factory.build(lusid.api.TransactionPortfoliosApi)
+
+    # In case a holdings check occurs for a portfolio code and effective at combination that contain no holdings,
+    # make sure to gracefully handle the exception so that any LUSID error from the main upload is not suppressed.
+    try:
+        response = transactions_api.get_holdings_adjustment(
+            scope=scope,
+            code=code_tuple[0],
+            effective_at=str(DateOrCutLabel(code_tuple[1])),
+        )
+    except lusid.ApiException as e:
+        if "HoldingsAdjustmentDoesNotExist" in str(e.body):
+            logging.info(
+                f"While validating holding upload, LUSID was unable to find any holdings within portfolio "
+                f"{code_tuple[0]} for effectiveAt {str(DateOrCutLabel(code_tuple[1]))}."
+            )
+            return []
+
+    # Create a list of holding objects with unmatched identifiers (e.g. with LUID_ZZZZZZZZ) from the response
+    return [
+        adjustment
+        for adjustment in response.adjustments
+        if adjustment.instrument_uid == "LUID_ZZZZZZZZ"
+    ]
+
+
+@checkargs
+def load_from_data_frame(
+        api_factory: lusid.SyncApiClientFactory,
+        scope: str,
+        data_frame: pd.DataFrame,
+        mapping_required: dict,
+        mapping_optional: dict,
+        file_type: str,
+        identifier_mapping: dict = None,
+        property_columns: list = None,
+        properties_scope: str = None,
+        batch_size: int = None,
+        remove_white_space: bool = True,
+        instrument_name_enrichment: bool = False,
+        transactions_commit_mode: str = None,
+        sub_holding_keys: list = None,
+        holdings_adjustment_only: bool = False,
+        thread_pool_max_workers: int = 5,
+        sub_holding_keys_scope: str = None,
+        return_unmatched_items: bool = False,
+        instrument_scope: str = None,
+):
+    """
+
+    Parameters
+    ----------
+    api_factory : lusid.SyncApiClientFactory api_factory
+        The api factory to use
+    scope : str
+        The scope of the resource to load the data into
+        If file_type="instrument" scope will only be used for instrument properties
+    data_frame : pd.DataFrame
+        The DataFrame containing the data
+    mapping_required : dict{str, str}
+        The dictionary mapping the DataFrame columns to LUSID's required attributes
+    mapping_optional : dict{str, str}
+        The dictionary mapping the DataFrame columns to LUSID's optional attributes
+    file_type : str
+        The type of file e.g. transactions, instruments, holdings, quotes, portfolios
+    identifier_mapping : dict{str, str}
+        The dictionary mapping of LUSID instrument identifiers to identifiers in the DataFrame
+    property_columns : list
+        The columns to create properties for
+    properties_scope : str
+        The scope to add the properties to
+    batch_size : int
+        The size of the batch to use when using upsert calls e.g. upsert instruments, upsert quotes etc.
+    remove_white_space : bool
+        remove whitespace either side of each value in the dataframe
+    instrument_name_enrichment : bool
+        request additional identifier information from open-figi
+    sub_holding_keys : list
+        The sub holding keys to use for this request. Can be a list of property keys or a list of
+        columns in the dataframe to use to create sub holdings
+    holdings_adjustment_only : bool
+        Whether to use the adjust_holdings api call rather than set_holdings when working with holdings
+    thread_pool_max_workers : int
+        The maximum number of workers to use in the thread pool used by the function
+    sub_holding_keys_scope : str
+        The scope to add the sub-holding keys to
+    return_unmatched_items : bool
+        When loading transactions or holdings, a 'True' flag will return a list of the transaction or holding
+        objects where their instruments were unmatched at the time of the upsert
+        (i.e. where instrument_uid is LUID_ZZZZZZ). This parameter will be ignored for file types other than
+        transactions or holdings
+    instrument_scope : str
+        The scope to upsert to when upseting instrument
+
+    Returns
+    -------
+    responses: dict
+        The responses from loading the data into LUSID
+
+    Examples
+    --------
+
+    * Loading Instruments
+
+    .. code-block:: none
+
+        result = finbourne_sdk_utils.cocoon.load_from_data_frame(
+            api_factory=api_factory,
+            scope=scope,
+            data_frame=instr_df,
+            mapping_required=mapping["instruments"]["required"],
+            mapping_optional={},
+            file_type="instruments",
+            identifier_mapping=mapping["instruments"]["identifier_mapping"],
+            property_columns=mapping["instruments"]["properties"],
+            properties_scope=scope,
+            instrument_scope=scope
+        )
+
+    * Loading Instrument Properties
+
+    .. code-block:: none
+
+        result = finbourne_sdk_utils.cocoon.load_from_data_frame(
+            api_factory=api_factory,
+            scope=scope,
+            data_frame=strat_properties,
+            mapping_required=strat_mapping,
+            mapping_optional={},
+            file_type="instrument_property",
+            property_columns=["block tag"],
+            properties_scope=scope
+        )
+
+    * Loading Portfolios
+
+    .. code-block:: none
+
+        result = finbourne_sdk_utils.cocoon.load_from_data_frame(
+            api_factory=api_factory,
+            scope=scope,
+            data_frame=portfolios,
+            mapping_required=mapping["portfolios"]["required"],
+            mapping_optional={},
+            file_type="portfolios"
+        )
+
+    * Loading Transactions
+
+    .. code-block:: none
+
+        result = finbourne_sdk_utils.cocoon.load_from_data_frame(
+            api_factory=api_factory,
+            scope=scope,
+            data_frame=txn_df,
+            mapping_required=mapping["transactions"]["required"],
+            mapping_optional=mapping["transactions"]["optional"],
+            file_type="transactions",
+            identifier_mapping=mapping["transactions"]["identifier_mapping"],
+            property_columns=mapping["transactions"]["properties"],
+            properties_scope=scope
+        )
+    
+    * Loading Transactions with Commit Mode
+
+    .. code-block:: none
+
+        result = finbourne_sdk_utils.cocoon.load_from_data_frame(
+            api_factory=api_factory,
+            scope=scope,
+            data_frame=txn_df,
+            mapping_required=mapping["transactions"]["required"],
+            mapping_optional=mapping["transactions"]["optional"],
+            file_type="transactions_with_commit",
+            transactions_commit_mode="(Atomic|Partial)"
+            identifier_mapping=mapping["transactions"]["identifier_mapping"],
+            property_columns=mapping["transactions"]["properties"],
+            properties_scope=scope
+        )
+
+
+    * Loading Quotes
+
+    .. code-block:: none
+
+        result = lpt.load_from_data_frame(
+            api_factory=api_factory,
+            scope=scope,
+            data_frame=df_adjusted_quotes,
+            mapping_required=mapping["quotes"]["required"],
+            mapping_optional={},
+            file_type="quotes"
+        )
+
+    * loading Holdings
+
+    .. code-block:: none
+
+        result = lpt.load_from_data_frame(
+            api_factory=api_factory,
+            scope=holdings_scope,
+            data_frame=seg_df,
+            mapping_required=mapping["holdings"]["required"],
+            mapping_optional=mapping["holdings"]["optional"],
+            identifier_mapping=holdings_mapping["holdings"]["identifier_mapping"],
+            file_type="holdings"
+        )
+
+    """
+
+    # A mapping between the file type and relevant attributes e.g. domain, top_level_model etc.
+    domain_lookup = cocoon.utilities.load_json_file("config/domain_settings.json")
+
+    # Convert the file type to lower case & singular as well as checking it is of the allowed value
+    file_type = (
+        Validator(file_type, "file_type")
+        .make_singular()
+        .make_lower()
+        .check_allowed_value(list(domain_lookup.keys()))
+        .value
+    )
+
+    # Ensures that it is a single index dataframe
+    Validator(data_frame.index, "data_frame_index").check_is_not_instance(pd.MultiIndex)
+
+    # Set defaults aligned with the data type of each argument, this allows for users to provide None
+    identifier_mapping = (
+        Validator(identifier_mapping, "identifier_mapping")
+        .set_default_value_if_none(default={})
+        .discard_dict_keys_none_value()
+        .value
+    )
+
+    properties_scope = (
+        Validator(properties_scope, "properties_scope")
+        .set_default_value_if_none(default=scope)
+        .value
+    )
+
+    sub_holding_keys_scope = (
+        Validator(sub_holding_keys_scope, "sub_holding_keys_scope")
+        .set_default_value_if_none(default=properties_scope)
+        .value
+    )
+
+    instrument_scope = (
+        Validator(instrument_scope, "instrument_scope")
+        .set_default_value_if_none(default="default")
+        .value
+    )
+
+    property_columns = (
+        Validator(property_columns, "property_columns")
+        .set_default_value_if_none(default=[])
+        .value
+    )
+
+    property_columns = [
+        {"source": column, "target": column} if isinstance(column, str) else column
+        for column in property_columns
+    ]
+
+    Validator(
+        property_columns, "property_columns"
+    ).check_entries_are_strings_or_dict_containing_key("source")
+
+    sub_holding_keys = (
+        Validator(sub_holding_keys, "sub_holding_keys")
+        .set_default_value_if_none(default=[])
+        .value
+    )
+
+    batch_size = (
+        Validator(batch_size, "batch_size")
+        .set_default_value_if_none(domain_lookup[file_type]["default_batch_size"])
+        .override_value(
+            not domain_lookup[file_type]["batch_allowed"],
+            domain_lookup[file_type]["default_batch_size"],
+        )
+        .value
+    )
+
+    # Discard mappings where the provided value is None
+    mapping_required = (
+        Validator(mapping_required, "mapping_required")
+        .discard_dict_keys_none_value()
+        .value
+    )
+
+    mapping_optional = (
+        Validator(mapping_optional, "mapping_optional")
+        .discard_dict_keys_none_value()
+        .value
+    )
+
+    required_call_attributes = domain_lookup[file_type]["required_call_attributes"]
+    if "scope" in required_call_attributes:
+        required_call_attributes.remove("scope")
+
+    # Check that all required parameters exist
+    Validator(
+        required_call_attributes, "required_attributes_for_call"
+    ).check_subset_of_list(list(mapping_required.keys()), "required_mapping")
+
+    # Verify that all the required attributes for this top level model exist in the provided required mapping
+    cocoon.utilities.verify_all_required_attributes_mapped(
+        mapping=mapping_required,
+        model_object_name=domain_lookup[file_type]["top_level_model"],
+        exempt_attributes=["identifiers", "properties", "instrument_identifiers"],
+    )
+
+    # Create the thread pool to use with the async_tools.run_in_executor decorator to make sync functions awaitable
+    thread_pool = ThreadPool(thread_pool_max_workers).thread_pool
+
+    if instrument_name_enrichment:
+        loop = cocoon.async_tools.start_event_loop_new_thread()
+
+        data_frame, mapping_required = asyncio.run_coroutine_threadsafe(
+            cocoon.instruments.enrich_instruments(
+                api_factory=api_factory,
+                data_frame=data_frame,
+                instrument_identifier_mapping=identifier_mapping,
+                mapping_required=mapping_required,
+                constant_prefix="$",
+                **{"thread_pool": thread_pool},
+            ),
+            loop,
+        ).result()
+
+        # Stop the additional event loop
+        cocoon.async_tools.stop_event_loop_new_thread(loop)
+
+    """
+    Unnest and populate defaults where a mapping is provided with column and/or default fields in a nested dictionary
+    
+    e.g.
+    {'name': {
+        'column': 'instrument_name',
+        'default': 'unknown_name'
+        }
+    }
+    
+    rather than simply
+    {'name': 'instrument_name'}
+    """
+    (
+        data_frame,
+        mapping_required,
+    ) = cocoon.utilities.handle_nested_default_and_column_mapping(
+        data_frame=data_frame, mapping=mapping_required, constant_prefix="$"
+    )
+    (
+        data_frame,
+        mapping_optional,
+    ) = cocoon.utilities.handle_nested_default_and_column_mapping(
+        data_frame=data_frame, mapping=mapping_optional, constant_prefix="$"
+    )
+
+    # Get all the DataFrame columns as well as those that contain at least one null value
+    data_frame_columns = list(data_frame.columns.values)
+    nan_columns = [
+        column for column in data_frame_columns if data_frame[column].isna().any()
+    ]
+
+    # Validate that none of the provided columns are missing or invalid
+    Validator(
+        mapping_required, "mapping_required"
+    ).get_dict_values().filter_list_using_first_character("$").check_subset_of_list(
+        data_frame_columns, "DataFrame Columns"
+    ).check_no_intersection_with_list(
+        nan_columns, "Columns with Missing Values"
+    )
+
+    Validator(
+        mapping_optional, "mapping_optional"
+    ).get_dict_values().filter_list_using_first_character("$").check_subset_of_list(
+        data_frame_columns, "DataFrame Columns"
+    )
+
+    Validator(
+        identifier_mapping, "identifier_mapping"
+    ).get_dict_values().filter_list_using_first_character("$").check_subset_of_list(
+        data_frame_columns, "DataFrame Columns"
+    )
+
+    source_columns = [column["source"] for column in property_columns]
+    Validator(source_columns, "property_columns").check_subset_of_list(
+        data_frame_columns, "DataFrame Columns"
+    )
+
+    # Converts higher level data types such as dictionaries and lists to strings
+    data_frame = data_frame.applymap(cocoon.utilities.convert_cell_value_to_string)
+
+    if remove_white_space:
+        column_list = [source_columns]
+        for col in [mapping_optional, mapping_required, identifier_mapping]:
+            column_list.append(col.values())
+
+        column_list = list(set([item for sublist in column_list for item in sublist]))
+        data_frame = strip_whitespace(data_frame, column_list)
+
+    # Get the types of the attributes on the top level model for this request
+    
+    open_api_types = get_attributes_and_types(getattr(
+        lusid.models, domain_lookup[file_type]["top_level_model"]
+    ))
+
+    # If there is a sub_holding_keys attribute and it has a dict type this means the sub_holding_keys
+    # need to have a property definition and be populated with values from the provided dataframe columns
+    if (
+            "sub_holding_keys" in open_api_types.keys()
+            and "Mapping[" in open_api_types["sub_holding_keys"]
+    ):
+        Validator(sub_holding_keys, "sub_holding_key_columns").check_subset_of_list(
+            data_frame_columns, "DataFrame Columns"
+        )
+
+        # Check for and create missing property definitions for the sub-holding-keys
+        data_frame = cocoon.properties.create_missing_property_definitions_from_file(
+            api_factory=api_factory,
+            properties_scope=sub_holding_keys_scope,
+            domain="Transaction",
+            data_frame=data_frame,
+            property_columns=[{"source": key} for key in sub_holding_keys],
+        )
+
+    # Check for and create missing property definitions for the properties
+    if domain_lookup[file_type]["domain"] is not None:
+        data_frame = cocoon.properties.create_missing_property_definitions_from_file(
+            api_factory=api_factory,
+            properties_scope=properties_scope,
+            domain=domain_lookup[file_type]["domain"],
+            data_frame=data_frame,
+            property_columns=property_columns,
+        )
+
+    # If the transaction contains subholding keys which aren't defined in the portfolio. We first create the
+    # properties that don't already exist, then we make the properties sub-holding keys in the portfolios.
+    if (
+            file_type in ("transaction", "transactions_commit_mode")
+            and sub_holding_keys is not None
+            and sub_holding_keys != []
+    ):
+        # if the SHK key is written in {domain}/{scope}/{code} form we extract the code since when we add it to
+        # the properties there will be issues due to different formats. Also, there are issues with the
+        # create_missing_property_definitions_from_file function as it extracts data from columns using the
+        # sub-holding keys.
+        sub_holding_keys_codes = [
+            key if "/" not in key else key.split("/")[2] for key in sub_holding_keys
+        ]
+
+        # Check for and create missing property definitions for the sub-holding-keys
+        data_frame = cocoon.properties.create_missing_property_definitions_from_file(
+            api_factory=api_factory,
+            properties_scope=properties_scope,
+            domain="Transaction",
+            data_frame=data_frame,
+            property_columns=[{"source": key} for key in sub_holding_keys_codes],
+        )
+
+        transaction_portfolio_api = api_factory.build(
+            lusid.api.TransactionPortfoliosApi
+        )
+
+        # Add subholding keys to the portfolios we are going to apply the transactions to
+        for code in set(data_frame[mapping_required["code"]]):
+            transaction_portfolio_api.patch_portfolio_details(
+                scope,
+                code,
+                [
+                    {
+                        "value": cocoon.properties._infer_full_property_keys(
+                            partial_keys=sub_holding_keys,
+                            properties_scope=properties_scope,
+                            domain="Transaction",
+                        ),
+                        "path": "/subHoldingKeys",
+                        "op": "add",
+                    }
+                ],
+            )
+
+        # Add sub-holding keys to the properties, so it is created for each transaction.
+        property_columns += [
+            {"source": sub_holding_key, "target": sub_holding_key}
+            for sub_holding_key in sub_holding_keys_codes
+        ]
+
+    # Start a new event loop in a new thread, this is required to run inside a Jupyter notebook
+    loop = cocoon.async_tools.start_event_loop_new_thread()
+
+    # Keyword arguments to be used in requests to the LUSID API
+    keyword_arguments = {
+        "scope": scope,
+        # This handles that identifiers need to be specified differently based on the request type, allowing users
+        # to provide either the entire key e.g. "Instrument/default/Figi" or just the code "Figi" for any request
+        "full_key_format": domain_lookup[file_type]["full_key_format"],
+        # Gets the allowed unique identifiers
+        "unique_identifiers": cocoon.instruments.get_unique_identifiers(
+            api_factory=api_factory
+        ),
+        "transactions_commit_mode": transactions_commit_mode,
+        "holdings_adjustment_only": holdings_adjustment_only,
+        "thread_pool": thread_pool,
+        "instrument_scope": instrument_scope,
+    }
+
+    # Get the responses from LUSID
+    logging.debug("constructing batches...")
+    responses = asyncio.run_coroutine_threadsafe(
+        _construct_batches(
+            api_factory=api_factory,
+            data_frame=data_frame,
+            mapping_required=mapping_required,
+            mapping_optional=mapping_optional,
+            property_columns=property_columns,
+            properties_scope=properties_scope,
+            instrument_identifier_mapping=identifier_mapping,
+            batch_size=batch_size,
+            file_type=file_type,
+            domain_lookup=domain_lookup,
+            sub_holding_keys=sub_holding_keys,
+            sub_holding_keys_scope=sub_holding_keys_scope,
+            return_unmatched_items=return_unmatched_items,
+            **keyword_arguments,
+        ),
+        loop,
+    ).result()
+
+    # Stop the additional event loop
+    cocoon.async_tools.stop_event_loop_new_thread(loop)
+
+    return {file_type + "s": responses}