wbfdm 1.43.1__py2.py3-none-any.whl

wbfdm/contrib/qa/jinja2/qa/sql/instruments.sql

@@ -0,0 +1,100 @@
+SELECT
+    'qa-ds2-security' AS 'source',
+    security.DsSecCode AS 'source_id',
+    security.IsMajorSec AS 'is_primary',
+    company.DsCmpyCode AS 'parent_id',
+    coalesce(cmp_ref.PrimaryName, company.DsCmpyName) AS 'name',
+    company.CmpyCtryCode AS 'country_id',
+    exchange_qt.ISOCurrCode AS 'currency_id',
+    country_qt.TypeCode AS 'instrument_type_id',
+    rkd_instrument.ISIN AS 'isin',
+    rkd_instrument.Ticker AS 'ticker',
+    rkd_instrument.RIC AS 'refinitiv_identifier_code',
+	country_qt.DsMnem AS 'refinitiv_mnemonic_code',
+    rkd_instrument.Cusip AS 'cusip',
+    rkd_instrument.Sedol AS 'sedol',
+    filing.TxtInfo AS 'description',
+    rkd_cmp_det.Employees AS 'employees',
+    (select top 1 concat('00', phone.CtryPh, phone.City, phone.PhoneNo) from RKDFndCmpPhone AS phone where phone.Code = rkd_instrument.Code AND PhTypeCode = 1) AS 'phone',
+    (select top 1 web.URL from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode = 1) AS 'primary_url',
+    (select string_agg(web.URL, ',') from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode <> 1) AS 'additional_urls',
+    concat(rkd_cmp_det.StAdd1, ', ', rkd_cmp_det.Post, ', ', rkd_cmp_det.City) AS 'headquarter_address',
+    convert(Date, COALESCE(rkd_cmp_det.PublicSince, (SELECT MIN(MarketDate) FROM DS2PrimQtPrc WHERE InfoCode = exchange_qt.InfoCode))) AS 'inception_date',
+    convert(Date, security.DelistDate) AS 'delisted_date',
+    convert(Date, cmp_ref.LatestFinAnnDt) AS 'last_annual_report',
+    convert(Date, cmp_ref.LatestFinIntmDt) AS 'last_interim_report',
+    -- MarketData DL
+    exchange_qt.InfoCode AS 'quote_code',
+    exchange_qt.ExchIntCode AS 'exchange_code',
+    -- Fundamental DL
+    rkd_instrument.Code AS 'rkd_code',
+    -- Forecast DL
+    ibes_mapping.EstPermID AS 'ibes_code'
+
+FROM DS2Security AS security
+
+LEFT JOIN DS2Company AS company
+	ON security.DsCmpyCode = company.DsCmpyCode
+
+LEFT JOIN DS2ExchQtInfo AS exchange_qt
+    ON exchange_qt.InfoCode = security.PrimQtInfoCode
+    AND exchange_qt.IsPrimExchQt = 'Y'
+
+LEFT JOIN DS2CtryQtInfo AS country_qt
+    ON exchange_qt.InfoCode = country_qt.InfoCode
+
+LEFT JOIN vw_SecurityMappingX AS mappingX
+    ON mappingX.vencode = security.PrimQtInfoCode
+    AND mappingX.ventype = 33
+    AND mappingX.rank = 1
+    AND mappingX.StartDate = (
+        SELECT MAX(I.StartDate)
+        FROM vw_SecurityMappingX AS I
+        WHERE I.typ = mappingX.typ AND I.vencode = mappingX.vencode AND I.ventype = mappingX.ventype
+    )
+
+LEFT JOIN vw_SecurityMappingX AS mappingRKD
+    ON mappingX.seccode = mappingRKD.seccode
+    AND mappingX.typ = mappingRKD.typ
+    AND mappingRKD.ventype = 26
+    AND mappingRKD.rank = 1
+    AND mappingRKD.StartDate = (
+        SELECT MAX(I.StartDate)
+        FROM vw_SecurityMappingX AS I
+        WHERE I.typ = mappingRKD.typ AND I.vencode = mappingRKD.vencode AND I.ventype = mappingRKD.ventype
+    )
+
+LEFT JOIN RKDFndCmpRefIssue AS rkd_instrument
+    ON rkd_instrument.IssueCode = mappingRKD.vencode
+
+LEFT JOIN RKDFndCmpDet AS rkd_cmp_det
+    ON rkd_cmp_det.Code = rkd_instrument.Code
+
+LEFT JOIN RKDFndCmpRef AS cmp_ref
+    ON cmp_ref.Code = rkd_instrument.Code
+
+LEFT JOIN RKDFndCmpFiling AS filing
+    ON filing.Code = rkd_instrument.Code
+    AND filing.TxtInfoTypeCode = 2
+
+LEFT JOIN vw_IBES2Mapping AS ibes_mapping
+    ON ibes_mapping.SecCode = mappingX.seccode
+    AND ibes_mapping.typ = mappingX.typ
+    AND ibes_mapping.Exchange = (
+        CASE
+            WHEN ibes_mapping.typ = 6 THEN 0
+            WHEN ibes_mapping.typ = 1 THEN 1
+        END
+    )
+
+{% if source_id %}
+where security.DsSecCode = {{ source_id }}
+{% endif %}
+-- where company.CmpyCtryCode = 'IS'
+
+ORDER BY company.DsCmpyCode
+
+{% if (offset == 0 or offset) and batch %}
+offset {{ offset|sqlsafe }} rows
+fetch next {{ batch|sqlsafe }} rows only
+{% endif %}

wbfdm/contrib/qa/jinja2/qa/sql/quotes.sql

@@ -0,0 +1,98 @@
+SELECT
+    'qa-ds2-quote' AS 'source',
+    CONCAT(exchange_qt.InfoCode, '-', exchange_qt.ExchIntCode) AS 'source_id',
+    security.DsSecCode AS 'parent_id',
+    case
+        when exchange_qt.IsPrimExchQt = 'Y' and country_qt.IsPrimQt = 1 and security.DelistDate is null then 'Y'
+        else 'N'
+    end as is_primary,
+    coalesce(cmp_ref.PrimaryName, exchange_qt.DsQtName) AS 'name',
+    country_qt.Region AS 'country_id',
+    exchange_qt.ISOCurrCode AS 'currency_id',
+    'quote' AS 'instrument_type_id',
+    rkd_instrument.ISIN AS 'isin',
+    rkd_instrument.Ticker AS 'ticker',
+    rkd_instrument.RIC AS 'refinitiv_identifier_code',
+	country_qt.DsMnem AS 'refinitiv_mnemonic_code',
+    rkd_instrument.Cusip AS 'cusip',
+    rkd_instrument.Sedol AS 'sedol',
+    filing.TxtInfo AS 'description',
+    rkd_cmp_det.Employees AS 'employees',
+    (select top 1 concat('00', phone.CtryPh, phone.City, phone.PhoneNo) from RKDFndCmpPhone AS phone where phone.Code = rkd_instrument.Code AND PhTypeCode = 1) AS 'phone',
+    (select top 1 web.URL from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode = 1) AS 'primary_url',
+    (select string_agg(web.URL, ',') from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode <> 1) AS 'additional_urls',
+    concat(rkd_cmp_det.StAdd1, ', ', rkd_cmp_det.Post, ', ', rkd_cmp_det.City) AS 'headquarter_address',
+    convert(Date, COALESCE(rkd_cmp_det.PublicSince, (SELECT MIN(MarketDate) FROM DS2PrimQtPrc WHERE InfoCode = exchange_qt.InfoCode))) AS 'inception_date',
+	convert(Date, security.DelistDate) AS 'delisted_date',
+    convert(Date, cmp_ref.LatestFinAnnDt) AS 'last_annual_report',
+    convert(Date, cmp_ref.LatestFinIntmDt) AS 'last_interim_report',
+    -- MarketData DL
+    exchange_qt.InfoCode AS 'quote_code',
+    exchange_qt.ExchIntCode AS 'exchange_code',
+    -- Fundamental DL
+    rkd_instrument.Code AS 'rkd_code',
+    -- Forecast DL
+    ibes_mapping.EstPermID AS 'ibes_code'
+
+FROM DS2ExchQtInfo AS exchange_qt
+
+LEFT JOIN DS2CtryQtInfo AS country_qt
+    ON exchange_qt.InfoCode = country_qt.InfoCode
+
+LEFT JOIN DS2Security AS security
+    ON country_qt.DsSecCode = security.DsSecCode
+
+LEFT JOIN vw_SecurityMappingX AS mappingX
+    ON mappingX.vencode = security.PrimQtInfoCode
+    AND mappingX.ventype = 33
+    AND mappingX.rank = 1
+    AND mappingX.StartDate = (
+        SELECT MAX(I.StartDate)
+        FROM vw_SecurityMappingX AS I
+        WHERE I.typ = mappingX.typ AND I.vencode = mappingX.vencode AND I.ventype = mappingX.ventype
+    )
+
+LEFT JOIN vw_SecurityMappingX AS mappingRKD
+    ON mappingX.seccode = mappingRKD.seccode
+    AND mappingX.typ = mappingRKD.typ
+    AND mappingRKD.ventype = 26
+    AND mappingRKD.rank = 1
+    AND mappingRKD.StartDate = (
+        SELECT MAX(I.StartDate)
+        FROM vw_SecurityMappingX AS I
+        WHERE I.typ = mappingRKD.typ AND I.vencode = mappingRKD.vencode AND I.ventype = mappingRKD.ventype
+	)
+
+LEFT JOIN RKDFndCmpRefIssue AS rkd_instrument
+    ON rkd_instrument.IssueCode = mappingRKD.vencode
+
+LEFT JOIN RKDFndCmpDet AS rkd_cmp_det
+    ON rkd_cmp_det.Code = rkd_instrument.Code
+
+LEFT JOIN RKDFndCmpRef AS cmp_ref
+    ON cmp_ref.Code = rkd_instrument.Code
+
+LEFT JOIN RKDFndCmpFiling AS filing
+    ON filing.Code = rkd_instrument.Code
+    AND filing.TxtInfoTypeCode = 2
+
+LEFT JOIN vw_IBES2Mapping AS ibes_mapping
+    ON ibes_mapping.SecCode = mappingX.seccode
+    AND ibes_mapping.typ = mappingX.typ
+    AND ibes_mapping.Exchange = (
+        CASE
+            WHEN ibes_mapping.typ = 6 THEN 0
+            WHEN ibes_mapping.typ = 1 THEN 1
+        END
+    )
+
+{% if source_id %}
+where CONCAT(exchange_qt.InfoCode, '-', exchange_qt.ExchIntCode) = {{ source_id }}
+{% endif %}
+
+ORDER BY exchange_qt.InfoCode, exchange_qt.ExchIntCode
+
+{% if (offset == 0 or offset) and batch %}
+offset {{ offset|sqlsafe }} rows
+fetch next {{ batch|sqlsafe }} rows only
+{% endif %}

wbfdm/contrib/qa/sync/exchanges.py

@@ -0,0 +1,70 @@
+import pytz
+from django.conf import settings
+from django.db import connections
+from django.db.models import Max
+from wbcore.contrib.dataloader.utils import dictfetchall, dictfetchone
+from wbcore.contrib.geography.models import Geography
+from wbcore.utils.cache import mapping
+from wbfdm.models import Exchange
+from wbfdm.sync.abstract import Sync
+
+
+class QAExchangeSync(Sync[Exchange]):
+    SOURCE = "qa-ds2-exchange"
+
+    def update(self):
+        sql = "SELECT ExchIntCode AS source_id, ExchName AS name, ExchCtryCode AS country_id, ExchMnem AS refinitiv_mnemonic FROM DS2Exchange"
+        Exchange.objects.bulk_create(
+            map(
+                lambda data: Exchange(
+                    source=self.SOURCE,
+                    source_id=data["source_id"],
+                    name=data["name"],
+                    country_id=mapping(Geography.countries, "code_2").get(data["country_id"]),
+                    refinitiv_mnemonic=data["refinitiv_mnemonic"],
+                ),
+                dictfetchall(connections["qa"].cursor().execute(sql)),
+            ),
+            update_conflicts=True,
+            update_fields=["name", "country", "refinitiv_mnemonic"],
+            unique_fields=["source", "source_id"],
+        )
+
+    def update_or_create_item(self, external_id: int) -> Exchange:
+        defaults = self.get_item(external_id)
+        defaults["country_id"] = mapping(Geography.countries, "code_2").get(defaults["country_id"])
+        exchange, _ = Exchange.objects.update_or_create(
+            source=self.SOURCE,
+            source_id=external_id,
+            defaults=defaults,
+        )
+        return exchange
+
+    def update_item(self, item: Exchange) -> Exchange:
+        return self.update_or_create_item(item.source_id)
+
+    def get_item(self, external_id: int) -> dict:
+        sql = "SELECT ExchName AS name, ExchCtryCode AS country_id, ExchMnem AS refinitiv_mnemonic FROM DS2Exchange WHERE ExchIntCode = %s"
+        return dictfetchone(connections["qa"].cursor().execute(sql, (external_id,)))
+
+    def trigger_partial_update(self):
+        max_last_updated = (
+            Exchange.objects.filter(source=self.SOURCE)
+            .aggregate(max_last_updated=Max("last_updated"))
+            .get("max_last_updated")
+        )
+        if max_last_updated is None:
+            self.update()
+        else:
+            with connections["qa"].cursor() as cursor:
+                cursor.execute(
+                    "SELECT MAX(last_user_update) FROM sys.dm_db_index_usage_stats WHERE OBJECT_NAME(object_id) = 'DS2Exchange_changes'"
+                )
+                max_last_updated_qa = (
+                    pytz.timezone(settings.TIME_ZONE).localize(result[0]) if (result := cursor.fetchone()) else None
+                )
+                if max_last_updated_qa and max_last_updated_qa > max_last_updated:
+                    for _, exchange_id in cursor.execute(
+                        "SELECT UpdateFlag_, ExchIntCode FROM DS2Exchange_changes"
+                    ).fetchall():
+                        self.update_or_create_item(exchange_id)

wbfdm/contrib/qa/sync/instruments.py

@@ -0,0 +1,94 @@
+from django.db import connections
+from wbfdm.contrib.qa.sync.utils import (
+    get_item,
+    trigger_partial_update,
+    update_instruments,
+    update_or_create_item,
+)
+from wbfdm.models import Instrument
+from wbfdm.sync.abstract import Sync
+
+
+class QACompanySync(Sync[Instrument]):
+    SOURCE = "qa-ds2-company"
+
+    def update(self, **kwargs):
+        update_instruments("qa/sql/companies.sql", **kwargs)
+
+    def update_or_create_item(self, external_id: int) -> Instrument:
+        return update_or_create_item(external_id, self.get_item, self.SOURCE)
+
+    def update_item(self, item: Instrument) -> Instrument:
+        return self.update_or_create_item(item.source_id)
+
+    def get_item(self, external_id: int) -> dict:
+        return get_item(external_id, "qa/sql/companies.sql")
+
+    def trigger_partial_update(self):
+        trigger_partial_update(
+            Instrument.objects.filter(source=self.SOURCE),
+            "last_update",
+            "DsCmpyCode",
+            "DS2Company_changes",
+            self.update,
+            self.update_or_create_item,
+        )
+
+
+class QAInstrumentSync(Sync[Instrument]):
+    SOURCE = "qa-ds2-security"
+
+    def update(self, **kwargs):
+        update_instruments("qa/sql/instruments.sql", parent_source="qa-ds2-company", **kwargs)
+
+    def update_or_create_item(self, external_id: int) -> Instrument:
+        return update_or_create_item(external_id, self.get_item, self.SOURCE, "qa-ds2-company")
+
+    def update_item(self, item: Instrument) -> Instrument:
+        return self.update_or_create_item(item.source_id)
+
+    def get_item(self, external_id: int) -> dict:
+        return get_item(external_id, "qa/sql/instruments.sql")
+
+    def trigger_partial_update(self):
+        trigger_partial_update(
+            Instrument.objects.filter(source=self.SOURCE),
+            "last_update",
+            "DsSecCode",
+            "DS2Security_changes",
+            self.update,
+            self.update_or_create_item,
+        )
+
+
+class QAQuoteSync(Sync[Instrument]):
+    SOURCE = "qa-ds2-quote"
+
+    def update(self, **kwargs):
+        count = connections["qa"].cursor().execute("SELECT COUNT(*) FROM DS2ExchQtInfo").fetchone()[0]
+        for offset in range(0, count, 100_000):
+            update_instruments(
+                "qa/sql/quotes.sql",
+                parent_source="qa-ds2-security",
+                context={"offset": offset, "batch": 100_000},
+                **kwargs,
+            )
+
+    def update_or_create_item(self, external_id: int) -> Instrument:
+        return update_or_create_item(external_id, self.get_item, self.SOURCE, "qa-ds2-security")
+
+    def update_item(self, item: Instrument) -> Instrument:
+        return self.update_or_create_item(item.source_id)
+
+    def get_item(self, external_id: int) -> dict:
+        return get_item(external_id, "qa/sql/quotes.sql")
+
+    def trigger_partial_update(self):
+        trigger_partial_update(
+            Instrument.objects.filter(source=self.SOURCE),
+            "last_update",
+            "CONCAT(InfoCode, '-', ExchIntCode)",
+            "DS2ExchQtInfo_changes",
+            self.update,
+            self.update_or_create_item,
+        )

wbfdm/contrib/qa/sync/utils.py

@@ -0,0 +1,241 @@
+from contextlib import suppress
+from typing import Callable
+
+import pytz
+from django.conf import settings
+from django.db import connections
+from django.db.models import Max, QuerySet
+from django.template.loader import get_template
+from jinjasql import JinjaSql  # type: ignore
+from tqdm import tqdm
+from wbcore.contrib.currency.models import Currency
+from wbcore.contrib.dataloader.utils import dictfetchall, dictfetchone
+from wbcore.contrib.geography.models import Geography
+from wbcore.utils.cache import mapping
+from wbfdm.models.exchanges.exchanges import Exchange
+from wbfdm.models.instruments.instruments import Instrument, InstrumentType
+
+BATCH_SIZE: int = 10000
+instrument_type_map = {
+    "ADR": "american_depository_receipt",
+    "CF": "close_ended_fund",
+    "EQ": "equity",
+    "ET": "exchange_traded_fund",
+    "ETC": "exchange_traded_commodity",
+    "ETN": "exchange_traded_note",
+    "EWT": "warrant",
+    "GDR": "global_depository_receipt",
+    "GNSH": "genussschein",
+    "INVT": "investment_trust",
+    "NVDR": "non_voting_depository_receipt",
+    "PREF": "preference_share",
+    "UT": "abc",
+}
+
+
+def get_dataloader_parameters(
+    quote_code: str | None = None,
+    exchange_code: str | None = None,
+    rkd_code: str | None = None,
+    ibes_code: str | None = None,
+) -> dict:
+    dataloader_parameters = dict()
+    if quote_code:
+        dataloader_parameters["adjustments"] = {
+            "path": "wbfdm.contrib.qa.dataloaders.adjustments.DatastreamAdjustmentsDataloader",
+            "parameters": quote_code,
+        }
+        dataloader_parameters["corporate_actions"] = {
+            "path": "wbfdm.contrib.qa.dataloaders.corporate_actions.DatastreamCorporateActionsDataloader",
+            "parameters": quote_code,
+        }
+        if exchange_code:
+            dataloader_parameters["market_data"] = {
+                "path": "wbfdm.contrib.qa.dataloaders.market_data.DatastreamMarketDataDataloader",
+                "parameters": [
+                    quote_code,
+                    exchange_code,
+                ],
+            }
+
+    if rkd_code:
+        dataloader_parameters["officers"] = {
+            "path": "wbfdm.contrib.qa.dataloaders.officers.RKDOfficersDataloader",
+            "parameters": rkd_code,
+        }
+        dataloader_parameters["statements"] = {
+            "path": "wbfdm.contrib.qa.dataloaders.statements.RKDStatementsDataloader",
+            "parameters": rkd_code,
+        }
+
+    if ibes_code:
+        dataloader_parameters["financials"] = {
+            "path": "wbfdm.contrib.qa.dataloaders.financials.IBESFinancialsDataloader",
+            "parameters": ibes_code,
+        }
+        dataloader_parameters["reporting_dates"] = {
+            "path": "wbfdm.contrib.qa.dataloaders.reporting_dates.IbesReportingDateDataloader",
+            "parameters": ibes_code,
+        }
+
+    return dataloader_parameters
+
+
+def convert_data(data, parent_source: str | None = None) -> dict:
+    if len(data.keys()) == 0:
+        return data
+    data["country_id"] = mapping(Geography.countries, "code_2").get(data["country_id"])
+    data["currency_id"] = mapping(Currency.objects).get(data["currency_id"])
+    if instrument_type_id := data.pop("instrument_type_id"):
+        try:
+            data["instrument_type_id"] = mapping(InstrumentType.objects)[
+                instrument_type_map.get(instrument_type_id, instrument_type_id)
+            ]
+        except KeyError:
+            data["instrument_type_id"] = InstrumentType.objects.get_or_create(
+                key=instrument_type_id.lower(),
+                defaults={"name": instrument_type_id.title(), "short_name": instrument_type_id.title()},
+            )[0].pk
+    data["exchange_id"] = mapping(Exchange.objects, "source_id", source="qa-ds2-exchange").get(
+        str(data["exchange_code"])
+    )
+    data["dl_parameters"] = get_dataloader_parameters(
+        data.pop("quote_code"), data.pop("exchange_code"), data.pop("rkd_code"), data.pop("ibes_code")
+    )
+    data["additional_urls"] = urls.split(",") if (urls := data.get("additional_urls")) else []
+    if parent_source:
+        data["parent_id"] = mapping(Instrument.objects, "source_id", source=parent_source).get(str(data["parent_id"]))
+    if (is_primary := data.get("is_primary")) and isinstance(is_primary, str):
+        data["is_primary"] = is_primary.lower().strip() == "y"
+    data.pop("phone")
+    return data
+
+
+def get_instrument_from_data(data, parent_source: str | None = None) -> Instrument | None:
+    if len(data.keys()) == 0:
+        return None
+    instrument = Instrument(**convert_data(data, parent_source))
+    instrument.pre_save()
+    instrument.computed_str = instrument.compute_str()
+    if (
+        instrument.name and instrument.currency and instrument.instrument_type
+    ):  # we return an instrument only if it contains the basic name, currency and type
+        return instrument
+
+
+def _bulk_create_instruments_chunk(instruments: list[Instrument], update_unique_identifiers: bool = False):
+    update_fields = [
+        "name",
+        "dl_parameters",
+        "description",
+        "country",
+        "currency",
+        "parent",
+        "primary_url",
+        "additional_urls",
+        "headquarter_address",
+        "inception_date",
+        "delisted_date",
+        "exchange",
+        "instrument_type",
+        "computed_str",
+        "search_vector",
+        "trigram_search_vector",
+        "is_primary",
+    ]
+    bulk_update_kwargs = {"update_conflicts": True}
+    if update_unique_identifiers:
+        update_fields.extend(
+            [
+                "refinitiv_identifier_code",
+                "refinitiv_mnemonic_code",
+                "isin",
+                "sedol",
+                "valoren",
+                "cusip",
+            ]
+        )
+        bulk_update_kwargs = {"ignore_conflicts": True}
+    Instrument.objects.bulk_create(
+        instruments, update_fields=update_fields, unique_fields=["source", "source_id"], **bulk_update_kwargs
+    )
+
+
+def update_instruments(sql_name: str, parent_source: str | None = None, context=None, debug: bool = False, **kwargs):
+    template = get_template(sql_name, using="jinja").template  # type: ignore
+    if context is None:
+        context = {}
+    query, params = JinjaSql(param_style="format").prepare_query(template, context)
+    instruments = []
+
+    gen = dictfetchall(connections["qa"].cursor().execute(query, params))
+    if debug:
+        gen = tqdm(gen)
+    # we update in batch to be sure to not exhaust resources
+    for row in gen:
+        with suppress(TypeError):  # we don't fail if the given data doesn't satisfy the schema
+            if instrument := get_instrument_from_data(row, parent_source=parent_source):
+                instruments.append(instrument)
+
+        if len(instruments) >= BATCH_SIZE:
+            _bulk_create_instruments_chunk(instruments, **kwargs)
+            instruments = []
+
+    _bulk_create_instruments_chunk(instruments, **kwargs)
+
+
+def update_or_create_item(
+    external_id: int, get_item: Callable, source: str, parent_source: str | None = None
+) -> Instrument:
+    defaults = convert_data(get_item(external_id), parent_source=parent_source)
+
+    dl_parameters = defaults.pop("dl_parameters", {})
+    defaults.pop("source", None)
+    defaults.pop("source_id", None)
+
+    instrument, _ = Instrument.objects.update_or_create(
+        source=source,
+        source_id=external_id,
+        defaults=defaults,
+    )
+
+    for key, value in dl_parameters.items():
+        instrument.dl_parameters[key] = value
+
+    instrument.save()
+
+    return instrument
+
+
+def get_item(external_id: int, template_name: str) -> dict:
+    template = get_template(template_name, using="jinja").template  # type: ignore
+    query, params = JinjaSql(param_style="format").prepare_query(template, {"source_id": external_id})
+    return dictfetchone(connections["qa"].cursor().execute(query, params))
+
+
+def trigger_partial_update(
+    queryset: QuerySet,
+    last_update_field: str,
+    id_field: str,
+    table_change_name: str,
+    update: Callable,
+    update_or_create_item: Callable,
+):
+    max_last_updated = queryset.aggregate(max_last_updated=Max(last_update_field)).get("max_last_updated")
+
+    if max_last_updated is None:
+        update()
+
+    else:
+        with connections["qa"].cursor() as cursor:
+            cursor.execute(
+                f"SELECT MAX(last_user_update) FROM sys.dm_db_index_usage_stats WHERE OBJECT_NAME(object_id) = '{table_change_name}'"
+            )
+            max_last_updated_qa = (
+                pytz.timezone(settings.TIME_ZONE).localize(result[0]) if (result := cursor.fetchone()) else None
+            )
+            if max_last_updated_qa and max_last_updated_qa > max_last_updated:
+                for _, security_id in cursor.execute(
+                    f"SELECT UpdateFlag_, {id_field} FROM {table_change_name}"
+                ).fetchall():
+                    update_or_create_item(security_id)