wbfdm 2.2.4__py2.py3-none-any.whl → 2.2.5__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wbfdm might be problematic. Click here for more details.
- wbfdm/backends/dto.py +36 -0
- wbfdm/contrib/dsws/dataloaders/market_data.py +130 -0
- wbfdm/contrib/qa/sync/exchanges.py +70 -0
- wbfdm/contrib/qa/sync/instruments.py +94 -0
- wbfdm/contrib/qa/sync/utils.py +241 -0
- {wbfdm-2.2.4.dist-info → wbfdm-2.2.5.dist-info}/METADATA +1 -1
- {wbfdm-2.2.4.dist-info → wbfdm-2.2.5.dist-info}/RECORD +8 -3
- {wbfdm-2.2.4.dist-info → wbfdm-2.2.5.dist-info}/WHEEL +0 -0
wbfdm/backends/dto.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from datetime import date
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass
|
|
6
|
+
class PriceDTO:
|
|
7
|
+
pk: str
|
|
8
|
+
instrument: int
|
|
9
|
+
date: date
|
|
10
|
+
open: float
|
|
11
|
+
close: float
|
|
12
|
+
high: float
|
|
13
|
+
low: float
|
|
14
|
+
volume: float
|
|
15
|
+
market_capitalization: float
|
|
16
|
+
outstanding_shares: float
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def net_value(self) -> float:
|
|
20
|
+
return self.close
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class AdjustmentDTO:
|
|
25
|
+
pk: str
|
|
26
|
+
date: date
|
|
27
|
+
adjustment_factor: float
|
|
28
|
+
cumulative_adjustment_factor: float
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class DividendDTO:
|
|
33
|
+
pk: str
|
|
34
|
+
rate: float
|
|
35
|
+
ex_dividend_date: date
|
|
36
|
+
payment_date: date
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
from contextlib import suppress
|
|
2
|
+
from datetime import date
|
|
3
|
+
from typing import Iterator
|
|
4
|
+
|
|
5
|
+
from DatastreamPy import DSUserObjectFault
|
|
6
|
+
from django.conf import settings
|
|
7
|
+
from pandas.tseries.offsets import BDay
|
|
8
|
+
from wbcore.contrib.dataloader.dataloaders import Dataloader
|
|
9
|
+
from wbfdm.dataloaders.protocols import MarketDataProtocol
|
|
10
|
+
from wbfdm.dataloaders.types import MarketDataDict
|
|
11
|
+
from wbfdm.enums import Frequency, MarketData
|
|
12
|
+
|
|
13
|
+
from ..client import Client
|
|
14
|
+
|
|
15
|
+
FIELD_MAP = {
|
|
16
|
+
"close": "P",
|
|
17
|
+
"open": "PO",
|
|
18
|
+
"high": "PH",
|
|
19
|
+
"low": "PL",
|
|
20
|
+
"bid": "PB",
|
|
21
|
+
"ask": "PA",
|
|
22
|
+
"vwap": "VWAP",
|
|
23
|
+
"volume": "VO",
|
|
24
|
+
"outstanding_shares": "NOSH",
|
|
25
|
+
"market_capitalization": "MV",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DSWSMarketDataDataloader(MarketDataProtocol, Dataloader):
|
|
30
|
+
def market_data(
|
|
31
|
+
self,
|
|
32
|
+
values: list[MarketData] = [MarketData.CLOSE],
|
|
33
|
+
from_date: date | None = None,
|
|
34
|
+
to_date: date | None = None,
|
|
35
|
+
exact_date: date | None = None,
|
|
36
|
+
frequency: Frequency = Frequency.DAILY,
|
|
37
|
+
**kwargs,
|
|
38
|
+
) -> Iterator[MarketDataDict]:
|
|
39
|
+
"""Get prices for instruments.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
values (list[MarketData]): List of values to include in the results.
|
|
43
|
+
from_date (date | None): The starting date for filtering prices. Defaults to None.
|
|
44
|
+
to_date (date | None): The ending date for filtering prices. Defaults to None.
|
|
45
|
+
frequency (Frequency): The frequency of the requested data
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Iterator[MarketDataDict]: An iterator of dictionaries conforming to the DailyValuationDict.
|
|
49
|
+
"""
|
|
50
|
+
default_lookup = {
|
|
51
|
+
k: {"id": p, "symbol": v}
|
|
52
|
+
for k, v, p in self.entities.values_list(
|
|
53
|
+
"dl_parameters__market_data__parameters__identifier",
|
|
54
|
+
"dl_parameters__market_data__parameters__price_symbol",
|
|
55
|
+
"id",
|
|
56
|
+
)
|
|
57
|
+
}
|
|
58
|
+
if (dsws_username := getattr(settings, "REFINITIV_DATASTREAM_USERNAME", None)) and (
|
|
59
|
+
dsws_password := getattr(settings, "REFINITIV_DATASTREAM_PASSWORD", None)
|
|
60
|
+
):
|
|
61
|
+
with suppress(DSUserObjectFault):
|
|
62
|
+
client = Client(username=dsws_username, password=dsws_password)
|
|
63
|
+
default_fields = list(set(map(lambda x: x[1]["symbol"], default_lookup.items())))
|
|
64
|
+
values_fields = [FIELD_MAP[v.value] for v in values if v.value in FIELD_MAP]
|
|
65
|
+
fields = list(set(default_fields + values_fields))
|
|
66
|
+
identifiers = list(default_lookup.keys())
|
|
67
|
+
parameters: dict[str, str | list[str] | date] = {}
|
|
68
|
+
if exact_date:
|
|
69
|
+
parameters["start"] = exact_date
|
|
70
|
+
parameters["end"] = (exact_date + BDay(1)).date()
|
|
71
|
+
else:
|
|
72
|
+
if from_date:
|
|
73
|
+
parameters["start"] = from_date.strftime("%Y-%m-%d")
|
|
74
|
+
if to_date:
|
|
75
|
+
parameters["end"] = to_date.strftime("%Y-%m-%d")
|
|
76
|
+
for chunked_identifiers in client.get_chunked_list(identifiers, len(fields)):
|
|
77
|
+
df = client.get_timeserie_df(chunked_identifiers, fields, **parameters)
|
|
78
|
+
for row in df.to_dict("records"):
|
|
79
|
+
jsondate = row["Dates"].date()
|
|
80
|
+
external_id = row["Instrument"]
|
|
81
|
+
|
|
82
|
+
data = {}
|
|
83
|
+
for market_value in values:
|
|
84
|
+
data[market_value.value] = row.get(FIELD_MAP[market_value.value], None)
|
|
85
|
+
with suppress(KeyError):
|
|
86
|
+
if default_symbol := default_lookup[external_id].get("symbol", None):
|
|
87
|
+
data["close"] = row[default_symbol]
|
|
88
|
+
yield MarketDataDict(
|
|
89
|
+
id=f"{default_lookup[external_id]['id']}_{jsondate}",
|
|
90
|
+
valuation_date=jsondate,
|
|
91
|
+
instrument_id=default_lookup[external_id]["id"],
|
|
92
|
+
external_id=external_id,
|
|
93
|
+
source="refinitiv-dsws",
|
|
94
|
+
**data,
|
|
95
|
+
# **{value: field for value, field in zip_longest(values, fields[1:])},
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
def get_adjustments(
|
|
99
|
+
self,
|
|
100
|
+
from_date: date | None = None,
|
|
101
|
+
to_date: date | None = None,
|
|
102
|
+
exact_date: date | None = None,
|
|
103
|
+
):
|
|
104
|
+
lookup = dict(self.entities.values_list("external_id", "id"))
|
|
105
|
+
if (dsws_username := getattr(settings, "REFINITIV_DATASTREAM_USERNAME", None)) and (
|
|
106
|
+
dsws_password := getattr(settings, "REFINITIV_DATASTREAM_PASSWORD", None)
|
|
107
|
+
):
|
|
108
|
+
with suppress(DSUserObjectFault):
|
|
109
|
+
client = Client(username=dsws_username, password=dsws_password)
|
|
110
|
+
identifiers = list(lookup.keys())
|
|
111
|
+
parameters: dict[str, str | list[str]] = {}
|
|
112
|
+
if from_date:
|
|
113
|
+
parameters["start"] = from_date.strftime("%Y-%m-%d")
|
|
114
|
+
if to_date:
|
|
115
|
+
parameters["end"] = to_date.strftime("%Y-%m-%d")
|
|
116
|
+
for chunked_identifiers in client.get_chunked_list(identifiers, 1):
|
|
117
|
+
df = client.get_timeserie_df(chunked_identifiers, ["AX"], **parameters)
|
|
118
|
+
for row in df.to_dict("records"):
|
|
119
|
+
jsondate = row["Dates"].date()
|
|
120
|
+
del row["Dates"]
|
|
121
|
+
external_id = row["Instrument"]
|
|
122
|
+
del row["Instrument"]
|
|
123
|
+
yield {
|
|
124
|
+
"id": f"{lookup[external_id]}_{jsondate}",
|
|
125
|
+
"adjustment_date": jsondate,
|
|
126
|
+
"instrument_id": lookup[external_id],
|
|
127
|
+
"external_id": external_id,
|
|
128
|
+
"source": "refinitiv-dsws",
|
|
129
|
+
"adjustment_factor": row["AX"],
|
|
130
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import pytz
|
|
2
|
+
from django.conf import settings
|
|
3
|
+
from django.db import connections
|
|
4
|
+
from django.db.models import Max
|
|
5
|
+
from wbcore.contrib.dataloader.utils import dictfetchall, dictfetchone
|
|
6
|
+
from wbcore.contrib.geography.models import Geography
|
|
7
|
+
from wbcore.utils.cache import mapping
|
|
8
|
+
from wbfdm.models import Exchange
|
|
9
|
+
from wbfdm.sync.abstract import Sync
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class QAExchangeSync(Sync[Exchange]):
|
|
13
|
+
SOURCE = "qa-ds2-exchange"
|
|
14
|
+
|
|
15
|
+
def update(self):
|
|
16
|
+
sql = "SELECT ExchIntCode AS source_id, ExchName AS name, ExchCtryCode AS country_id, ExchMnem AS refinitiv_mnemonic FROM DS2Exchange"
|
|
17
|
+
Exchange.objects.bulk_create(
|
|
18
|
+
map(
|
|
19
|
+
lambda data: Exchange(
|
|
20
|
+
source=self.SOURCE,
|
|
21
|
+
source_id=data["source_id"],
|
|
22
|
+
name=data["name"],
|
|
23
|
+
country_id=mapping(Geography.countries, "code_2").get(data["country_id"]),
|
|
24
|
+
refinitiv_mnemonic=data["refinitiv_mnemonic"],
|
|
25
|
+
),
|
|
26
|
+
dictfetchall(connections["qa"].cursor().execute(sql)),
|
|
27
|
+
),
|
|
28
|
+
update_conflicts=True,
|
|
29
|
+
update_fields=["name", "country", "refinitiv_mnemonic"],
|
|
30
|
+
unique_fields=["source", "source_id"],
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def update_or_create_item(self, external_id: int) -> Exchange:
|
|
34
|
+
defaults = self.get_item(external_id)
|
|
35
|
+
defaults["country_id"] = mapping(Geography.countries, "code_2").get(defaults["country_id"])
|
|
36
|
+
exchange, _ = Exchange.objects.update_or_create(
|
|
37
|
+
source=self.SOURCE,
|
|
38
|
+
source_id=external_id,
|
|
39
|
+
defaults=defaults,
|
|
40
|
+
)
|
|
41
|
+
return exchange
|
|
42
|
+
|
|
43
|
+
def update_item(self, item: Exchange) -> Exchange:
|
|
44
|
+
return self.update_or_create_item(item.source_id)
|
|
45
|
+
|
|
46
|
+
def get_item(self, external_id: int) -> dict:
|
|
47
|
+
sql = "SELECT ExchName AS name, ExchCtryCode AS country_id, ExchMnem AS refinitiv_mnemonic FROM DS2Exchange WHERE ExchIntCode = %s"
|
|
48
|
+
return dictfetchone(connections["qa"].cursor().execute(sql, (external_id,)))
|
|
49
|
+
|
|
50
|
+
def trigger_partial_update(self):
|
|
51
|
+
max_last_updated = (
|
|
52
|
+
Exchange.objects.filter(source=self.SOURCE)
|
|
53
|
+
.aggregate(max_last_updated=Max("last_updated"))
|
|
54
|
+
.get("max_last_updated")
|
|
55
|
+
)
|
|
56
|
+
if max_last_updated is None:
|
|
57
|
+
self.update()
|
|
58
|
+
else:
|
|
59
|
+
with connections["qa"].cursor() as cursor:
|
|
60
|
+
cursor.execute(
|
|
61
|
+
"SELECT MAX(last_user_update) FROM sys.dm_db_index_usage_stats WHERE OBJECT_NAME(object_id) = 'DS2Exchange_changes'"
|
|
62
|
+
)
|
|
63
|
+
max_last_updated_qa = (
|
|
64
|
+
pytz.timezone(settings.TIME_ZONE).localize(result[0]) if (result := cursor.fetchone()) else None
|
|
65
|
+
)
|
|
66
|
+
if max_last_updated_qa and max_last_updated_qa > max_last_updated:
|
|
67
|
+
for _, exchange_id in cursor.execute(
|
|
68
|
+
"SELECT UpdateFlag_, ExchIntCode FROM DS2Exchange_changes"
|
|
69
|
+
).fetchall():
|
|
70
|
+
self.update_or_create_item(exchange_id)
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
from django.db import connections
|
|
2
|
+
from wbfdm.contrib.qa.sync.utils import (
|
|
3
|
+
get_item,
|
|
4
|
+
trigger_partial_update,
|
|
5
|
+
update_instruments,
|
|
6
|
+
update_or_create_item,
|
|
7
|
+
)
|
|
8
|
+
from wbfdm.models import Instrument
|
|
9
|
+
from wbfdm.sync.abstract import Sync
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class QACompanySync(Sync[Instrument]):
|
|
13
|
+
SOURCE = "qa-ds2-company"
|
|
14
|
+
|
|
15
|
+
def update(self, **kwargs):
|
|
16
|
+
update_instruments("qa/sql/companies.sql", **kwargs)
|
|
17
|
+
|
|
18
|
+
def update_or_create_item(self, external_id: int) -> Instrument:
|
|
19
|
+
return update_or_create_item(external_id, self.get_item, self.SOURCE)
|
|
20
|
+
|
|
21
|
+
def update_item(self, item: Instrument) -> Instrument:
|
|
22
|
+
return self.update_or_create_item(item.source_id)
|
|
23
|
+
|
|
24
|
+
def get_item(self, external_id: int) -> dict:
|
|
25
|
+
return get_item(external_id, "qa/sql/companies.sql")
|
|
26
|
+
|
|
27
|
+
def trigger_partial_update(self):
|
|
28
|
+
trigger_partial_update(
|
|
29
|
+
Instrument.objects.filter(source=self.SOURCE),
|
|
30
|
+
"last_update",
|
|
31
|
+
"DsCmpyCode",
|
|
32
|
+
"DS2Company_changes",
|
|
33
|
+
self.update,
|
|
34
|
+
self.update_or_create_item,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class QAInstrumentSync(Sync[Instrument]):
|
|
39
|
+
SOURCE = "qa-ds2-security"
|
|
40
|
+
|
|
41
|
+
def update(self, **kwargs):
|
|
42
|
+
update_instruments("qa/sql/instruments.sql", parent_source="qa-ds2-company", **kwargs)
|
|
43
|
+
|
|
44
|
+
def update_or_create_item(self, external_id: int) -> Instrument:
|
|
45
|
+
return update_or_create_item(external_id, self.get_item, self.SOURCE, "qa-ds2-company")
|
|
46
|
+
|
|
47
|
+
def update_item(self, item: Instrument) -> Instrument:
|
|
48
|
+
return self.update_or_create_item(item.source_id)
|
|
49
|
+
|
|
50
|
+
def get_item(self, external_id: int) -> dict:
|
|
51
|
+
return get_item(external_id, "qa/sql/instruments.sql")
|
|
52
|
+
|
|
53
|
+
def trigger_partial_update(self):
|
|
54
|
+
trigger_partial_update(
|
|
55
|
+
Instrument.objects.filter(source=self.SOURCE),
|
|
56
|
+
"last_update",
|
|
57
|
+
"DsSecCode",
|
|
58
|
+
"DS2Security_changes",
|
|
59
|
+
self.update,
|
|
60
|
+
self.update_or_create_item,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class QAQuoteSync(Sync[Instrument]):
|
|
65
|
+
SOURCE = "qa-ds2-quote"
|
|
66
|
+
|
|
67
|
+
def update(self, **kwargs):
|
|
68
|
+
count = connections["qa"].cursor().execute("SELECT COUNT(*) FROM DS2ExchQtInfo").fetchone()[0]
|
|
69
|
+
for offset in range(0, count, 100_000):
|
|
70
|
+
update_instruments(
|
|
71
|
+
"qa/sql/quotes.sql",
|
|
72
|
+
parent_source="qa-ds2-security",
|
|
73
|
+
context={"offset": offset, "batch": 100_000},
|
|
74
|
+
**kwargs,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def update_or_create_item(self, external_id: int) -> Instrument:
|
|
78
|
+
return update_or_create_item(external_id, self.get_item, self.SOURCE, "qa-ds2-security")
|
|
79
|
+
|
|
80
|
+
def update_item(self, item: Instrument) -> Instrument:
|
|
81
|
+
return self.update_or_create_item(item.source_id)
|
|
82
|
+
|
|
83
|
+
def get_item(self, external_id: int) -> dict:
|
|
84
|
+
return get_item(external_id, "qa/sql/quotes.sql")
|
|
85
|
+
|
|
86
|
+
def trigger_partial_update(self):
|
|
87
|
+
trigger_partial_update(
|
|
88
|
+
Instrument.objects.filter(source=self.SOURCE),
|
|
89
|
+
"last_update",
|
|
90
|
+
"CONCAT(InfoCode, '-', ExchIntCode)",
|
|
91
|
+
"DS2ExchQtInfo_changes",
|
|
92
|
+
self.update,
|
|
93
|
+
self.update_or_create_item,
|
|
94
|
+
)
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
from contextlib import suppress
|
|
2
|
+
from typing import Callable
|
|
3
|
+
|
|
4
|
+
import pytz
|
|
5
|
+
from django.conf import settings
|
|
6
|
+
from django.db import connections
|
|
7
|
+
from django.db.models import Max, QuerySet
|
|
8
|
+
from django.template.loader import get_template
|
|
9
|
+
from jinjasql import JinjaSql # type: ignore
|
|
10
|
+
from tqdm import tqdm
|
|
11
|
+
from wbcore.contrib.currency.models import Currency
|
|
12
|
+
from wbcore.contrib.dataloader.utils import dictfetchall, dictfetchone
|
|
13
|
+
from wbcore.contrib.geography.models import Geography
|
|
14
|
+
from wbcore.utils.cache import mapping
|
|
15
|
+
from wbfdm.models.exchanges.exchanges import Exchange
|
|
16
|
+
from wbfdm.models.instruments.instruments import Instrument, InstrumentType
|
|
17
|
+
|
|
18
|
+
BATCH_SIZE: int = 10000
|
|
19
|
+
instrument_type_map = {
|
|
20
|
+
"ADR": "american_depository_receipt",
|
|
21
|
+
"CF": "close_ended_fund",
|
|
22
|
+
"EQ": "equity",
|
|
23
|
+
"ET": "exchange_traded_fund",
|
|
24
|
+
"ETC": "exchange_traded_commodity",
|
|
25
|
+
"ETN": "exchange_traded_note",
|
|
26
|
+
"EWT": "warrant",
|
|
27
|
+
"GDR": "global_depository_receipt",
|
|
28
|
+
"GNSH": "genussschein",
|
|
29
|
+
"INVT": "investment_trust",
|
|
30
|
+
"NVDR": "non_voting_depository_receipt",
|
|
31
|
+
"PREF": "preference_share",
|
|
32
|
+
"UT": "abc",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def get_dataloader_parameters(
|
|
37
|
+
quote_code: str | None = None,
|
|
38
|
+
exchange_code: str | None = None,
|
|
39
|
+
rkd_code: str | None = None,
|
|
40
|
+
ibes_code: str | None = None,
|
|
41
|
+
) -> dict:
|
|
42
|
+
dataloader_parameters = dict()
|
|
43
|
+
if quote_code:
|
|
44
|
+
dataloader_parameters["adjustments"] = {
|
|
45
|
+
"path": "wbfdm.contrib.qa.dataloaders.adjustments.DatastreamAdjustmentsDataloader",
|
|
46
|
+
"parameters": quote_code,
|
|
47
|
+
}
|
|
48
|
+
dataloader_parameters["corporate_actions"] = {
|
|
49
|
+
"path": "wbfdm.contrib.qa.dataloaders.corporate_actions.DatastreamCorporateActionsDataloader",
|
|
50
|
+
"parameters": quote_code,
|
|
51
|
+
}
|
|
52
|
+
if exchange_code:
|
|
53
|
+
dataloader_parameters["market_data"] = {
|
|
54
|
+
"path": "wbfdm.contrib.qa.dataloaders.market_data.DatastreamMarketDataDataloader",
|
|
55
|
+
"parameters": [
|
|
56
|
+
quote_code,
|
|
57
|
+
exchange_code,
|
|
58
|
+
],
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if rkd_code:
|
|
62
|
+
dataloader_parameters["officers"] = {
|
|
63
|
+
"path": "wbfdm.contrib.qa.dataloaders.officers.RKDOfficersDataloader",
|
|
64
|
+
"parameters": rkd_code,
|
|
65
|
+
}
|
|
66
|
+
dataloader_parameters["statements"] = {
|
|
67
|
+
"path": "wbfdm.contrib.qa.dataloaders.statements.RKDStatementsDataloader",
|
|
68
|
+
"parameters": rkd_code,
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if ibes_code:
|
|
72
|
+
dataloader_parameters["financials"] = {
|
|
73
|
+
"path": "wbfdm.contrib.qa.dataloaders.financials.IBESFinancialsDataloader",
|
|
74
|
+
"parameters": ibes_code,
|
|
75
|
+
}
|
|
76
|
+
dataloader_parameters["reporting_dates"] = {
|
|
77
|
+
"path": "wbfdm.contrib.qa.dataloaders.reporting_dates.IbesReportingDateDataloader",
|
|
78
|
+
"parameters": ibes_code,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
return dataloader_parameters
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def convert_data(data, parent_source: str | None = None) -> dict:
|
|
85
|
+
if len(data.keys()) == 0:
|
|
86
|
+
return data
|
|
87
|
+
data["country_id"] = mapping(Geography.countries, "code_2").get(data["country_id"])
|
|
88
|
+
data["currency_id"] = mapping(Currency.objects).get(data["currency_id"])
|
|
89
|
+
if instrument_type_id := data.pop("instrument_type_id"):
|
|
90
|
+
try:
|
|
91
|
+
data["instrument_type_id"] = mapping(InstrumentType.objects)[
|
|
92
|
+
instrument_type_map.get(instrument_type_id, instrument_type_id)
|
|
93
|
+
]
|
|
94
|
+
except KeyError:
|
|
95
|
+
data["instrument_type_id"] = InstrumentType.objects.get_or_create(
|
|
96
|
+
key=instrument_type_id.lower(),
|
|
97
|
+
defaults={"name": instrument_type_id.title(), "short_name": instrument_type_id.title()},
|
|
98
|
+
)[0].pk
|
|
99
|
+
data["exchange_id"] = mapping(Exchange.objects, "source_id", source="qa-ds2-exchange").get(
|
|
100
|
+
str(data["exchange_code"])
|
|
101
|
+
)
|
|
102
|
+
data["dl_parameters"] = get_dataloader_parameters(
|
|
103
|
+
data.pop("quote_code"), data.pop("exchange_code"), data.pop("rkd_code"), data.pop("ibes_code")
|
|
104
|
+
)
|
|
105
|
+
data["additional_urls"] = urls.split(",") if (urls := data.get("additional_urls")) else []
|
|
106
|
+
if parent_source:
|
|
107
|
+
data["parent_id"] = mapping(Instrument.objects, "source_id", source=parent_source).get(str(data["parent_id"]))
|
|
108
|
+
if (is_primary := data.get("is_primary")) and isinstance(is_primary, str):
|
|
109
|
+
data["is_primary"] = is_primary.lower().strip() == "y"
|
|
110
|
+
data.pop("phone")
|
|
111
|
+
return data
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def get_instrument_from_data(data, parent_source: str | None = None) -> Instrument | None:
|
|
115
|
+
if len(data.keys()) == 0:
|
|
116
|
+
return None
|
|
117
|
+
instrument = Instrument(**convert_data(data, parent_source))
|
|
118
|
+
instrument.pre_save()
|
|
119
|
+
instrument.computed_str = instrument.compute_str()
|
|
120
|
+
if (
|
|
121
|
+
instrument.name and instrument.currency and instrument.instrument_type
|
|
122
|
+
): # we return an instrument only if it contains the basic name, currency and type
|
|
123
|
+
return instrument
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _bulk_create_instruments_chunk(instruments: list[Instrument], update_unique_identifiers: bool = False):
|
|
127
|
+
update_fields = [
|
|
128
|
+
"name",
|
|
129
|
+
"dl_parameters",
|
|
130
|
+
"description",
|
|
131
|
+
"country",
|
|
132
|
+
"currency",
|
|
133
|
+
"parent",
|
|
134
|
+
"primary_url",
|
|
135
|
+
"additional_urls",
|
|
136
|
+
"headquarter_address",
|
|
137
|
+
"inception_date",
|
|
138
|
+
"delisted_date",
|
|
139
|
+
"exchange",
|
|
140
|
+
"instrument_type",
|
|
141
|
+
"computed_str",
|
|
142
|
+
"search_vector",
|
|
143
|
+
"trigram_search_vector",
|
|
144
|
+
"is_primary",
|
|
145
|
+
]
|
|
146
|
+
bulk_update_kwargs = {"update_conflicts": True}
|
|
147
|
+
if update_unique_identifiers:
|
|
148
|
+
update_fields.extend(
|
|
149
|
+
[
|
|
150
|
+
"refinitiv_identifier_code",
|
|
151
|
+
"refinitiv_mnemonic_code",
|
|
152
|
+
"isin",
|
|
153
|
+
"sedol",
|
|
154
|
+
"valoren",
|
|
155
|
+
"cusip",
|
|
156
|
+
]
|
|
157
|
+
)
|
|
158
|
+
bulk_update_kwargs = {"ignore_conflicts": True}
|
|
159
|
+
Instrument.objects.bulk_create(
|
|
160
|
+
instruments, update_fields=update_fields, unique_fields=["source", "source_id"], **bulk_update_kwargs
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def update_instruments(sql_name: str, parent_source: str | None = None, context=None, debug: bool = False, **kwargs):
|
|
165
|
+
template = get_template(sql_name, using="jinja").template # type: ignore
|
|
166
|
+
if context is None:
|
|
167
|
+
context = {}
|
|
168
|
+
query, params = JinjaSql(param_style="format").prepare_query(template, context)
|
|
169
|
+
instruments = []
|
|
170
|
+
|
|
171
|
+
gen = dictfetchall(connections["qa"].cursor().execute(query, params))
|
|
172
|
+
if debug:
|
|
173
|
+
gen = tqdm(gen)
|
|
174
|
+
# we update in batch to be sure to not exhaust resources
|
|
175
|
+
for row in gen:
|
|
176
|
+
with suppress(TypeError): # we don't fail if the given data doesn't satisfy the schema
|
|
177
|
+
if instrument := get_instrument_from_data(row, parent_source=parent_source):
|
|
178
|
+
instruments.append(instrument)
|
|
179
|
+
|
|
180
|
+
if len(instruments) >= BATCH_SIZE:
|
|
181
|
+
_bulk_create_instruments_chunk(instruments, **kwargs)
|
|
182
|
+
instruments = []
|
|
183
|
+
|
|
184
|
+
_bulk_create_instruments_chunk(instruments, **kwargs)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def update_or_create_item(
|
|
188
|
+
external_id: int, get_item: Callable, source: str, parent_source: str | None = None
|
|
189
|
+
) -> Instrument:
|
|
190
|
+
defaults = convert_data(get_item(external_id), parent_source=parent_source)
|
|
191
|
+
|
|
192
|
+
dl_parameters = defaults.pop("dl_parameters", {})
|
|
193
|
+
defaults.pop("source", None)
|
|
194
|
+
defaults.pop("source_id", None)
|
|
195
|
+
|
|
196
|
+
instrument, _ = Instrument.objects.update_or_create(
|
|
197
|
+
source=source,
|
|
198
|
+
source_id=external_id,
|
|
199
|
+
defaults=defaults,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
for key, value in dl_parameters.items():
|
|
203
|
+
instrument.dl_parameters[key] = value
|
|
204
|
+
|
|
205
|
+
instrument.save()
|
|
206
|
+
|
|
207
|
+
return instrument
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def get_item(external_id: int, template_name: str) -> dict:
|
|
211
|
+
template = get_template(template_name, using="jinja").template # type: ignore
|
|
212
|
+
query, params = JinjaSql(param_style="format").prepare_query(template, {"source_id": external_id})
|
|
213
|
+
return dictfetchone(connections["qa"].cursor().execute(query, params))
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def trigger_partial_update(
|
|
217
|
+
queryset: QuerySet,
|
|
218
|
+
last_update_field: str,
|
|
219
|
+
id_field: str,
|
|
220
|
+
table_change_name: str,
|
|
221
|
+
update: Callable,
|
|
222
|
+
update_or_create_item: Callable,
|
|
223
|
+
):
|
|
224
|
+
max_last_updated = queryset.aggregate(max_last_updated=Max(last_update_field)).get("max_last_updated")
|
|
225
|
+
|
|
226
|
+
if max_last_updated is None:
|
|
227
|
+
update()
|
|
228
|
+
|
|
229
|
+
else:
|
|
230
|
+
with connections["qa"].cursor() as cursor:
|
|
231
|
+
cursor.execute(
|
|
232
|
+
f"SELECT MAX(last_user_update) FROM sys.dm_db_index_usage_stats WHERE OBJECT_NAME(object_id) = '{table_change_name}'"
|
|
233
|
+
)
|
|
234
|
+
max_last_updated_qa = (
|
|
235
|
+
pytz.timezone(settings.TIME_ZONE).localize(result[0]) if (result := cursor.fetchone()) else None
|
|
236
|
+
)
|
|
237
|
+
if max_last_updated_qa and max_last_updated_qa > max_last_updated:
|
|
238
|
+
for _, security_id in cursor.execute(
|
|
239
|
+
f"SELECT UpdateFlag_, {id_field} FROM {table_change_name}"
|
|
240
|
+
).fetchall():
|
|
241
|
+
update_or_create_item(security_id)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.3
|
|
2
2
|
Name: wbfdm
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.5
|
|
4
4
|
Summary: The workbench module ensures rapid access to diverse financial data (market, fundamental, forecasts, ESG), with features for storing instruments, classifying them, and conducting financial analysis.
|
|
5
5
|
Author-email: Christopher Wittlinger <c.wittlinger@stainly.com>
|
|
6
6
|
Requires-Dist: roman==4.*
|
|
@@ -34,9 +34,11 @@ wbfdm/analysis/financial_analysis/utils.py,sha256=ko88qzmt0zv0D1xjvK4OU5VRF7Uu45
|
|
|
34
34
|
wbfdm/analysis/technical_analysis/__init__.py,sha256=nQdpDl2okzWID6UT7fyfl36RSLNO1ZZzxRt_mkGVVkY,50
|
|
35
35
|
wbfdm/analysis/technical_analysis/technical_analysis.py,sha256=MkK5HrLz9_TCVAv4qDDZjx6EIqsvusMYB7ybg3MvSsg,4961
|
|
36
36
|
wbfdm/analysis/technical_analysis/traces.py,sha256=1wzmW7JcQXki0q14TdG36tacZpIFJ_bO4KHx_5TgNxc,6371
|
|
37
|
+
wbfdm/backends/dto.py,sha256=5IdeGVsrk8trEi9rqtq-zisqiEDE_VLBP8RxlfZZnjk,596
|
|
37
38
|
wbfdm/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
39
|
wbfdm/contrib/dsws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
39
40
|
wbfdm/contrib/dsws/client.py,sha256=DFsKj8LultW1MlTlxl1EHuzuZW7UcIIy25v0L2HM5Z4,13788
|
|
41
|
+
wbfdm/contrib/dsws/dataloaders/market_data.py,sha256=TVL0Vzmg8hDzjbqBWx1342dvT_qC5gVp8E4i3hVcivA,6003
|
|
40
42
|
wbfdm/contrib/internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
43
|
wbfdm/contrib/internal/dataloaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
42
44
|
wbfdm/contrib/internal/dataloaders/market_data.py,sha256=V-kCHgQZVgLkxAmB0xp0BOV48xkcl_YoQzFokKSoL3M,3259
|
|
@@ -101,6 +103,9 @@ wbfdm/contrib/qa/dataloaders/market_data.py,sha256=gAzy00nhFWqABG7kjvCfvpyYnehWI
|
|
|
101
103
|
wbfdm/contrib/qa/dataloaders/officers.py,sha256=QfsMbnlWOhvJc4jyPHJX3STqIwOUqlCCoZddELc405E,2268
|
|
102
104
|
wbfdm/contrib/qa/dataloaders/reporting_dates.py,sha256=Nq83u7eAhsPe9b4ePI5xcgvWEgW1XwHLyVzxpbgcdnQ,2492
|
|
103
105
|
wbfdm/contrib/qa/dataloaders/statements.py,sha256=tE5ux6bFmAezivssRkm3cNW51lJLoAjHbUao1gR4JMM,10093
|
|
106
|
+
wbfdm/contrib/qa/sync/exchanges.py,sha256=831-0bqxq4LaHEkyvMGfEBkNFm5LyLCSpjZbPdTEF0A,3079
|
|
107
|
+
wbfdm/contrib/qa/sync/instruments.py,sha256=8aTQVJ_cw1phe4FWikn79pjCfUijaTcwkdhQCtSXKH0,3156
|
|
108
|
+
wbfdm/contrib/qa/sync/utils.py,sha256=TYZY3QB75POUmme1U71yo6WPzf4AXs4pWpFdI1Hq2GY,8936
|
|
104
109
|
wbfdm/dataloaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
105
110
|
wbfdm/dataloaders/cache.py,sha256=K9BeVxT7p-BMvjurINt18bfrUDccp840uIjfDBLJRNk,4841
|
|
106
111
|
wbfdm/dataloaders/protocols.py,sha256=wCBHWPkTGVHMcAJU6VkuwisKyvMNJt1jTsSerJDuX2M,3024
|
|
@@ -332,6 +337,6 @@ wbfdm/viewsets/statements/__init__.py,sha256=odxtFYUDICPmz8WCE3nx93EvKZLSPBEI4d7
|
|
|
332
337
|
wbfdm/viewsets/statements/statements.py,sha256=AVU5A7p63uWITEcMDl95iKyN-fBO5RwJlbxHrJZvz7o,4083
|
|
333
338
|
wbfdm/viewsets/technical_analysis/__init__.py,sha256=qtCIBg0uSiZeJq_1tEQFilnorMBkMe6uCMfqar6-cLE,77
|
|
334
339
|
wbfdm/viewsets/technical_analysis/monthly_performances.py,sha256=8VgafjW4efEJKTfBqUvAu-EG2eNawl2Cmh9QFt4sN8w,3973
|
|
335
|
-
wbfdm-2.2.
|
|
336
|
-
wbfdm-2.2.
|
|
337
|
-
wbfdm-2.2.
|
|
340
|
+
wbfdm-2.2.5.dist-info/METADATA,sha256=qNsybSbkZC6nDfL1NlItisKlgveNge-_jZ3pp7x1_No,667
|
|
341
|
+
wbfdm-2.2.5.dist-info/WHEEL,sha256=aO3RJuuiFXItVSnAUEmQ0yRBvv9e1sbJh68PtuQkyAE,105
|
|
342
|
+
wbfdm-2.2.5.dist-info/RECORD,,
|
|
File without changes
|