wbfdm 2.2.4__py2.py3-none-any.whl → 2.2.6__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of wbfdm might be problematic. Click here for more details.
- wbfdm/backends/dto.py +36 -0
- wbfdm/contrib/dsws/dataloaders/market_data.py +130 -0
- wbfdm/contrib/qa/jinja2/qa/sql/companies.sql +100 -0
- wbfdm/contrib/qa/jinja2/qa/sql/ibes/base_estimates.sql +33 -0
- wbfdm/contrib/qa/jinja2/qa/sql/ibes/calendarized.sql +37 -0
- wbfdm/contrib/qa/jinja2/qa/sql/ibes/complete.sql +9 -0
- wbfdm/contrib/qa/jinja2/qa/sql/ibes/estimates.sql +3 -0
- wbfdm/contrib/qa/jinja2/qa/sql/ibes/financials.sql +79 -0
- wbfdm/contrib/qa/jinja2/qa/sql/instruments.sql +100 -0
- wbfdm/contrib/qa/jinja2/qa/sql/quotes.sql +98 -0
- wbfdm/contrib/qa/sync/exchanges.py +70 -0
- wbfdm/contrib/qa/sync/instruments.py +94 -0
- wbfdm/contrib/qa/sync/utils.py +241 -0
- {wbfdm-2.2.4.dist-info → wbfdm-2.2.6.dist-info}/METADATA +2 -2
- {wbfdm-2.2.4.dist-info → wbfdm-2.2.6.dist-info}/RECORD +16 -3
- {wbfdm-2.2.4.dist-info → wbfdm-2.2.6.dist-info}/WHEEL +1 -1
wbfdm/backends/dto.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from datetime import date
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass
|
|
6
|
+
class PriceDTO:
|
|
7
|
+
pk: str
|
|
8
|
+
instrument: int
|
|
9
|
+
date: date
|
|
10
|
+
open: float
|
|
11
|
+
close: float
|
|
12
|
+
high: float
|
|
13
|
+
low: float
|
|
14
|
+
volume: float
|
|
15
|
+
market_capitalization: float
|
|
16
|
+
outstanding_shares: float
|
|
17
|
+
|
|
18
|
+
@property
|
|
19
|
+
def net_value(self) -> float:
|
|
20
|
+
return self.close
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class AdjustmentDTO:
|
|
25
|
+
pk: str
|
|
26
|
+
date: date
|
|
27
|
+
adjustment_factor: float
|
|
28
|
+
cumulative_adjustment_factor: float
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class DividendDTO:
|
|
33
|
+
pk: str
|
|
34
|
+
rate: float
|
|
35
|
+
ex_dividend_date: date
|
|
36
|
+
payment_date: date
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
from contextlib import suppress
|
|
2
|
+
from datetime import date
|
|
3
|
+
from typing import Iterator
|
|
4
|
+
|
|
5
|
+
from DatastreamPy import DSUserObjectFault
|
|
6
|
+
from django.conf import settings
|
|
7
|
+
from pandas.tseries.offsets import BDay
|
|
8
|
+
from wbcore.contrib.dataloader.dataloaders import Dataloader
|
|
9
|
+
from wbfdm.dataloaders.protocols import MarketDataProtocol
|
|
10
|
+
from wbfdm.dataloaders.types import MarketDataDict
|
|
11
|
+
from wbfdm.enums import Frequency, MarketData
|
|
12
|
+
|
|
13
|
+
from ..client import Client
|
|
14
|
+
|
|
15
|
+
FIELD_MAP = {
|
|
16
|
+
"close": "P",
|
|
17
|
+
"open": "PO",
|
|
18
|
+
"high": "PH",
|
|
19
|
+
"low": "PL",
|
|
20
|
+
"bid": "PB",
|
|
21
|
+
"ask": "PA",
|
|
22
|
+
"vwap": "VWAP",
|
|
23
|
+
"volume": "VO",
|
|
24
|
+
"outstanding_shares": "NOSH",
|
|
25
|
+
"market_capitalization": "MV",
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class DSWSMarketDataDataloader(MarketDataProtocol, Dataloader):
|
|
30
|
+
def market_data(
|
|
31
|
+
self,
|
|
32
|
+
values: list[MarketData] = [MarketData.CLOSE],
|
|
33
|
+
from_date: date | None = None,
|
|
34
|
+
to_date: date | None = None,
|
|
35
|
+
exact_date: date | None = None,
|
|
36
|
+
frequency: Frequency = Frequency.DAILY,
|
|
37
|
+
**kwargs,
|
|
38
|
+
) -> Iterator[MarketDataDict]:
|
|
39
|
+
"""Get prices for instruments.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
values (list[MarketData]): List of values to include in the results.
|
|
43
|
+
from_date (date | None): The starting date for filtering prices. Defaults to None.
|
|
44
|
+
to_date (date | None): The ending date for filtering prices. Defaults to None.
|
|
45
|
+
frequency (Frequency): The frequency of the requested data
|
|
46
|
+
|
|
47
|
+
Returns:
|
|
48
|
+
Iterator[MarketDataDict]: An iterator of dictionaries conforming to the DailyValuationDict.
|
|
49
|
+
"""
|
|
50
|
+
default_lookup = {
|
|
51
|
+
k: {"id": p, "symbol": v}
|
|
52
|
+
for k, v, p in self.entities.values_list(
|
|
53
|
+
"dl_parameters__market_data__parameters__identifier",
|
|
54
|
+
"dl_parameters__market_data__parameters__price_symbol",
|
|
55
|
+
"id",
|
|
56
|
+
)
|
|
57
|
+
}
|
|
58
|
+
if (dsws_username := getattr(settings, "REFINITIV_DATASTREAM_USERNAME", None)) and (
|
|
59
|
+
dsws_password := getattr(settings, "REFINITIV_DATASTREAM_PASSWORD", None)
|
|
60
|
+
):
|
|
61
|
+
with suppress(DSUserObjectFault):
|
|
62
|
+
client = Client(username=dsws_username, password=dsws_password)
|
|
63
|
+
default_fields = list(set(map(lambda x: x[1]["symbol"], default_lookup.items())))
|
|
64
|
+
values_fields = [FIELD_MAP[v.value] for v in values if v.value in FIELD_MAP]
|
|
65
|
+
fields = list(set(default_fields + values_fields))
|
|
66
|
+
identifiers = list(default_lookup.keys())
|
|
67
|
+
parameters: dict[str, str | list[str] | date] = {}
|
|
68
|
+
if exact_date:
|
|
69
|
+
parameters["start"] = exact_date
|
|
70
|
+
parameters["end"] = (exact_date + BDay(1)).date()
|
|
71
|
+
else:
|
|
72
|
+
if from_date:
|
|
73
|
+
parameters["start"] = from_date.strftime("%Y-%m-%d")
|
|
74
|
+
if to_date:
|
|
75
|
+
parameters["end"] = to_date.strftime("%Y-%m-%d")
|
|
76
|
+
for chunked_identifiers in client.get_chunked_list(identifiers, len(fields)):
|
|
77
|
+
df = client.get_timeserie_df(chunked_identifiers, fields, **parameters)
|
|
78
|
+
for row in df.to_dict("records"):
|
|
79
|
+
jsondate = row["Dates"].date()
|
|
80
|
+
external_id = row["Instrument"]
|
|
81
|
+
|
|
82
|
+
data = {}
|
|
83
|
+
for market_value in values:
|
|
84
|
+
data[market_value.value] = row.get(FIELD_MAP[market_value.value], None)
|
|
85
|
+
with suppress(KeyError):
|
|
86
|
+
if default_symbol := default_lookup[external_id].get("symbol", None):
|
|
87
|
+
data["close"] = row[default_symbol]
|
|
88
|
+
yield MarketDataDict(
|
|
89
|
+
id=f"{default_lookup[external_id]['id']}_{jsondate}",
|
|
90
|
+
valuation_date=jsondate,
|
|
91
|
+
instrument_id=default_lookup[external_id]["id"],
|
|
92
|
+
external_id=external_id,
|
|
93
|
+
source="refinitiv-dsws",
|
|
94
|
+
**data,
|
|
95
|
+
# **{value: field for value, field in zip_longest(values, fields[1:])},
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
def get_adjustments(
|
|
99
|
+
self,
|
|
100
|
+
from_date: date | None = None,
|
|
101
|
+
to_date: date | None = None,
|
|
102
|
+
exact_date: date | None = None,
|
|
103
|
+
):
|
|
104
|
+
lookup = dict(self.entities.values_list("external_id", "id"))
|
|
105
|
+
if (dsws_username := getattr(settings, "REFINITIV_DATASTREAM_USERNAME", None)) and (
|
|
106
|
+
dsws_password := getattr(settings, "REFINITIV_DATASTREAM_PASSWORD", None)
|
|
107
|
+
):
|
|
108
|
+
with suppress(DSUserObjectFault):
|
|
109
|
+
client = Client(username=dsws_username, password=dsws_password)
|
|
110
|
+
identifiers = list(lookup.keys())
|
|
111
|
+
parameters: dict[str, str | list[str]] = {}
|
|
112
|
+
if from_date:
|
|
113
|
+
parameters["start"] = from_date.strftime("%Y-%m-%d")
|
|
114
|
+
if to_date:
|
|
115
|
+
parameters["end"] = to_date.strftime("%Y-%m-%d")
|
|
116
|
+
for chunked_identifiers in client.get_chunked_list(identifiers, 1):
|
|
117
|
+
df = client.get_timeserie_df(chunked_identifiers, ["AX"], **parameters)
|
|
118
|
+
for row in df.to_dict("records"):
|
|
119
|
+
jsondate = row["Dates"].date()
|
|
120
|
+
del row["Dates"]
|
|
121
|
+
external_id = row["Instrument"]
|
|
122
|
+
del row["Instrument"]
|
|
123
|
+
yield {
|
|
124
|
+
"id": f"{lookup[external_id]}_{jsondate}",
|
|
125
|
+
"adjustment_date": jsondate,
|
|
126
|
+
"instrument_id": lookup[external_id],
|
|
127
|
+
"external_id": external_id,
|
|
128
|
+
"source": "refinitiv-dsws",
|
|
129
|
+
"adjustment_factor": row["AX"],
|
|
130
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
SELECT
|
|
2
|
+
'qa-ds2-company' AS 'source',
|
|
3
|
+
company.DsCmpyCode AS 'source_id',
|
|
4
|
+
NULL AS 'parent_id',
|
|
5
|
+
coalesce(cmp_ref.PrimaryName, company.DsCmpyName) AS 'name',
|
|
6
|
+
company.CmpyCtryCode AS 'country_id',
|
|
7
|
+
exchange_qt.ISOCurrCode AS 'currency_id',
|
|
8
|
+
'company' AS 'instrument_type_id',
|
|
9
|
+
rkd_instrument.ISIN AS 'isin',
|
|
10
|
+
rkd_instrument.Ticker AS 'ticker',
|
|
11
|
+
rkd_instrument.RIC AS 'refinitiv_identifier_code',
|
|
12
|
+
country_qt.DsMnem AS 'refinitiv_mnemonic_code',
|
|
13
|
+
rkd_instrument.Cusip AS 'cusip',
|
|
14
|
+
rkd_instrument.Sedol AS 'sedol',
|
|
15
|
+
filing.TxtInfo AS 'description',
|
|
16
|
+
rkd_cmp_det.Employees AS 'employees',
|
|
17
|
+
(select top 1 concat('00', phone.CtryPh, phone.City, phone.PhoneNo) from RKDFndCmpPhone AS phone where phone.Code = rkd_instrument.Code AND PhTypeCode = 1) AS 'phone',
|
|
18
|
+
(select top 1 web.URL from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode = 1) AS 'primary_url',
|
|
19
|
+
(select string_agg(web.URL, ',') from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode <> 1) AS 'additional_urls',
|
|
20
|
+
concat(rkd_cmp_det.StAdd1, ', ', rkd_cmp_det.Post, ', ', rkd_cmp_det.City) AS 'headquarter_address',
|
|
21
|
+
convert(Date, COALESCE(rkd_cmp_det.PublicSince, (SELECT MIN(MarketDate) FROM DS2PrimQtPrc WHERE InfoCode = exchange_qt.InfoCode))) AS 'inception_date',
|
|
22
|
+
NULL AS 'delisted_date',
|
|
23
|
+
convert(Date, cmp_ref.LatestFinAnnDt) AS 'last_annual_report',
|
|
24
|
+
convert(Date, cmp_ref.LatestFinIntmDt) AS 'last_interim_report',
|
|
25
|
+
-- MarketData DL
|
|
26
|
+
exchange_qt.InfoCode AS 'quote_code',
|
|
27
|
+
exchange_qt.ExchIntCode AS 'exchange_code',
|
|
28
|
+
-- Fundamental DL
|
|
29
|
+
rkd_instrument.Code AS 'rkd_code',
|
|
30
|
+
-- Forecast DL
|
|
31
|
+
ibes_mapping.EstPermID AS 'ibes_code'
|
|
32
|
+
|
|
33
|
+
FROM DS2Company AS company
|
|
34
|
+
|
|
35
|
+
LEFT JOIN DS2Security AS security
|
|
36
|
+
ON company.DsCmpyCode = security.DsCmpyCode
|
|
37
|
+
AND security.IsMajorSec = 'Y'
|
|
38
|
+
|
|
39
|
+
LEFT JOIN DS2ExchQtInfo AS exchange_qt
|
|
40
|
+
ON exchange_qt.InfoCode = security.PrimQtInfoCode
|
|
41
|
+
AND exchange_qt.IsPrimExchQt = 'Y'
|
|
42
|
+
|
|
43
|
+
LEFT JOIN DS2CtryQtInfo AS country_qt
|
|
44
|
+
ON exchange_qt.InfoCode = country_qt.InfoCode
|
|
45
|
+
|
|
46
|
+
LEFT JOIN vw_SecurityMappingX AS mappingX
|
|
47
|
+
ON mappingX.vencode = security.PrimQtInfoCode
|
|
48
|
+
AND mappingX.ventype = 33
|
|
49
|
+
AND mappingX.rank = 1
|
|
50
|
+
AND mappingX.StartDate = (
|
|
51
|
+
SELECT MAX(I.StartDate)
|
|
52
|
+
FROM vw_SecurityMappingX AS I
|
|
53
|
+
WHERE I.typ = mappingX.typ AND I.vencode = mappingX.vencode AND I.ventype = mappingX.ventype
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
LEFT JOIN vw_SecurityMappingX AS mappingRKD
|
|
57
|
+
ON mappingX.seccode = mappingRKD.seccode
|
|
58
|
+
AND mappingX.typ = mappingRKD.typ
|
|
59
|
+
AND mappingRKD.ventype = 26
|
|
60
|
+
AND mappingRKD.rank = 1
|
|
61
|
+
AND mappingRKD.StartDate = (
|
|
62
|
+
SELECT MAX(I.StartDate)
|
|
63
|
+
FROM vw_SecurityMappingX AS I
|
|
64
|
+
WHERE I.typ = mappingRKD.typ AND I.vencode = mappingRKD.vencode AND I.ventype = mappingRKD.ventype
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
LEFT JOIN RKDFndCmpRefIssue AS rkd_instrument
|
|
68
|
+
ON rkd_instrument.IssueCode = mappingRKD.vencode
|
|
69
|
+
|
|
70
|
+
LEFT JOIN RKDFndCmpDet AS rkd_cmp_det
|
|
71
|
+
ON rkd_cmp_det.Code = rkd_instrument.Code
|
|
72
|
+
|
|
73
|
+
LEFT JOIN RKDFndCmpRef AS cmp_ref
|
|
74
|
+
ON cmp_ref.Code = rkd_instrument.Code
|
|
75
|
+
|
|
76
|
+
LEFT JOIN RKDFndCmpFiling AS filing
|
|
77
|
+
ON filing.Code = rkd_instrument.Code
|
|
78
|
+
AND filing.TxtInfoTypeCode = 2
|
|
79
|
+
|
|
80
|
+
LEFT JOIN vw_IBES2Mapping AS ibes_mapping
|
|
81
|
+
ON ibes_mapping.SecCode = mappingX.seccode
|
|
82
|
+
AND ibes_mapping.typ = mappingX.typ
|
|
83
|
+
AND ibes_mapping.Exchange = (
|
|
84
|
+
CASE
|
|
85
|
+
WHEN ibes_mapping.typ = 6 THEN 0
|
|
86
|
+
WHEN ibes_mapping.typ = 1 THEN 1
|
|
87
|
+
END
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
{% if source_id %}
|
|
91
|
+
where company.DsCmpyCode = {{ source_id }}
|
|
92
|
+
{% endif %}
|
|
93
|
+
-- where company.CmpyCtryCode = 'IS'
|
|
94
|
+
|
|
95
|
+
ORDER BY company.DsCmpyCode
|
|
96
|
+
|
|
97
|
+
{% if (offset == 0 or offset) and batch %}
|
|
98
|
+
offset {{ offset|sqlsafe }} rows
|
|
99
|
+
fetch next {{ batch|sqlsafe }} rows only
|
|
100
|
+
{% endif %}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{% extends 'qa/sql/ibes/financials.sql' %}
|
|
2
|
+
|
|
3
|
+
{% block additional_fields %}
|
|
4
|
+
(
|
|
5
|
+
select DefActValue * DefScale
|
|
6
|
+
from TreActRpt
|
|
7
|
+
where
|
|
8
|
+
fin.EstPermID = EstPermID
|
|
9
|
+
and fin.PerEndDate = PerEndDate
|
|
10
|
+
and fin.Measure = Measure
|
|
11
|
+
and fin.PerType = PerType
|
|
12
|
+
and ExpireDate is null
|
|
13
|
+
and fin.IsParent = IsParent
|
|
14
|
+
) as actual_value,
|
|
15
|
+
case
|
|
16
|
+
when fin.DefMeanEst = 0 then null
|
|
17
|
+
else ((
|
|
18
|
+
select DefActValue * DefScale
|
|
19
|
+
from TREActRpt
|
|
20
|
+
where
|
|
21
|
+
fin.EstPermID = EstPermID
|
|
22
|
+
and fin.PerEndDate = PerEndDate
|
|
23
|
+
and fin.Measure = Measure
|
|
24
|
+
and fin.PerType = PerType
|
|
25
|
+
and ExpireDate IS NULL
|
|
26
|
+
and fin.IsParent = IsParent
|
|
27
|
+
) - (fin.DefMeanEst * fin.DefScale)) / (fin.DefMeanEst * fin.DefScale)
|
|
28
|
+
end AS difference_pct,
|
|
29
|
+
fin.DefHighEst * fin.DefScale AS value_high,
|
|
30
|
+
fin.DefLowEst * fin.DefScale AS value_low,
|
|
31
|
+
fin.DefStdDev * fin.DefScale AS value_stdev,
|
|
32
|
+
fin.NumEsts AS value_amount,
|
|
33
|
+
{% endblock %}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
with data as (
|
|
2
|
+
{% with financial_table="TreActRpt", value="DefActValue", only_valid=True, estimate=False %}
|
|
3
|
+
{% include 'qa/sql/ibes/financials.sql' %}
|
|
4
|
+
{% endwith %}
|
|
5
|
+
|
|
6
|
+
union
|
|
7
|
+
|
|
8
|
+
{% with financial_table="TreSumPer", value="DefMeanEst", only_valid=True, from_index=1, estimate=True %}
|
|
9
|
+
{% include 'qa/sql/ibes/financials.sql' %}
|
|
10
|
+
{% endwith %}
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
select
|
|
14
|
+
*,
|
|
15
|
+
case
|
|
16
|
+
when period_type = 'Y' and month(period_end_date) = 12 then value
|
|
17
|
+
when period_type = 'Y'
|
|
18
|
+
then month(period_end_date) * value / 12 + (12 - month(period_end_date)) * lead(value, 1) over (partition by financial, external_identifier, period_type order by period_end_date) / 12
|
|
19
|
+
when period_type = 'Q' and (
|
|
20
|
+
(interim = 1 and month(period_end_date) = 3)
|
|
21
|
+
or (interim = 2 and month(period_end_date) = 6)
|
|
22
|
+
or (interim = 3 and month(period_end_date) = 9)
|
|
23
|
+
or (interim = 4 and month(period_end_date) = 12)
|
|
24
|
+
)
|
|
25
|
+
then value
|
|
26
|
+
when period_type = 'Q'
|
|
27
|
+
then month(period_end_date) * value / 3 + (3 - month(period_end_date)) * lead(value, 1) over (partition by financial, external_identifier, period_type order by period_end_date) / 3
|
|
28
|
+
|
|
29
|
+
when period_type = 'S' and (
|
|
30
|
+
(interim = 1 and month(period_end_date) = 6)
|
|
31
|
+
or (interim = 2 and month(period_end_date) = 12)
|
|
32
|
+
)
|
|
33
|
+
then value
|
|
34
|
+
when period_type = 'S'
|
|
35
|
+
then month(period_end_date) * value / 6 + (6 - month(period_end_date)) * lead(value, 1) over (partition by financial, external_identifier, period_type order by period_end_date) / 6
|
|
36
|
+
end as _value
|
|
37
|
+
from data
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{% with financial_table="TreActRpt", value="DefActValue", only_valid=True, estimate=False %}
|
|
2
|
+
{% include 'qa/sql/ibes/financials.sql' %}
|
|
3
|
+
{% endwith %}
|
|
4
|
+
|
|
5
|
+
union
|
|
6
|
+
|
|
7
|
+
{% with financial_table="TreSumPer", value="DefMeanEst", only_valid=True, from_index=1, estimate=True %}
|
|
8
|
+
{% include 'qa/sql/ibes/financials.sql' %}
|
|
9
|
+
{% endwith %}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
select
|
|
2
|
+
case
|
|
3
|
+
when fin.PerType = 2 then 'M'
|
|
4
|
+
when fin.PerType = 3 then 'Q'
|
|
5
|
+
when fin.PerType = 4 then 'Y'
|
|
6
|
+
when fin.PerType = 5 then 'S'
|
|
7
|
+
end as period_type,
|
|
8
|
+
convert(date, fin.ExpireDate) as valid_until,
|
|
9
|
+
fin.EstPermID as external_identifier,
|
|
10
|
+
convert(date, fin.PerEndDate) as period_end_date,
|
|
11
|
+
|
|
12
|
+
idx.FiscalYear as year,
|
|
13
|
+
idx.FiscalIndex as interim,
|
|
14
|
+
idx.PerIndex as 'index',
|
|
15
|
+
|
|
16
|
+
case
|
|
17
|
+
when (code.Description collate Latin1_General_Bin) = 'GBp' then {{ value|identifier }} * DefScale / 100
|
|
18
|
+
else {{ value | identifier }} * DefScale
|
|
19
|
+
end as value,
|
|
20
|
+
|
|
21
|
+
code.Description as currency,
|
|
22
|
+
mapping.financial as financial,
|
|
23
|
+
{% block additional_fields%}{% endblock %}
|
|
24
|
+
{% if estimate %}1{% else %}0{% endif %} as estimate,
|
|
25
|
+
'qa-ibes' as source
|
|
26
|
+
|
|
27
|
+
from {{ financial_table |identifier }} as fin
|
|
28
|
+
|
|
29
|
+
join TrePerIndex as idx
|
|
30
|
+
on idx.EstPermID = fin.EstPermID
|
|
31
|
+
and idx.PerType = fin.PerType
|
|
32
|
+
and idx.PerEndDate = fin.PerEndDate
|
|
33
|
+
|
|
34
|
+
join stainly_financial_mapping as mapping
|
|
35
|
+
on mapping.ibes_financial = fin.Measure
|
|
36
|
+
|
|
37
|
+
left join TreCode as code
|
|
38
|
+
on code.Code = fin.DefCurrPermID
|
|
39
|
+
and code.CodeType = 7
|
|
40
|
+
|
|
41
|
+
where
|
|
42
|
+
fin.EffectiveDate = (
|
|
43
|
+
select max(EffectiveDate)
|
|
44
|
+
from {{ financial_table |identifier }}
|
|
45
|
+
where
|
|
46
|
+
EstPermID = fin.EstPermID
|
|
47
|
+
and PerEndDate = fin.PerEndDate
|
|
48
|
+
and PerType = fin.PerType
|
|
49
|
+
and Measure = fin.Measure
|
|
50
|
+
and (
|
|
51
|
+
ExpireDate = fin.ExpireDate
|
|
52
|
+
or (fin.ExpireDate is null and ExpireDate is null)
|
|
53
|
+
)
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
and fin.EstPermID in (
|
|
57
|
+
{% for instrument in instruments %}
|
|
58
|
+
{{ instrument }}{% if not loop.last %},{% endif %}
|
|
59
|
+
{% endfor %}
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
and mapping.financial in (
|
|
63
|
+
{% for financial in financials %}
|
|
64
|
+
{{ financial }}{% if not loop.last %},{% endif %}
|
|
65
|
+
{% endfor %}
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
{% if only_valid %}and fin.ExpireDate is null{% endif %}
|
|
69
|
+
|
|
70
|
+
{% if from_year %}and idx.FiscalYear >= {{ from_year }}{% endif %}
|
|
71
|
+
{% if to_year %}and idx.FiscalYear <= {{ to_year }}{% endif %}
|
|
72
|
+
|
|
73
|
+
{% if from_date %}and fin.PerEndDate >= {{ from_date }}{% endif %}
|
|
74
|
+
{% if to_date %}and fin.PerEndDate <= {{ to_date }}{% endif %}
|
|
75
|
+
|
|
76
|
+
{% if from_index or from_index == 0 %}and idx.PerIndex >= {{ from_index }}{% endif %}
|
|
77
|
+
{% if to_index or to_index == 0 %}and idx.PerIndex <= {{ to_index }}{% endif %}
|
|
78
|
+
|
|
79
|
+
{% if period_type == 'annual' %}and idx.FiscalIndex = 0{% elif period_type == 'interim' %}and idx.FiscalIndex > 0{% endif %}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
SELECT
|
|
2
|
+
'qa-ds2-security' AS 'source',
|
|
3
|
+
security.DsSecCode AS 'source_id',
|
|
4
|
+
security.IsMajorSec AS 'is_primary',
|
|
5
|
+
company.DsCmpyCode AS 'parent_id',
|
|
6
|
+
coalesce(cmp_ref.PrimaryName, company.DsCmpyName) AS 'name',
|
|
7
|
+
company.CmpyCtryCode AS 'country_id',
|
|
8
|
+
exchange_qt.ISOCurrCode AS 'currency_id',
|
|
9
|
+
country_qt.TypeCode AS 'instrument_type_id',
|
|
10
|
+
rkd_instrument.ISIN AS 'isin',
|
|
11
|
+
rkd_instrument.Ticker AS 'ticker',
|
|
12
|
+
rkd_instrument.RIC AS 'refinitiv_identifier_code',
|
|
13
|
+
country_qt.DsMnem AS 'refinitiv_mnemonic_code',
|
|
14
|
+
rkd_instrument.Cusip AS 'cusip',
|
|
15
|
+
rkd_instrument.Sedol AS 'sedol',
|
|
16
|
+
filing.TxtInfo AS 'description',
|
|
17
|
+
rkd_cmp_det.Employees AS 'employees',
|
|
18
|
+
(select top 1 concat('00', phone.CtryPh, phone.City, phone.PhoneNo) from RKDFndCmpPhone AS phone where phone.Code = rkd_instrument.Code AND PhTypeCode = 1) AS 'phone',
|
|
19
|
+
(select top 1 web.URL from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode = 1) AS 'primary_url',
|
|
20
|
+
(select string_agg(web.URL, ',') from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode <> 1) AS 'additional_urls',
|
|
21
|
+
concat(rkd_cmp_det.StAdd1, ', ', rkd_cmp_det.Post, ', ', rkd_cmp_det.City) AS 'headquarter_address',
|
|
22
|
+
convert(Date, COALESCE(rkd_cmp_det.PublicSince, (SELECT MIN(MarketDate) FROM DS2PrimQtPrc WHERE InfoCode = exchange_qt.InfoCode))) AS 'inception_date',
|
|
23
|
+
convert(Date, security.DelistDate) AS 'delisted_date',
|
|
24
|
+
convert(Date, cmp_ref.LatestFinAnnDt) AS 'last_annual_report',
|
|
25
|
+
convert(Date, cmp_ref.LatestFinIntmDt) AS 'last_interim_report',
|
|
26
|
+
-- MarketData DL
|
|
27
|
+
exchange_qt.InfoCode AS 'quote_code',
|
|
28
|
+
exchange_qt.ExchIntCode AS 'exchange_code',
|
|
29
|
+
-- Fundamental DL
|
|
30
|
+
rkd_instrument.Code AS 'rkd_code',
|
|
31
|
+
-- Forecast DL
|
|
32
|
+
ibes_mapping.EstPermID AS 'ibes_code'
|
|
33
|
+
|
|
34
|
+
FROM DS2Security AS security
|
|
35
|
+
|
|
36
|
+
LEFT JOIN DS2Company AS company
|
|
37
|
+
ON security.DsCmpyCode = company.DsCmpyCode
|
|
38
|
+
|
|
39
|
+
LEFT JOIN DS2ExchQtInfo AS exchange_qt
|
|
40
|
+
ON exchange_qt.InfoCode = security.PrimQtInfoCode
|
|
41
|
+
AND exchange_qt.IsPrimExchQt = 'Y'
|
|
42
|
+
|
|
43
|
+
LEFT JOIN DS2CtryQtInfo AS country_qt
|
|
44
|
+
ON exchange_qt.InfoCode = country_qt.InfoCode
|
|
45
|
+
|
|
46
|
+
LEFT JOIN vw_SecurityMappingX AS mappingX
|
|
47
|
+
ON mappingX.vencode = security.PrimQtInfoCode
|
|
48
|
+
AND mappingX.ventype = 33
|
|
49
|
+
AND mappingX.rank = 1
|
|
50
|
+
AND mappingX.StartDate = (
|
|
51
|
+
SELECT MAX(I.StartDate)
|
|
52
|
+
FROM vw_SecurityMappingX AS I
|
|
53
|
+
WHERE I.typ = mappingX.typ AND I.vencode = mappingX.vencode AND I.ventype = mappingX.ventype
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
LEFT JOIN vw_SecurityMappingX AS mappingRKD
|
|
57
|
+
ON mappingX.seccode = mappingRKD.seccode
|
|
58
|
+
AND mappingX.typ = mappingRKD.typ
|
|
59
|
+
AND mappingRKD.ventype = 26
|
|
60
|
+
AND mappingRKD.rank = 1
|
|
61
|
+
AND mappingRKD.StartDate = (
|
|
62
|
+
SELECT MAX(I.StartDate)
|
|
63
|
+
FROM vw_SecurityMappingX AS I
|
|
64
|
+
WHERE I.typ = mappingRKD.typ AND I.vencode = mappingRKD.vencode AND I.ventype = mappingRKD.ventype
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
LEFT JOIN RKDFndCmpRefIssue AS rkd_instrument
|
|
68
|
+
ON rkd_instrument.IssueCode = mappingRKD.vencode
|
|
69
|
+
|
|
70
|
+
LEFT JOIN RKDFndCmpDet AS rkd_cmp_det
|
|
71
|
+
ON rkd_cmp_det.Code = rkd_instrument.Code
|
|
72
|
+
|
|
73
|
+
LEFT JOIN RKDFndCmpRef AS cmp_ref
|
|
74
|
+
ON cmp_ref.Code = rkd_instrument.Code
|
|
75
|
+
|
|
76
|
+
LEFT JOIN RKDFndCmpFiling AS filing
|
|
77
|
+
ON filing.Code = rkd_instrument.Code
|
|
78
|
+
AND filing.TxtInfoTypeCode = 2
|
|
79
|
+
|
|
80
|
+
LEFT JOIN vw_IBES2Mapping AS ibes_mapping
|
|
81
|
+
ON ibes_mapping.SecCode = mappingX.seccode
|
|
82
|
+
AND ibes_mapping.typ = mappingX.typ
|
|
83
|
+
AND ibes_mapping.Exchange = (
|
|
84
|
+
CASE
|
|
85
|
+
WHEN ibes_mapping.typ = 6 THEN 0
|
|
86
|
+
WHEN ibes_mapping.typ = 1 THEN 1
|
|
87
|
+
END
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
{% if source_id %}
|
|
91
|
+
where security.DsSecCode = {{ source_id }}
|
|
92
|
+
{% endif %}
|
|
93
|
+
-- where company.CmpyCtryCode = 'IS'
|
|
94
|
+
|
|
95
|
+
ORDER BY company.DsCmpyCode
|
|
96
|
+
|
|
97
|
+
{% if (offset == 0 or offset) and batch %}
|
|
98
|
+
offset {{ offset|sqlsafe }} rows
|
|
99
|
+
fetch next {{ batch|sqlsafe }} rows only
|
|
100
|
+
{% endif %}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
SELECT
|
|
2
|
+
'qa-ds2-quote' AS 'source',
|
|
3
|
+
CONCAT(exchange_qt.InfoCode, '-', exchange_qt.ExchIntCode) AS 'source_id',
|
|
4
|
+
security.DsSecCode AS 'parent_id',
|
|
5
|
+
case
|
|
6
|
+
when exchange_qt.IsPrimExchQt = 'Y' and country_qt.IsPrimQt = 1 and security.DelistDate is null then 'Y'
|
|
7
|
+
else 'N'
|
|
8
|
+
end as is_primary,
|
|
9
|
+
coalesce(cmp_ref.PrimaryName, exchange_qt.DsQtName) AS 'name',
|
|
10
|
+
country_qt.Region AS 'country_id',
|
|
11
|
+
exchange_qt.ISOCurrCode AS 'currency_id',
|
|
12
|
+
'quote' AS 'instrument_type_id',
|
|
13
|
+
rkd_instrument.ISIN AS 'isin',
|
|
14
|
+
rkd_instrument.Ticker AS 'ticker',
|
|
15
|
+
rkd_instrument.RIC AS 'refinitiv_identifier_code',
|
|
16
|
+
country_qt.DsMnem AS 'refinitiv_mnemonic_code',
|
|
17
|
+
rkd_instrument.Cusip AS 'cusip',
|
|
18
|
+
rkd_instrument.Sedol AS 'sedol',
|
|
19
|
+
filing.TxtInfo AS 'description',
|
|
20
|
+
rkd_cmp_det.Employees AS 'employees',
|
|
21
|
+
(select top 1 concat('00', phone.CtryPh, phone.City, phone.PhoneNo) from RKDFndCmpPhone AS phone where phone.Code = rkd_instrument.Code AND PhTypeCode = 1) AS 'phone',
|
|
22
|
+
(select top 1 web.URL from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode = 1) AS 'primary_url',
|
|
23
|
+
(select string_agg(web.URL, ',') from RKDFndCmpWebLink AS web where web.Code = rkd_instrument.Code AND web.URLTypeCode <> 1) AS 'additional_urls',
|
|
24
|
+
concat(rkd_cmp_det.StAdd1, ', ', rkd_cmp_det.Post, ', ', rkd_cmp_det.City) AS 'headquarter_address',
|
|
25
|
+
convert(Date, COALESCE(rkd_cmp_det.PublicSince, (SELECT MIN(MarketDate) FROM DS2PrimQtPrc WHERE InfoCode = exchange_qt.InfoCode))) AS 'inception_date',
|
|
26
|
+
convert(Date, security.DelistDate) AS 'delisted_date',
|
|
27
|
+
convert(Date, cmp_ref.LatestFinAnnDt) AS 'last_annual_report',
|
|
28
|
+
convert(Date, cmp_ref.LatestFinIntmDt) AS 'last_interim_report',
|
|
29
|
+
-- MarketData DL
|
|
30
|
+
exchange_qt.InfoCode AS 'quote_code',
|
|
31
|
+
exchange_qt.ExchIntCode AS 'exchange_code',
|
|
32
|
+
-- Fundamental DL
|
|
33
|
+
rkd_instrument.Code AS 'rkd_code',
|
|
34
|
+
-- Forecast DL
|
|
35
|
+
ibes_mapping.EstPermID AS 'ibes_code'
|
|
36
|
+
|
|
37
|
+
FROM DS2ExchQtInfo AS exchange_qt
|
|
38
|
+
|
|
39
|
+
LEFT JOIN DS2CtryQtInfo AS country_qt
|
|
40
|
+
ON exchange_qt.InfoCode = country_qt.InfoCode
|
|
41
|
+
|
|
42
|
+
LEFT JOIN DS2Security AS security
|
|
43
|
+
ON country_qt.DsSecCode = security.DsSecCode
|
|
44
|
+
|
|
45
|
+
LEFT JOIN vw_SecurityMappingX AS mappingX
|
|
46
|
+
ON mappingX.vencode = security.PrimQtInfoCode
|
|
47
|
+
AND mappingX.ventype = 33
|
|
48
|
+
AND mappingX.rank = 1
|
|
49
|
+
AND mappingX.StartDate = (
|
|
50
|
+
SELECT MAX(I.StartDate)
|
|
51
|
+
FROM vw_SecurityMappingX AS I
|
|
52
|
+
WHERE I.typ = mappingX.typ AND I.vencode = mappingX.vencode AND I.ventype = mappingX.ventype
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
LEFT JOIN vw_SecurityMappingX AS mappingRKD
|
|
56
|
+
ON mappingX.seccode = mappingRKD.seccode
|
|
57
|
+
AND mappingX.typ = mappingRKD.typ
|
|
58
|
+
AND mappingRKD.ventype = 26
|
|
59
|
+
AND mappingRKD.rank = 1
|
|
60
|
+
AND mappingRKD.StartDate = (
|
|
61
|
+
SELECT MAX(I.StartDate)
|
|
62
|
+
FROM vw_SecurityMappingX AS I
|
|
63
|
+
WHERE I.typ = mappingRKD.typ AND I.vencode = mappingRKD.vencode AND I.ventype = mappingRKD.ventype
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
LEFT JOIN RKDFndCmpRefIssue AS rkd_instrument
|
|
67
|
+
ON rkd_instrument.IssueCode = mappingRKD.vencode
|
|
68
|
+
|
|
69
|
+
LEFT JOIN RKDFndCmpDet AS rkd_cmp_det
|
|
70
|
+
ON rkd_cmp_det.Code = rkd_instrument.Code
|
|
71
|
+
|
|
72
|
+
LEFT JOIN RKDFndCmpRef AS cmp_ref
|
|
73
|
+
ON cmp_ref.Code = rkd_instrument.Code
|
|
74
|
+
|
|
75
|
+
LEFT JOIN RKDFndCmpFiling AS filing
|
|
76
|
+
ON filing.Code = rkd_instrument.Code
|
|
77
|
+
AND filing.TxtInfoTypeCode = 2
|
|
78
|
+
|
|
79
|
+
LEFT JOIN vw_IBES2Mapping AS ibes_mapping
|
|
80
|
+
ON ibes_mapping.SecCode = mappingX.seccode
|
|
81
|
+
AND ibes_mapping.typ = mappingX.typ
|
|
82
|
+
AND ibes_mapping.Exchange = (
|
|
83
|
+
CASE
|
|
84
|
+
WHEN ibes_mapping.typ = 6 THEN 0
|
|
85
|
+
WHEN ibes_mapping.typ = 1 THEN 1
|
|
86
|
+
END
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
{% if source_id %}
|
|
90
|
+
where CONCAT(exchange_qt.InfoCode, '-', exchange_qt.ExchIntCode) = {{ source_id }}
|
|
91
|
+
{% endif %}
|
|
92
|
+
|
|
93
|
+
ORDER BY exchange_qt.InfoCode, exchange_qt.ExchIntCode
|
|
94
|
+
|
|
95
|
+
{% if (offset == 0 or offset) and batch %}
|
|
96
|
+
offset {{ offset|sqlsafe }} rows
|
|
97
|
+
fetch next {{ batch|sqlsafe }} rows only
|
|
98
|
+
{% endif %}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import pytz
|
|
2
|
+
from django.conf import settings
|
|
3
|
+
from django.db import connections
|
|
4
|
+
from django.db.models import Max
|
|
5
|
+
from wbcore.contrib.dataloader.utils import dictfetchall, dictfetchone
|
|
6
|
+
from wbcore.contrib.geography.models import Geography
|
|
7
|
+
from wbcore.utils.cache import mapping
|
|
8
|
+
from wbfdm.models import Exchange
|
|
9
|
+
from wbfdm.sync.abstract import Sync
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class QAExchangeSync(Sync[Exchange]):
|
|
13
|
+
SOURCE = "qa-ds2-exchange"
|
|
14
|
+
|
|
15
|
+
def update(self):
|
|
16
|
+
sql = "SELECT ExchIntCode AS source_id, ExchName AS name, ExchCtryCode AS country_id, ExchMnem AS refinitiv_mnemonic FROM DS2Exchange"
|
|
17
|
+
Exchange.objects.bulk_create(
|
|
18
|
+
map(
|
|
19
|
+
lambda data: Exchange(
|
|
20
|
+
source=self.SOURCE,
|
|
21
|
+
source_id=data["source_id"],
|
|
22
|
+
name=data["name"],
|
|
23
|
+
country_id=mapping(Geography.countries, "code_2").get(data["country_id"]),
|
|
24
|
+
refinitiv_mnemonic=data["refinitiv_mnemonic"],
|
|
25
|
+
),
|
|
26
|
+
dictfetchall(connections["qa"].cursor().execute(sql)),
|
|
27
|
+
),
|
|
28
|
+
update_conflicts=True,
|
|
29
|
+
update_fields=["name", "country", "refinitiv_mnemonic"],
|
|
30
|
+
unique_fields=["source", "source_id"],
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
def update_or_create_item(self, external_id: int) -> Exchange:
|
|
34
|
+
defaults = self.get_item(external_id)
|
|
35
|
+
defaults["country_id"] = mapping(Geography.countries, "code_2").get(defaults["country_id"])
|
|
36
|
+
exchange, _ = Exchange.objects.update_or_create(
|
|
37
|
+
source=self.SOURCE,
|
|
38
|
+
source_id=external_id,
|
|
39
|
+
defaults=defaults,
|
|
40
|
+
)
|
|
41
|
+
return exchange
|
|
42
|
+
|
|
43
|
+
def update_item(self, item: Exchange) -> Exchange:
|
|
44
|
+
return self.update_or_create_item(item.source_id)
|
|
45
|
+
|
|
46
|
+
def get_item(self, external_id: int) -> dict:
|
|
47
|
+
sql = "SELECT ExchName AS name, ExchCtryCode AS country_id, ExchMnem AS refinitiv_mnemonic FROM DS2Exchange WHERE ExchIntCode = %s"
|
|
48
|
+
return dictfetchone(connections["qa"].cursor().execute(sql, (external_id,)))
|
|
49
|
+
|
|
50
|
+
def trigger_partial_update(self):
|
|
51
|
+
max_last_updated = (
|
|
52
|
+
Exchange.objects.filter(source=self.SOURCE)
|
|
53
|
+
.aggregate(max_last_updated=Max("last_updated"))
|
|
54
|
+
.get("max_last_updated")
|
|
55
|
+
)
|
|
56
|
+
if max_last_updated is None:
|
|
57
|
+
self.update()
|
|
58
|
+
else:
|
|
59
|
+
with connections["qa"].cursor() as cursor:
|
|
60
|
+
cursor.execute(
|
|
61
|
+
"SELECT MAX(last_user_update) FROM sys.dm_db_index_usage_stats WHERE OBJECT_NAME(object_id) = 'DS2Exchange_changes'"
|
|
62
|
+
)
|
|
63
|
+
max_last_updated_qa = (
|
|
64
|
+
pytz.timezone(settings.TIME_ZONE).localize(result[0]) if (result := cursor.fetchone()) else None
|
|
65
|
+
)
|
|
66
|
+
if max_last_updated_qa and max_last_updated_qa > max_last_updated:
|
|
67
|
+
for _, exchange_id in cursor.execute(
|
|
68
|
+
"SELECT UpdateFlag_, ExchIntCode FROM DS2Exchange_changes"
|
|
69
|
+
).fetchall():
|
|
70
|
+
self.update_or_create_item(exchange_id)
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
from django.db import connections
|
|
2
|
+
from wbfdm.contrib.qa.sync.utils import (
|
|
3
|
+
get_item,
|
|
4
|
+
trigger_partial_update,
|
|
5
|
+
update_instruments,
|
|
6
|
+
update_or_create_item,
|
|
7
|
+
)
|
|
8
|
+
from wbfdm.models import Instrument
|
|
9
|
+
from wbfdm.sync.abstract import Sync
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class QACompanySync(Sync[Instrument]):
|
|
13
|
+
SOURCE = "qa-ds2-company"
|
|
14
|
+
|
|
15
|
+
def update(self, **kwargs):
|
|
16
|
+
update_instruments("qa/sql/companies.sql", **kwargs)
|
|
17
|
+
|
|
18
|
+
def update_or_create_item(self, external_id: int) -> Instrument:
|
|
19
|
+
return update_or_create_item(external_id, self.get_item, self.SOURCE)
|
|
20
|
+
|
|
21
|
+
def update_item(self, item: Instrument) -> Instrument:
|
|
22
|
+
return self.update_or_create_item(item.source_id)
|
|
23
|
+
|
|
24
|
+
def get_item(self, external_id: int) -> dict:
|
|
25
|
+
return get_item(external_id, "qa/sql/companies.sql")
|
|
26
|
+
|
|
27
|
+
def trigger_partial_update(self):
|
|
28
|
+
trigger_partial_update(
|
|
29
|
+
Instrument.objects.filter(source=self.SOURCE),
|
|
30
|
+
"last_update",
|
|
31
|
+
"DsCmpyCode",
|
|
32
|
+
"DS2Company_changes",
|
|
33
|
+
self.update,
|
|
34
|
+
self.update_or_create_item,
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class QAInstrumentSync(Sync[Instrument]):
|
|
39
|
+
SOURCE = "qa-ds2-security"
|
|
40
|
+
|
|
41
|
+
def update(self, **kwargs):
|
|
42
|
+
update_instruments("qa/sql/instruments.sql", parent_source="qa-ds2-company", **kwargs)
|
|
43
|
+
|
|
44
|
+
def update_or_create_item(self, external_id: int) -> Instrument:
|
|
45
|
+
return update_or_create_item(external_id, self.get_item, self.SOURCE, "qa-ds2-company")
|
|
46
|
+
|
|
47
|
+
def update_item(self, item: Instrument) -> Instrument:
|
|
48
|
+
return self.update_or_create_item(item.source_id)
|
|
49
|
+
|
|
50
|
+
def get_item(self, external_id: int) -> dict:
|
|
51
|
+
return get_item(external_id, "qa/sql/instruments.sql")
|
|
52
|
+
|
|
53
|
+
def trigger_partial_update(self):
|
|
54
|
+
trigger_partial_update(
|
|
55
|
+
Instrument.objects.filter(source=self.SOURCE),
|
|
56
|
+
"last_update",
|
|
57
|
+
"DsSecCode",
|
|
58
|
+
"DS2Security_changes",
|
|
59
|
+
self.update,
|
|
60
|
+
self.update_or_create_item,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class QAQuoteSync(Sync[Instrument]):
|
|
65
|
+
SOURCE = "qa-ds2-quote"
|
|
66
|
+
|
|
67
|
+
def update(self, **kwargs):
|
|
68
|
+
count = connections["qa"].cursor().execute("SELECT COUNT(*) FROM DS2ExchQtInfo").fetchone()[0]
|
|
69
|
+
for offset in range(0, count, 100_000):
|
|
70
|
+
update_instruments(
|
|
71
|
+
"qa/sql/quotes.sql",
|
|
72
|
+
parent_source="qa-ds2-security",
|
|
73
|
+
context={"offset": offset, "batch": 100_000},
|
|
74
|
+
**kwargs,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def update_or_create_item(self, external_id: int) -> Instrument:
|
|
78
|
+
return update_or_create_item(external_id, self.get_item, self.SOURCE, "qa-ds2-security")
|
|
79
|
+
|
|
80
|
+
def update_item(self, item: Instrument) -> Instrument:
|
|
81
|
+
return self.update_or_create_item(item.source_id)
|
|
82
|
+
|
|
83
|
+
def get_item(self, external_id: int) -> dict:
|
|
84
|
+
return get_item(external_id, "qa/sql/quotes.sql")
|
|
85
|
+
|
|
86
|
+
def trigger_partial_update(self):
|
|
87
|
+
trigger_partial_update(
|
|
88
|
+
Instrument.objects.filter(source=self.SOURCE),
|
|
89
|
+
"last_update",
|
|
90
|
+
"CONCAT(InfoCode, '-', ExchIntCode)",
|
|
91
|
+
"DS2ExchQtInfo_changes",
|
|
92
|
+
self.update,
|
|
93
|
+
self.update_or_create_item,
|
|
94
|
+
)
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
from contextlib import suppress
|
|
2
|
+
from typing import Callable
|
|
3
|
+
|
|
4
|
+
import pytz
|
|
5
|
+
from django.conf import settings
|
|
6
|
+
from django.db import connections
|
|
7
|
+
from django.db.models import Max, QuerySet
|
|
8
|
+
from django.template.loader import get_template
|
|
9
|
+
from jinjasql import JinjaSql # type: ignore
|
|
10
|
+
from tqdm import tqdm
|
|
11
|
+
from wbcore.contrib.currency.models import Currency
|
|
12
|
+
from wbcore.contrib.dataloader.utils import dictfetchall, dictfetchone
|
|
13
|
+
from wbcore.contrib.geography.models import Geography
|
|
14
|
+
from wbcore.utils.cache import mapping
|
|
15
|
+
from wbfdm.models.exchanges.exchanges import Exchange
|
|
16
|
+
from wbfdm.models.instruments.instruments import Instrument, InstrumentType
|
|
17
|
+
|
|
18
|
+
BATCH_SIZE: int = 10000
|
|
19
|
+
instrument_type_map = {
|
|
20
|
+
"ADR": "american_depository_receipt",
|
|
21
|
+
"CF": "close_ended_fund",
|
|
22
|
+
"EQ": "equity",
|
|
23
|
+
"ET": "exchange_traded_fund",
|
|
24
|
+
"ETC": "exchange_traded_commodity",
|
|
25
|
+
"ETN": "exchange_traded_note",
|
|
26
|
+
"EWT": "warrant",
|
|
27
|
+
"GDR": "global_depository_receipt",
|
|
28
|
+
"GNSH": "genussschein",
|
|
29
|
+
"INVT": "investment_trust",
|
|
30
|
+
"NVDR": "non_voting_depository_receipt",
|
|
31
|
+
"PREF": "preference_share",
|
|
32
|
+
"UT": "abc",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def get_dataloader_parameters(
|
|
37
|
+
quote_code: str | None = None,
|
|
38
|
+
exchange_code: str | None = None,
|
|
39
|
+
rkd_code: str | None = None,
|
|
40
|
+
ibes_code: str | None = None,
|
|
41
|
+
) -> dict:
|
|
42
|
+
dataloader_parameters = dict()
|
|
43
|
+
if quote_code:
|
|
44
|
+
dataloader_parameters["adjustments"] = {
|
|
45
|
+
"path": "wbfdm.contrib.qa.dataloaders.adjustments.DatastreamAdjustmentsDataloader",
|
|
46
|
+
"parameters": quote_code,
|
|
47
|
+
}
|
|
48
|
+
dataloader_parameters["corporate_actions"] = {
|
|
49
|
+
"path": "wbfdm.contrib.qa.dataloaders.corporate_actions.DatastreamCorporateActionsDataloader",
|
|
50
|
+
"parameters": quote_code,
|
|
51
|
+
}
|
|
52
|
+
if exchange_code:
|
|
53
|
+
dataloader_parameters["market_data"] = {
|
|
54
|
+
"path": "wbfdm.contrib.qa.dataloaders.market_data.DatastreamMarketDataDataloader",
|
|
55
|
+
"parameters": [
|
|
56
|
+
quote_code,
|
|
57
|
+
exchange_code,
|
|
58
|
+
],
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if rkd_code:
|
|
62
|
+
dataloader_parameters["officers"] = {
|
|
63
|
+
"path": "wbfdm.contrib.qa.dataloaders.officers.RKDOfficersDataloader",
|
|
64
|
+
"parameters": rkd_code,
|
|
65
|
+
}
|
|
66
|
+
dataloader_parameters["statements"] = {
|
|
67
|
+
"path": "wbfdm.contrib.qa.dataloaders.statements.RKDStatementsDataloader",
|
|
68
|
+
"parameters": rkd_code,
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if ibes_code:
|
|
72
|
+
dataloader_parameters["financials"] = {
|
|
73
|
+
"path": "wbfdm.contrib.qa.dataloaders.financials.IBESFinancialsDataloader",
|
|
74
|
+
"parameters": ibes_code,
|
|
75
|
+
}
|
|
76
|
+
dataloader_parameters["reporting_dates"] = {
|
|
77
|
+
"path": "wbfdm.contrib.qa.dataloaders.reporting_dates.IbesReportingDateDataloader",
|
|
78
|
+
"parameters": ibes_code,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
return dataloader_parameters
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def convert_data(data, parent_source: str | None = None) -> dict:
|
|
85
|
+
if len(data.keys()) == 0:
|
|
86
|
+
return data
|
|
87
|
+
data["country_id"] = mapping(Geography.countries, "code_2").get(data["country_id"])
|
|
88
|
+
data["currency_id"] = mapping(Currency.objects).get(data["currency_id"])
|
|
89
|
+
if instrument_type_id := data.pop("instrument_type_id"):
|
|
90
|
+
try:
|
|
91
|
+
data["instrument_type_id"] = mapping(InstrumentType.objects)[
|
|
92
|
+
instrument_type_map.get(instrument_type_id, instrument_type_id)
|
|
93
|
+
]
|
|
94
|
+
except KeyError:
|
|
95
|
+
data["instrument_type_id"] = InstrumentType.objects.get_or_create(
|
|
96
|
+
key=instrument_type_id.lower(),
|
|
97
|
+
defaults={"name": instrument_type_id.title(), "short_name": instrument_type_id.title()},
|
|
98
|
+
)[0].pk
|
|
99
|
+
data["exchange_id"] = mapping(Exchange.objects, "source_id", source="qa-ds2-exchange").get(
|
|
100
|
+
str(data["exchange_code"])
|
|
101
|
+
)
|
|
102
|
+
data["dl_parameters"] = get_dataloader_parameters(
|
|
103
|
+
data.pop("quote_code"), data.pop("exchange_code"), data.pop("rkd_code"), data.pop("ibes_code")
|
|
104
|
+
)
|
|
105
|
+
data["additional_urls"] = urls.split(",") if (urls := data.get("additional_urls")) else []
|
|
106
|
+
if parent_source:
|
|
107
|
+
data["parent_id"] = mapping(Instrument.objects, "source_id", source=parent_source).get(str(data["parent_id"]))
|
|
108
|
+
if (is_primary := data.get("is_primary")) and isinstance(is_primary, str):
|
|
109
|
+
data["is_primary"] = is_primary.lower().strip() == "y"
|
|
110
|
+
data.pop("phone")
|
|
111
|
+
return data
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def get_instrument_from_data(data, parent_source: str | None = None) -> Instrument | None:
|
|
115
|
+
if len(data.keys()) == 0:
|
|
116
|
+
return None
|
|
117
|
+
instrument = Instrument(**convert_data(data, parent_source))
|
|
118
|
+
instrument.pre_save()
|
|
119
|
+
instrument.computed_str = instrument.compute_str()
|
|
120
|
+
if (
|
|
121
|
+
instrument.name and instrument.currency and instrument.instrument_type
|
|
122
|
+
): # we return an instrument only if it contains the basic name, currency and type
|
|
123
|
+
return instrument
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _bulk_create_instruments_chunk(instruments: list[Instrument], update_unique_identifiers: bool = False):
|
|
127
|
+
update_fields = [
|
|
128
|
+
"name",
|
|
129
|
+
"dl_parameters",
|
|
130
|
+
"description",
|
|
131
|
+
"country",
|
|
132
|
+
"currency",
|
|
133
|
+
"parent",
|
|
134
|
+
"primary_url",
|
|
135
|
+
"additional_urls",
|
|
136
|
+
"headquarter_address",
|
|
137
|
+
"inception_date",
|
|
138
|
+
"delisted_date",
|
|
139
|
+
"exchange",
|
|
140
|
+
"instrument_type",
|
|
141
|
+
"computed_str",
|
|
142
|
+
"search_vector",
|
|
143
|
+
"trigram_search_vector",
|
|
144
|
+
"is_primary",
|
|
145
|
+
]
|
|
146
|
+
bulk_update_kwargs = {"update_conflicts": True}
|
|
147
|
+
if update_unique_identifiers:
|
|
148
|
+
update_fields.extend(
|
|
149
|
+
[
|
|
150
|
+
"refinitiv_identifier_code",
|
|
151
|
+
"refinitiv_mnemonic_code",
|
|
152
|
+
"isin",
|
|
153
|
+
"sedol",
|
|
154
|
+
"valoren",
|
|
155
|
+
"cusip",
|
|
156
|
+
]
|
|
157
|
+
)
|
|
158
|
+
bulk_update_kwargs = {"ignore_conflicts": True}
|
|
159
|
+
Instrument.objects.bulk_create(
|
|
160
|
+
instruments, update_fields=update_fields, unique_fields=["source", "source_id"], **bulk_update_kwargs
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def update_instruments(sql_name: str, parent_source: str | None = None, context=None, debug: bool = False, **kwargs):
|
|
165
|
+
template = get_template(sql_name, using="jinja").template # type: ignore
|
|
166
|
+
if context is None:
|
|
167
|
+
context = {}
|
|
168
|
+
query, params = JinjaSql(param_style="format").prepare_query(template, context)
|
|
169
|
+
instruments = []
|
|
170
|
+
|
|
171
|
+
gen = dictfetchall(connections["qa"].cursor().execute(query, params))
|
|
172
|
+
if debug:
|
|
173
|
+
gen = tqdm(gen)
|
|
174
|
+
# we update in batch to be sure to not exhaust resources
|
|
175
|
+
for row in gen:
|
|
176
|
+
with suppress(TypeError): # we don't fail if the given data doesn't satisfy the schema
|
|
177
|
+
if instrument := get_instrument_from_data(row, parent_source=parent_source):
|
|
178
|
+
instruments.append(instrument)
|
|
179
|
+
|
|
180
|
+
if len(instruments) >= BATCH_SIZE:
|
|
181
|
+
_bulk_create_instruments_chunk(instruments, **kwargs)
|
|
182
|
+
instruments = []
|
|
183
|
+
|
|
184
|
+
_bulk_create_instruments_chunk(instruments, **kwargs)
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def update_or_create_item(
|
|
188
|
+
external_id: int, get_item: Callable, source: str, parent_source: str | None = None
|
|
189
|
+
) -> Instrument:
|
|
190
|
+
defaults = convert_data(get_item(external_id), parent_source=parent_source)
|
|
191
|
+
|
|
192
|
+
dl_parameters = defaults.pop("dl_parameters", {})
|
|
193
|
+
defaults.pop("source", None)
|
|
194
|
+
defaults.pop("source_id", None)
|
|
195
|
+
|
|
196
|
+
instrument, _ = Instrument.objects.update_or_create(
|
|
197
|
+
source=source,
|
|
198
|
+
source_id=external_id,
|
|
199
|
+
defaults=defaults,
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
for key, value in dl_parameters.items():
|
|
203
|
+
instrument.dl_parameters[key] = value
|
|
204
|
+
|
|
205
|
+
instrument.save()
|
|
206
|
+
|
|
207
|
+
return instrument
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def get_item(external_id: int, template_name: str) -> dict:
|
|
211
|
+
template = get_template(template_name, using="jinja").template # type: ignore
|
|
212
|
+
query, params = JinjaSql(param_style="format").prepare_query(template, {"source_id": external_id})
|
|
213
|
+
return dictfetchone(connections["qa"].cursor().execute(query, params))
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def trigger_partial_update(
|
|
217
|
+
queryset: QuerySet,
|
|
218
|
+
last_update_field: str,
|
|
219
|
+
id_field: str,
|
|
220
|
+
table_change_name: str,
|
|
221
|
+
update: Callable,
|
|
222
|
+
update_or_create_item: Callable,
|
|
223
|
+
):
|
|
224
|
+
max_last_updated = queryset.aggregate(max_last_updated=Max(last_update_field)).get("max_last_updated")
|
|
225
|
+
|
|
226
|
+
if max_last_updated is None:
|
|
227
|
+
update()
|
|
228
|
+
|
|
229
|
+
else:
|
|
230
|
+
with connections["qa"].cursor() as cursor:
|
|
231
|
+
cursor.execute(
|
|
232
|
+
f"SELECT MAX(last_user_update) FROM sys.dm_db_index_usage_stats WHERE OBJECT_NAME(object_id) = '{table_change_name}'"
|
|
233
|
+
)
|
|
234
|
+
max_last_updated_qa = (
|
|
235
|
+
pytz.timezone(settings.TIME_ZONE).localize(result[0]) if (result := cursor.fetchone()) else None
|
|
236
|
+
)
|
|
237
|
+
if max_last_updated_qa and max_last_updated_qa > max_last_updated:
|
|
238
|
+
for _, security_id in cursor.execute(
|
|
239
|
+
f"SELECT UpdateFlag_, {id_field} FROM {table_change_name}"
|
|
240
|
+
).fetchall():
|
|
241
|
+
update_or_create_item(security_id)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: wbfdm
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.6
|
|
4
4
|
Summary: The workbench module ensures rapid access to diverse financial data (market, fundamental, forecasts, ESG), with features for storing instruments, classifying them, and conducting financial analysis.
|
|
5
5
|
Author-email: Christopher Wittlinger <c.wittlinger@stainly.com>
|
|
6
6
|
Requires-Dist: roman==4.*
|
|
@@ -34,9 +34,11 @@ wbfdm/analysis/financial_analysis/utils.py,sha256=ko88qzmt0zv0D1xjvK4OU5VRF7Uu45
|
|
|
34
34
|
wbfdm/analysis/technical_analysis/__init__.py,sha256=nQdpDl2okzWID6UT7fyfl36RSLNO1ZZzxRt_mkGVVkY,50
|
|
35
35
|
wbfdm/analysis/technical_analysis/technical_analysis.py,sha256=MkK5HrLz9_TCVAv4qDDZjx6EIqsvusMYB7ybg3MvSsg,4961
|
|
36
36
|
wbfdm/analysis/technical_analysis/traces.py,sha256=1wzmW7JcQXki0q14TdG36tacZpIFJ_bO4KHx_5TgNxc,6371
|
|
37
|
+
wbfdm/backends/dto.py,sha256=5IdeGVsrk8trEi9rqtq-zisqiEDE_VLBP8RxlfZZnjk,596
|
|
37
38
|
wbfdm/contrib/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
38
39
|
wbfdm/contrib/dsws/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
39
40
|
wbfdm/contrib/dsws/client.py,sha256=DFsKj8LultW1MlTlxl1EHuzuZW7UcIIy25v0L2HM5Z4,13788
|
|
41
|
+
wbfdm/contrib/dsws/dataloaders/market_data.py,sha256=TVL0Vzmg8hDzjbqBWx1342dvT_qC5gVp8E4i3hVcivA,6003
|
|
40
42
|
wbfdm/contrib/internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
41
43
|
wbfdm/contrib/internal/dataloaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
42
44
|
wbfdm/contrib/internal/dataloaders/market_data.py,sha256=V-kCHgQZVgLkxAmB0xp0BOV48xkcl_YoQzFokKSoL3M,3259
|
|
@@ -101,6 +103,17 @@ wbfdm/contrib/qa/dataloaders/market_data.py,sha256=gAzy00nhFWqABG7kjvCfvpyYnehWI
|
|
|
101
103
|
wbfdm/contrib/qa/dataloaders/officers.py,sha256=QfsMbnlWOhvJc4jyPHJX3STqIwOUqlCCoZddELc405E,2268
|
|
102
104
|
wbfdm/contrib/qa/dataloaders/reporting_dates.py,sha256=Nq83u7eAhsPe9b4ePI5xcgvWEgW1XwHLyVzxpbgcdnQ,2492
|
|
103
105
|
wbfdm/contrib/qa/dataloaders/statements.py,sha256=tE5ux6bFmAezivssRkm3cNW51lJLoAjHbUao1gR4JMM,10093
|
|
106
|
+
wbfdm/contrib/qa/jinja2/qa/sql/companies.sql,sha256=RzTkfVjBVOgyirgKxp2rnJdjsKl8d3YM-d2qdjHx9cw,3801
|
|
107
|
+
wbfdm/contrib/qa/jinja2/qa/sql/instruments.sql,sha256=7p31aJDtKusfSzhrjyUo8FlmbPXNyffWtNclm8DZkFM,3863
|
|
108
|
+
wbfdm/contrib/qa/jinja2/qa/sql/quotes.sql,sha256=JEljMTlHM4978cvJkiKnKXn09FrcgcQqq4KkwWQUwcM,3910
|
|
109
|
+
wbfdm/contrib/qa/jinja2/qa/sql/ibes/base_estimates.sql,sha256=xmIrDN0LSzWPRFp2cdZbKq8h-pL80UIAmwflbxVdKWg,1005
|
|
110
|
+
wbfdm/contrib/qa/jinja2/qa/sql/ibes/calendarized.sql,sha256=w2vi0lgNDDMoFB07bNv4HB_FULQ-kbcb2VAhaQIc5pc,1671
|
|
111
|
+
wbfdm/contrib/qa/jinja2/qa/sql/ibes/complete.sql,sha256=bkF1h7FcLdeXmmKy2W4rdnBZw9xRVIo_Cu1sVobU-RM,328
|
|
112
|
+
wbfdm/contrib/qa/jinja2/qa/sql/ibes/estimates.sql,sha256=OWgeogSFj7-OdXvJTvNsThNBz-7kxtygKYvS7dkU3aw,156
|
|
113
|
+
wbfdm/contrib/qa/jinja2/qa/sql/ibes/financials.sql,sha256=0jXNMdX8ixVRJ8YyvtoJRQ510pHFYEioMy7NTp5ecck,2582
|
|
114
|
+
wbfdm/contrib/qa/sync/exchanges.py,sha256=831-0bqxq4LaHEkyvMGfEBkNFm5LyLCSpjZbPdTEF0A,3079
|
|
115
|
+
wbfdm/contrib/qa/sync/instruments.py,sha256=8aTQVJ_cw1phe4FWikn79pjCfUijaTcwkdhQCtSXKH0,3156
|
|
116
|
+
wbfdm/contrib/qa/sync/utils.py,sha256=TYZY3QB75POUmme1U71yo6WPzf4AXs4pWpFdI1Hq2GY,8936
|
|
104
117
|
wbfdm/dataloaders/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
105
118
|
wbfdm/dataloaders/cache.py,sha256=K9BeVxT7p-BMvjurINt18bfrUDccp840uIjfDBLJRNk,4841
|
|
106
119
|
wbfdm/dataloaders/protocols.py,sha256=wCBHWPkTGVHMcAJU6VkuwisKyvMNJt1jTsSerJDuX2M,3024
|
|
@@ -332,6 +345,6 @@ wbfdm/viewsets/statements/__init__.py,sha256=odxtFYUDICPmz8WCE3nx93EvKZLSPBEI4d7
|
|
|
332
345
|
wbfdm/viewsets/statements/statements.py,sha256=AVU5A7p63uWITEcMDl95iKyN-fBO5RwJlbxHrJZvz7o,4083
|
|
333
346
|
wbfdm/viewsets/technical_analysis/__init__.py,sha256=qtCIBg0uSiZeJq_1tEQFilnorMBkMe6uCMfqar6-cLE,77
|
|
334
347
|
wbfdm/viewsets/technical_analysis/monthly_performances.py,sha256=8VgafjW4efEJKTfBqUvAu-EG2eNawl2Cmh9QFt4sN8w,3973
|
|
335
|
-
wbfdm-2.2.
|
|
336
|
-
wbfdm-2.2.
|
|
337
|
-
wbfdm-2.2.
|
|
348
|
+
wbfdm-2.2.6.dist-info/METADATA,sha256=rx0b7o7rcZWujx_C7xPj6iyGVlqo0irHJxgosC5WXQg,667
|
|
349
|
+
wbfdm-2.2.6.dist-info/WHEEL,sha256=tkmg4JIqwd9H8mL30xA7crRmoStyCtGp0VWshokd1Jc,105
|
|
350
|
+
wbfdm-2.2.6.dist-info/RECORD,,
|