PYield 0.7.9__tar.gz → 0.7.10__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. {pyield-0.7.9 → pyield-0.7.10}/PKG-INFO +2 -2
  2. {pyield-0.7.9 → pyield-0.7.10}/README.md +1 -1
  3. pyield-0.7.10/pyield/__about__.py +1 -0
  4. {pyield-0.7.9 → pyield-0.7.10}/pyield/futures/historical.py +9 -13
  5. {pyield-0.7.9 → pyield-0.7.10}/pyield/futures/intraday.py +64 -12
  6. pyield-0.7.10/pyield/futures/xml.py +261 -0
  7. {pyield-0.7.9 → pyield-0.7.10}/pyproject.toml +1 -1
  8. pyield-0.7.10/tests/data/SPRD231222.zip +0 -0
  9. pyield-0.7.10/tests/data/SPRD240426.zip +0 -0
  10. pyield-0.7.9/pyield/__about__.py +0 -1
  11. pyield-0.7.9/tests/xml.py +0 -349
  12. {pyield-0.7.9 → pyield-0.7.10}/LICENSE +0 -0
  13. {pyield-0.7.9 → pyield-0.7.10}/pyield/__init__.py +0 -0
  14. {pyield-0.7.9 → pyield-0.7.10}/pyield/bday.py +0 -0
  15. {pyield-0.7.9 → pyield-0.7.10}/pyield/data_access.py +0 -0
  16. {pyield-0.7.9 → pyield-0.7.10}/pyield/futures/__init__.py +0 -0
  17. {pyield-0.7.9 → pyield-0.7.10}/pyield/holidays/__init__.py +0 -0
  18. {pyield-0.7.9 → pyield-0.7.10}/pyield/holidays/br_holidays_new.txt +0 -0
  19. {pyield-0.7.9 → pyield-0.7.10}/pyield/holidays/br_holidays_old.txt +0 -0
  20. {pyield-0.7.9 → pyield-0.7.10}/pyield/holidays/core.py +0 -0
  21. {pyield-0.7.9 → pyield-0.7.10}/pyield/indicators.py +0 -0
  22. {pyield-0.7.9 → pyield-0.7.10}/pyield/projections.py +0 -0
  23. {pyield-0.7.9 → pyield-0.7.10}/pyield/py.typed +0 -0
  24. {pyield-0.7.9 → pyield-0.7.10}/pyield/spreads.py +0 -0
  25. {pyield-0.7.9 → pyield-0.7.10}/pyield/treasuries.py +0 -0
  26. {pyield-0.7.9 → pyield-0.7.10}/pyield/utils.py +0 -0
  27. {pyield-0.7.9 → pyield-0.7.10}/tests/__init__.py +0 -0
  28. {pyield-0.7.9 → pyield-0.7.10}/tests/test_bday.py +0 -0
  29. {pyield-0.7.9 → pyield-0.7.10}/tests/test_futures.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: PYield
3
- Version: 0.7.9
3
+ Version: 0.7.10
4
4
  Summary: A Python library for analysis of fixed income instruments in Brazil
5
5
  Keywords: fixed-income, brazil, finance, analysis, bonds
6
6
  Author-Email: Carlos Carvalho <cr.cj@outlook.com>
@@ -101,7 +101,7 @@ dtype: datetime64[ns]
101
101
  ```python
102
102
  # Fetch current DI Futures data from B3 (15 minutes delay)
103
103
  >>> yd.fetch_asset(asset_code="DI1")
104
- TradeTimestamp TickerSymbol ExpirationDate BDaysToExp ... MaxRate LastAskRate LastBidRate CurrentRate
104
+ TradeTime TickerSymbol ExpirationDate BDaysToExp ... MaxRate LastAskRate LastBidRate CurrentRate
105
105
  2024-04-21 13:37:39 DI1K24 2024-05-02 7 ... 0.10660 0.10652 0.10660 0.10660
106
106
  2024-04-21 13:37:39 DI1M24 2024-06-03 28 ... 0.10518 0.10510 0.10516 0.10518
107
107
  2024-04-21 13:37:39 DI1N24 2024-07-01 48 ... 0.10480 0.10456 0.10462 0.10460
@@ -58,7 +58,7 @@ dtype: datetime64[ns]
58
58
  ```python
59
59
  # Fetch current DI Futures data from B3 (15 minutes delay)
60
60
  >>> yd.fetch_asset(asset_code="DI1")
61
- TradeTimestamp TickerSymbol ExpirationDate BDaysToExp ... MaxRate LastAskRate LastBidRate CurrentRate
61
+ TradeTime TickerSymbol ExpirationDate BDaysToExp ... MaxRate LastAskRate LastBidRate CurrentRate
62
62
  2024-04-21 13:37:39 DI1K24 2024-05-02 7 ... 0.10660 0.10652 0.10660 0.10660
63
63
  2024-04-21 13:37:39 DI1M24 2024-06-03 28 ... 0.10518 0.10510 0.10516 0.10518
64
64
  2024-04-21 13:37:39 DI1N24 2024-07-01 48 ... 0.10480 0.10456 0.10462 0.10460
@@ -0,0 +1 @@
1
+ __version__ = "0.7.10"
@@ -185,12 +185,12 @@ def fetch_raw_df(asset_code: str, trade_date: pd.Timestamp) -> pd.DataFrame:
185
185
  r = requests.get(url)
186
186
 
187
187
  text = r.text
188
- if "AJUSTE" not in text:
188
+ if "VENCTO" not in text:
189
189
  return pd.DataFrame()
190
190
 
191
191
  df = pd.read_html(
192
192
  io.StringIO(text),
193
- match="AJUSTE",
193
+ match="VENCTO",
194
194
  header=1,
195
195
  thousands=".",
196
196
  decimal=",",
@@ -239,7 +239,7 @@ def rename_columns(df: pd.DataFrame) -> pd.DataFrame:
239
239
  "PREÇO MÍN.": "MinRate",
240
240
  "PREÇO MÉD.": "AvgRate",
241
241
  "PREÇO MÁX.": "MaxRate",
242
- "PREÇO ABERTU.": "FirstRate",
242
+ "PREÇO ABERTU.": "OpenRate",
243
243
  "ÚLT. PREÇO": "CloseRate",
244
244
  "VAR. PTOS.": "PointsVariation",
245
245
  # Attention: bid/ask rates are inverted
@@ -251,10 +251,9 @@ def rename_columns(df: pd.DataFrame) -> pd.DataFrame:
251
251
 
252
252
 
253
253
  def process_df(
254
- df: pd.DataFrame,
255
- trade_date: pd.Timestamp,
256
- asset_code: str,
254
+ input_df: pd.DataFrame, trade_date: pd.Timestamp, asset_code: str
257
255
  ) -> pd.DataFrame:
256
+ df = input_df.copy()
258
257
  df["TradeDate"] = trade_date
259
258
  # Convert to datetime64[ns] since it is pandas default type for timestamps
260
259
  df["TradeDate"] = df["TradeDate"].astype("datetime64[ns]")
@@ -308,7 +307,7 @@ def process_df(
308
307
  return df
309
308
 
310
309
 
311
- def reorder_columns(df: pd.DataFrame):
310
+ def select_and_reorder_columns(df: pd.DataFrame):
312
311
  all_columns = [
313
312
  "TradeDate",
314
313
  "TickerSymbol",
@@ -324,7 +323,7 @@ def reorder_columns(df: pd.DataFrame):
324
323
  "FinancialVolume",
325
324
  "SettlementPrice",
326
325
  "SettlementRate",
327
- "FirstRate",
326
+ "OpenRate",
328
327
  "MinRate",
329
328
  "AvgRate",
330
329
  "MaxRate",
@@ -336,10 +335,7 @@ def reorder_columns(df: pd.DataFrame):
336
335
  return df[reordered_columns]
337
336
 
338
337
 
339
- def fetch_historical_df(
340
- asset_code: str,
341
- trade_date: pd.Timestamp,
342
- ) -> pd.DataFrame:
338
+ def fetch_historical_df(asset_code: str, trade_date: pd.Timestamp) -> pd.DataFrame:
343
339
  """
344
340
  Fetchs the futures data for a given date from B3.
345
341
 
@@ -360,5 +356,5 @@ def fetch_historical_df(
360
356
  return df_raw
361
357
  df = rename_columns(df_raw)
362
358
  df = process_df(df, trade_date, asset_code)
363
- df = reorder_columns(df)
359
+ df = select_and_reorder_columns(df)
364
360
  return df
@@ -1,6 +1,8 @@
1
1
  import pandas as pd
2
2
  import requests
3
3
 
4
+ from .. import bday
5
+
4
6
 
5
7
  def _fetch_raw_df(future_code: str) -> pd.DataFrame:
6
8
  """
@@ -50,7 +52,7 @@ def _fetch_raw_df(future_code: str) -> pd.DataFrame:
50
52
  now = pd.Timestamp.now().round("s")
51
53
  # Subtract 15 minutes from the current time to account for API delay
52
54
  trade_ts = now - pd.Timedelta(minutes=15)
53
- df["TradeTimestamp"] = trade_ts
55
+ df["TradeTime"] = trade_ts
54
56
 
55
57
  # Convert DataFrame to use nullable data types for better type consistency
56
58
  df = df.convert_dtypes(dtype_backend="numpy_nullable")
@@ -58,15 +60,19 @@ def _fetch_raw_df(future_code: str) -> pd.DataFrame:
58
60
  return df
59
61
 
60
62
 
61
- def _process_raw_df(raw_df: pd.DataFrame) -> pd.DataFrame:
62
- df = raw_df.copy()
63
+ def _rename_columns(df: pd.DataFrame) -> pd.DataFrame:
64
+ """
65
+ Rename the columns of a DataFrame containing the futures data.
63
66
 
64
- # Columns to be renamed
67
+ Args:
68
+ df (pd.DataFrame): A DataFrame containing futures data.
69
+
70
+ Returns:
71
+ pd.DataFrame: A DataFrame with the columns renamed.
72
+ """
65
73
  all_columns = {
66
- "TradeTimestamp": "TradeTimestamp",
67
74
  "symb": "TickerSymbol",
68
75
  "mtrtyCode": "ExpirationDate",
69
- "BDaysToExp": "BDaysToExp",
70
76
  "opnCtrcts": "OpenContracts",
71
77
  "tradQty": "TradeCount",
72
78
  "traddCtrctsQty": "TradeVolume",
@@ -85,23 +91,69 @@ def _process_raw_df(raw_df: pd.DataFrame) -> pd.DataFrame:
85
91
  # Check which columns are present in the DataFrame before renaming
86
92
  rename_dict = {c: all_columns[c] for c in all_columns if c in df.columns}
87
93
  df = df.rename(columns=rename_dict)
94
+ return df
95
+
96
+
97
+ def _process_df(raw_df: pd.DataFrame) -> pd.DataFrame:
98
+ df = raw_df.copy()
88
99
 
89
- # df["BDaysToExp"] = bd.count_bdays(df["TradeTimestamp"], df["ExpirationDate"])
100
+ df["BDaysToExp"] = bday.count_bdays(df["TradeTime"], df["ExpirationDate"])
101
+
102
+ df["DaysToExp"] = (df["ExpirationDate"] - df["TradeTime"]).dt.days
103
+ # Convert to nullable integer, since it is the default type in the library
104
+ df["DaysToExp"] = df["DaysToExp"].astype(pd.Int64Dtype())
90
105
 
91
106
  # Remove percentage in all rate columns
92
107
  rate_cols = [col for col in df.columns if "Rate" in col]
93
- df[rate_cols] = df[rate_cols] / 100
108
+ df[rate_cols] = df[rate_cols].div(100).round(5)
109
+
110
+ return df
94
111
 
95
- # Reorder columns based on the order of the dictionary
96
- return df[rename_dict.values()]
112
+
113
+ def _select_and_reorder_columns(df: pd.DataFrame) -> pd.DataFrame:
114
+ """
115
+ Select and reorder columns in the DataFrame.
116
+
117
+ Args:
118
+ df (pd.DataFrame): A DataFrame containing futures data.
119
+
120
+ Returns:
121
+ pd.DataFrame: A DataFrame with the columns selected and reordered.
122
+ """
123
+ columns = [
124
+ "TradeTime",
125
+ "TickerSymbol",
126
+ "ExpirationDate",
127
+ "BDaysToExp",
128
+ "OpenContracts",
129
+ "TradeCount",
130
+ "TradeVolume",
131
+ "FinancialVolume",
132
+ "PrevSettlementRate",
133
+ "MinLimitRate",
134
+ "MaxLimitRate",
135
+ "OpenRate",
136
+ "MinRate",
137
+ "AvgRate",
138
+ "MaxRate",
139
+ "CurrentAskRate",
140
+ "CurrentBidRate",
141
+ "CurrentRate",
142
+ ]
143
+ return df[columns]
97
144
 
98
145
 
99
146
  def fetch_intraday_df(future_code: str) -> pd.DataFrame:
100
147
  """
101
- Fetch the latest DI futures data from B3.
148
+ Fetch the latest futures data from B3.
102
149
 
103
150
  Returns:
104
151
  pd.DataFrame: A Pandas pd.DataFrame containing the latest DI futures data.
105
152
  """
106
153
  raw_df = _fetch_raw_df(future_code)
107
- return _process_raw_df(raw_df)
154
+ if raw_df.empty:
155
+ return raw_df
156
+ df = _rename_columns(raw_df)
157
+ df = _process_df(df)
158
+ df = _select_and_reorder_columns(df)
159
+ return df
@@ -0,0 +1,261 @@
1
+ import io
2
+ import zipfile
3
+ from pathlib import Path
4
+ from typing import Literal
5
+
6
+ import pandas as pd
7
+ import requests
8
+ from lxml import etree
9
+ from pandas import DataFrame, Timestamp
10
+
11
+ import pyield as yd
12
+ from pyield.futures import historical as fh
13
+
14
+
15
+ def _get_file_from_path(file_path: Path) -> io.BytesIO:
16
+ # Check if a file path was not provided
17
+ if not isinstance(file_path, Path):
18
+ raise ValueError("A file path must be provided.")
19
+ if not file_path.exists():
20
+ raise FileNotFoundError(f"No file found at {file_path}.")
21
+
22
+ content = file_path.read_bytes()
23
+ return io.BytesIO(content)
24
+
25
+
26
+ def _get_file_from_url(trade_date: Timestamp, source_type: str) -> io.BytesIO:
27
+ """
28
+ Types of XML files available:
29
+ Full Price Report (all assets)
30
+ - aprox. 5 MB zipped file;
31
+ - url example: https://www.b3.com.br/pesquisapregao/download?filelist=PR231228.zip
32
+ Simplified Price Report (derivatives)
33
+ - aprox. 50kB zipped file;
34
+ url example: https://www.b3.com.br/pesquisapregao/download?filelist=SPRD240216.zip
35
+ """
36
+
37
+ trade_date_str = trade_date.strftime("%y%m%d")
38
+
39
+ if source_type == "PR":
40
+ url = f"https://www.b3.com.br/pesquisapregao/download?filelist=PR{trade_date_str}.zip"
41
+ elif source_type == "SPR":
42
+ url = f"https://www.b3.com.br/pesquisapregao/download?filelist=SPRD{trade_date_str}.zip"
43
+ else:
44
+ raise ValueError("Invalid source type. Must be either 'PR' or 'SPR'.")
45
+
46
+ response = requests.get(url)
47
+
48
+ # File will be considered invalid if it is too small
49
+ if response.status_code != 200 or len(response.content) < 1024:
50
+ trade_date_str = trade_date.strftime("%Y-%m-%d")
51
+ raise ValueError(f"There is no data available for {trade_date_str}.")
52
+
53
+ return io.BytesIO(response.content)
54
+
55
+
56
+ def _extract_xml_from_zip(zip_file: io.BytesIO) -> io.BytesIO:
57
+ # First, read the outer file
58
+ with zipfile.ZipFile(zip_file, "r") as outer_zip:
59
+ outer_file_name = outer_zip.namelist()[0]
60
+ outer_file_content = outer_zip.read(outer_file_name)
61
+ outer_file = io.BytesIO(outer_file_content)
62
+
63
+ # Then, read the inner file
64
+ with zipfile.ZipFile(outer_file, "r") as inner_zip:
65
+ filenames = inner_zip.namelist()
66
+ # Filter only xml files
67
+ xml_filenames = [name for name in filenames if name.endswith(".xml")]
68
+ xml_filenames.sort()
69
+ # Unzip last file (the most recent as per B3's name convention)
70
+ inner_file_content = inner_zip.read(xml_filenames[-1])
71
+
72
+ return io.BytesIO(inner_file_content)
73
+
74
+
75
+ def _extract_data_from_xml(xml_file: io.BytesIO, asset_code: str) -> list[dict]:
76
+ parser = etree.XMLParser(
77
+ ns_clean=True, remove_blank_text=True, remove_comments=True, recover=True
78
+ )
79
+ tree = etree.parse(xml_file, parser)
80
+
81
+ # XPath para encontrar elementos cujo texto começa com código do ativo: DI1, FRC...
82
+ namespaces = {"ns": "urn:bvmf.217.01.xsd"}
83
+ path = f'//ns:TckrSymb[starts-with(text(), "{asset_code}")]'
84
+ tickers = tree.xpath(path, namespaces=namespaces)
85
+
86
+ if tickers is None or not isinstance(tickers, list) or len(tickers) == 0:
87
+ return []
88
+
89
+ # Lista para armazenar os dados
90
+ di_data = []
91
+
92
+ # Processar cada TckrSymb encontrado
93
+ for ticker in tickers:
94
+ if isinstance(ticker, etree._Element):
95
+ price_report = ticker.getparent()
96
+ if price_report is not None:
97
+ price_report = price_report.getparent()
98
+ else:
99
+ # Handle the case where tckr_symb doesn't have a parent
100
+ continue
101
+ else:
102
+ # Handle the case where tckr_symb is not an _Element
103
+ continue
104
+
105
+ # Extrair a data de negociação
106
+ if price_report is None:
107
+ continue
108
+ trade_date = price_report.find(".//ns:TradDt/ns:Dt", namespaces)
109
+
110
+ # Preparar o dicionário de dados do ticker com a data de negociação
111
+ if trade_date is None:
112
+ continue
113
+ ticker_data = {"TradDt": trade_date.text, "TckrSymb": ticker.text}
114
+
115
+ # Acessar o elemento FinInstrmAttrbts que contém o TckrSymb
116
+ fin_instrm_attrbts = price_report.find(".//ns:FinInstrmAttrbts", namespaces)
117
+ # Verificar se FinInstrmAttrbts existe
118
+ if fin_instrm_attrbts is None:
119
+ continue # Pular para o próximo TckrSymb se FinInstrmAttrbts não existir
120
+ # Extrair os dados de FinInstrmAttrbts
121
+ for attr in fin_instrm_attrbts:
122
+ tag_name = etree.QName(attr).localname
123
+ ticker_data[tag_name] = attr.text
124
+
125
+ # Adicionar o dicionário à lista
126
+ di_data.append(ticker_data)
127
+
128
+ return di_data
129
+
130
+
131
+ def _create_df_from_data(di1_data: list) -> DataFrame:
132
+ # Criar um DataFrame com os dados coletados
133
+ df = pd.DataFrame(di1_data)
134
+
135
+ # Convert to CSV and then back to pandas to get automatic type conversion
136
+ file = io.StringIO(df.to_csv(index=False))
137
+ return pd.read_csv(file, dtype_backend="numpy_nullable")
138
+
139
+
140
+ def _rename_columns(df: DataFrame) -> DataFrame:
141
+ all_columns = {
142
+ "TradDt": "TradeDate",
143
+ "TckrSymb": "TickerSymbol",
144
+ # "MktDataStrmId"
145
+ # "IntlFinVol",
146
+ "OpnIntrst": "OpenContracts",
147
+ "FinInstrmQty": "TradeVolume",
148
+ "NtlFinVol": "FinancialVolume",
149
+ "AdjstdQt": "SettlementPrice",
150
+ "MinTradLmt": "MinLimitRate",
151
+ "MaxTradLmt": "MaxLimitRate",
152
+ # Must invert bid/ask for rates
153
+ "BestAskPric": "BestBidRate",
154
+ "BestBidPric": "BestAskRate",
155
+ "MinPric": "MinRate",
156
+ "TradAvrgPric": "AvgRate",
157
+ "MaxPric": "MaxRate",
158
+ "FrstPric": "OpenRate",
159
+ "LastPric": "CloseRate",
160
+ "AdjstdQtTax": "SettlementRate",
161
+ "RglrTxsQty": "TradeCount",
162
+ # "RglrTraddCtrcts"
163
+ # "NtlRglrVol"
164
+ # "IntlRglrVol",
165
+ # "AdjstdQtStin",
166
+ # "PrvsAdjstdQt",
167
+ # "PrvsAdjstdQtTax",
168
+ # "PrvsAdjstdQtStin",
169
+ # "OscnPctg",
170
+ # "VartnPts",
171
+ # "AdjstdValCtrct",
172
+ }
173
+ all_columns = {c: all_columns[c] for c in all_columns if c in df.columns}
174
+ return df.rename(columns=all_columns)
175
+
176
+
177
+ def _process_df(df_raw: DataFrame) -> DataFrame:
178
+ df = df_raw.copy()
179
+ # Convert to datetime64[ns] since it is pandas default type for timestamps
180
+ df["TradeDate"] = df["TradeDate"].astype("datetime64[ns]")
181
+
182
+ expiration_code = df["TickerSymbol"].str[3:]
183
+ df["ExpirationDate"] = expiration_code.apply(fh.get_expiration_date)
184
+
185
+ df["DaysToExp"] = (df["ExpirationDate"] - df["TradeDate"]).dt.days
186
+ # Convert to nullable integer, since it is the default type in the library
187
+ df["DaysToExp"] = df["DaysToExp"].astype(pd.Int64Dtype())
188
+ # Remove expired contracts
189
+ df.query("DaysToExp > 0", inplace=True)
190
+
191
+ df["BDaysToExp"] = yd.bday.count_bdays(df["TradeDate"], df["ExpirationDate"])
192
+
193
+ rate_cols = [col for col in df.columns if "Rate" in col]
194
+ # Remove % and round to 5 (3 in %) dec. places in rate columns
195
+ df[rate_cols] = df[rate_cols].div(100).round(5)
196
+
197
+ # Columns where NaN means 0
198
+ zero_cols = ["OpenContracts", "TradeCount", "TradeVolume", "FinancialVolume"]
199
+ for col in zero_cols:
200
+ if col in df.columns:
201
+ df[col] = df[col].fillna(0)
202
+
203
+ return df.sort_values(by=["ExpirationDate"], ignore_index=True)
204
+
205
+
206
+ def _select_and_reorder_columns(df: DataFrame) -> DataFrame:
207
+ # All SPRD columns are present in PR
208
+ all_columns = [
209
+ "TradeDate",
210
+ "TickerSymbol",
211
+ "ExpirationDate",
212
+ "BDaysToExp",
213
+ "DaysToExp",
214
+ "OpenContracts",
215
+ "TradeCount",
216
+ "TradeVolume",
217
+ "FinancialVolume",
218
+ "SettlementPrice",
219
+ "SettlementRate",
220
+ "MinLimitRate",
221
+ "MaxLimitRate",
222
+ "BestBidRate",
223
+ "BestAskRate",
224
+ "OpenRate",
225
+ "MinRate",
226
+ "AvgRate",
227
+ "MaxRate",
228
+ "CloseRate",
229
+ ]
230
+ selected_columns = [col for col in all_columns if col in df.columns]
231
+ return df[selected_columns]
232
+
233
+
234
+ def process_zip_file(zip_file: io.BytesIO, asset_code: str) -> DataFrame:
235
+ xml_file = _extract_xml_from_zip(zip_file)
236
+
237
+ di_data = _extract_data_from_xml(xml_file, asset_code)
238
+
239
+ df_raw = _create_df_from_data(di_data)
240
+
241
+ df = _rename_columns(df_raw)
242
+
243
+ df = _process_df(df)
244
+
245
+ df = _select_and_reorder_columns(df)
246
+
247
+ return df
248
+
249
+
250
+ def fetch_df(
251
+ trade_date: Timestamp, asset_code: str, source_type: Literal["PR", "SPR"]
252
+ ) -> DataFrame:
253
+ zip_file = _get_file_from_url(trade_date, source_type)
254
+ df = process_zip_file(zip_file, asset_code)
255
+ return df
256
+
257
+
258
+ def read_df(file_path: Path, asset_code: str) -> DataFrame:
259
+ zip_file = _get_file_from_path(file_path)
260
+ df = process_zip_file(zip_file, asset_code)
261
+ return df
@@ -32,7 +32,7 @@ dependencies = [
32
32
  "requests>=2.31.0",
33
33
  ]
34
34
  dynamic = []
35
- version = "0.7.9"
35
+ version = "0.7.10"
36
36
 
37
37
  [project.license]
38
38
  file = "LICENSE"
Binary file
Binary file
@@ -1 +0,0 @@
1
- __version__ = "0.7.9"
pyield-0.7.9/tests/xml.py DELETED
@@ -1,349 +0,0 @@
1
- import io
2
- import zipfile
3
- from pathlib import Path
4
-
5
- import pandas as pd
6
- import requests
7
- from lxml import etree
8
- from pandas import DataFrame, Timestamp
9
-
10
- from ..pyield import bday
11
- from ..pyield.futures import historical as ht
12
-
13
-
14
- def _get_file_from_url(trade_date: Timestamp, source_type: str) -> io.BytesIO:
15
- """
16
- Types of XML files available:
17
- Full Price Report (all assets)
18
- - aprox. 5 MB zipped file;
19
- - url example: https://www.b3.com.br/pesquisapregao/download?filelist=PR231228.zip
20
- Simplified Price Report (derivatives)
21
- - aprox. 50kB zipped file;
22
- url example: https://www.b3.com.br/pesquisapregao/download?filelist=SPRD240216.zip
23
- """
24
-
25
- formatted_date = trade_date.strftime("%y%m%d")
26
-
27
- if source_type == "b3":
28
- url = f"https://www.b3.com.br/pesquisapregao/download?filelist=PR{formatted_date}.zip"
29
- else: # source_type == "b3s"
30
- url = f"https://www.b3.com.br/pesquisapregao/download?filelist=SPRD{formatted_date}.zip"
31
-
32
- response = requests.get(url)
33
-
34
- # File will be considered invalid if it is too small
35
- if response.status_code != 200 or len(response.content) < 1024:
36
- formatted_date = trade_date.strftime("%Y-%m-%d")
37
- raise ValueError(f"There is no data available for {formatted_date}.")
38
-
39
- return io.BytesIO(response.content)
40
-
41
-
42
- def _extract_xml_from_zip(zip_file: io.BytesIO) -> io.BytesIO:
43
- # First, read the outer file
44
- with zipfile.ZipFile(zip_file, "r") as outer_zip:
45
- outer_file_name = outer_zip.namelist()[0]
46
- outer_file_content = outer_zip.read(outer_file_name)
47
- outer_file = io.BytesIO(outer_file_content)
48
-
49
- # Then, read the inner file
50
- with zipfile.ZipFile(outer_file, "r") as inner_zip:
51
- filenames = inner_zip.namelist()
52
- # Filter only xml files
53
- xml_filenames = [name for name in filenames if name.endswith(".xml")]
54
- xml_filenames.sort()
55
- # Unzip last file (the most recent as per B3's name convention)
56
- inner_file_content = inner_zip.read(xml_filenames[-1])
57
-
58
- return io.BytesIO(inner_file_content)
59
-
60
-
61
- def _extract_di_data_from_xml(xml_file: io.BytesIO) -> list[dict]:
62
- parser = etree.XMLParser(
63
- ns_clean=True, remove_blank_text=True, remove_comments=True, recover=True
64
- )
65
- tree = etree.parse(xml_file, parser)
66
- namespaces = {"ns": "urn:bvmf.217.01.xsd"}
67
-
68
- # XPath para encontrar elementos cujo texto começa com "DI1"
69
- tckr_symbols = tree.xpath(
70
- '//ns:TckrSymb[starts-with(text(), "DI1")]', namespaces=namespaces
71
- )
72
-
73
- if (
74
- tckr_symbols is None
75
- or not isinstance(tckr_symbols, list)
76
- or len(tckr_symbols) == 0
77
- ):
78
- return []
79
-
80
- # Lista para armazenar os dados com type hinting
81
- # di_data: list[dict] = []
82
- di_data = []
83
-
84
- # Processar cada TckrSymb encontrado
85
- for tckr_symb in tckr_symbols:
86
- if isinstance(tckr_symb, etree._Element):
87
- price_report = tckr_symb.getparent()
88
- if price_report is not None:
89
- price_report = price_report.getparent()
90
- else:
91
- # Handle the case where tckr_symb doesn't have a parent
92
- continue
93
- else:
94
- # Handle the case where tckr_symb is not an _Element
95
- continue
96
-
97
- # Extrair a data de negociação
98
- if price_report is None:
99
- continue
100
- trade_date = price_report.find(".//ns:TradeDate/ns:Dt", namespaces)
101
-
102
- # Preparar o dicionário de dados do ticker com a data de negociação
103
- if trade_date is None:
104
- continue
105
- ticker_data = {"TradeDate": trade_date.text, "TckrSymb": tckr_symb.text}
106
-
107
- # Acessar o elemento FinInstrmAttrbts que contém o TckrSymb
108
- fin_instrm_attrbts = price_report.find(".//ns:FinInstrmAttrbts", namespaces)
109
- # Verificar se FinInstrmAttrbts existe
110
- if fin_instrm_attrbts is None:
111
- continue # Pular para o próximo TckrSymb se FinInstrmAttrbts não existir
112
- # Extrair os dados de FinInstrmAttrbts
113
- for attr in fin_instrm_attrbts:
114
- tag_name = etree.QName(attr).localname
115
- ticker_data[tag_name] = attr.text
116
-
117
- # Adicionar o dicionário à lista
118
- di_data.append(ticker_data)
119
-
120
- return di_data
121
-
122
-
123
- def _create_df_from_di_data(di1_data: list) -> DataFrame:
124
- # Criar um DataFrame com os dados coletados
125
- df = pd.DataFrame(di1_data)
126
-
127
- # Convert to CSV and then back to pandas to get automatic type conversion
128
- file = io.StringIO(df.to_csv(index=False))
129
- return pd.read_csv(file, dtype_backend="numpy_nullable")
130
-
131
-
132
- def _filter_and_order_pr_df(df: DataFrame) -> DataFrame:
133
- selected_columns = [
134
- "TradDt",
135
- "TckrSymb",
136
- # "MktDataStrmId",
137
- "OpnIntrst",
138
- "FinInstrmQty",
139
- "NtlFinVol",
140
- # "IntlFinVol",
141
- "AdjstdQt",
142
- "MinTradLmt",
143
- "MaxTradLmt",
144
- "BestBidPric",
145
- "BestAskPric",
146
- "MinPric",
147
- "TradAvrgPric",
148
- "MaxPric",
149
- "FrstPric",
150
- "LastPric",
151
- "AdjstdQtTax",
152
- # "RglrTxsQty",
153
- # "RglrTraddCtrcts",
154
- # "NtlRglrVol",
155
- # "IntlRglrVol",
156
- # "AdjstdQtStin",
157
- # "PrvsAdjstdQt",
158
- # "PrvsAdjstdQtTax",
159
- # "PrvsAdjstdQtStin",
160
- # "OscnPctg",
161
- # "VartnPts",
162
- # "AdjstdValCtrct",
163
- ]
164
-
165
- return df[selected_columns]
166
-
167
-
168
- def _filter_and_order_sprd_df(df: DataFrame) -> DataFrame:
169
- cols = [
170
- "TradDt",
171
- "TckrSymb",
172
- "OpnIntrst",
173
- "AdjstdQt",
174
- "MinPric",
175
- "TradAvrgPric",
176
- "MaxPric",
177
- "FrstPric",
178
- "LastPric",
179
- "AdjstdQtTax",
180
- # "RglrTxsQty",
181
- # "AdjstdQtStin", # Constant column
182
- # "PrvsAdjstdQt",
183
- # "PrvsAdjstdQtTax",
184
- # "PrvsAdjstdQtStin", # Constant column
185
- ]
186
-
187
- return df[cols]
188
-
189
-
190
- def _standardize_column_names(df: DataFrame) -> DataFrame:
191
- rename_dict = {
192
- "TradDt": "TradeDate",
193
- "TckrSymb": "Ticker",
194
- # "MktDataStrmId"
195
- # "IntlFinVol",
196
- "OpnIntrst": "OpenContracts",
197
- "FinInstrmQty": "TradeVolume",
198
- "NtlFinVol": "FinancialVolume",
199
- "AdjstdQt": "SettlementPrice",
200
- "MinTradLmt": "MinTradeLimitRate",
201
- "MaxTradLmt": "MaxTradeLimitRate",
202
- # Must invert bid/ask for rates
203
- "BestAskPric": "BestBidRate",
204
- "BestBidPric": "BestAskRate",
205
- "MinPric": "MinRate",
206
- "TradAvrgPric": "AvgRate",
207
- "MaxPric": "MaxRate",
208
- "FrstPric": "FirstRate",
209
- "LastPric": "CloseRate",
210
- "AdjstdQtTax": "SettlementRate",
211
- # "RglrTxsQty"
212
- # "RglrTraddCtrcts"
213
- # "NtlRglrVol"
214
- # "IntlRglrVol",
215
- # "AdjstdQtStin",
216
- # "PrvsAdjstdQt",
217
- # "PrvsAdjstdQtTax",
218
- # "PrvsAdjstdQtStin",
219
- # "OscnPctg",
220
- # "VartnPts",
221
- # "AdjstdValCtrct",
222
- }
223
-
224
- return df.rename(columns=rename_dict)
225
-
226
-
227
- def _process_di_df(df_raw: DataFrame) -> DataFrame:
228
- df = df_raw.copy()
229
- # Convert to datetime64[ns] since it is pandas default type for timestamps
230
- df["TradDt"] = df["TradDt"].astype("datetime64[ns]")
231
-
232
- expiration = df["TckrSymb"].str[3:].apply(ht.get_expiration_date)
233
- df.insert(2, "ExpirationDate", expiration)
234
-
235
- business_days = bday.count_bdays(df["TradDt"], df["ExpirationDate"])
236
- df.insert(3, "BDaysToExp", business_days)
237
-
238
- # Convert to nullable integer, since other columns use this data type
239
- df["BDaysToExp"] = df["BDaysToExp"].astype(pd.Int64Dtype())
240
-
241
- # Remove expired contracts
242
- df.query("BDaysToExp > 0", inplace=True)
243
-
244
- return df.sort_values(by=["ExpirationDate"], ignore_index=True)
245
-
246
-
247
- def read_xml(trade_date: Timestamp, source_type: str, return_raw: bool) -> DataFrame:
248
- zip_file = _get_file_from_url(trade_date, source_type)
249
-
250
- xml_file = _extract_xml_from_zip(zip_file)
251
-
252
- di_data = _extract_di_data_from_xml(xml_file)
253
-
254
- df_raw = _create_df_from_di_data(di_data)
255
- if return_raw:
256
- return df_raw
257
-
258
- # Remove unnecessary columns
259
- if source_type == "b3":
260
- df_di = _filter_and_order_pr_df(df_raw)
261
- elif source_type == "b3s":
262
- df_di = _filter_and_order_sprd_df(df_raw)
263
- else:
264
- raise ValueError("Invalid source type. Must be 'b3' or 'b3s'.")
265
-
266
- # Process and transform data
267
- df_di = _process_di_df(df_di)
268
-
269
- # Standardize column names
270
- df_di = _standardize_column_names(df_di)
271
-
272
- return df_di
273
-
274
-
275
- def read_di(file_path: Path, return_raw: bool = False) -> DataFrame:
276
- if file_path:
277
- if file_path.exists():
278
- content = file_path.read_bytes()
279
- zip_file = io.BytesIO(content)
280
- else:
281
- raise FileNotFoundError(f"No file found at {file_path}.")
282
-
283
- xml_file = _extract_xml_from_zip(zip_file)
284
-
285
- di_data = _extract_di_data_from_xml(xml_file)
286
-
287
- df_raw = _create_df_from_di_data(di_data)
288
- if return_raw:
289
- return df_raw
290
-
291
- # Filename examples: PR231228.zip or SPRD240216.zip
292
- file_stem = file_path.stem
293
- if "PR" in file_stem:
294
- df_di = _filter_and_order_pr_df(df_raw)
295
- elif "SPRD" in file_stem:
296
- df_di = _filter_and_order_sprd_df(df_raw)
297
- else:
298
- raise ValueError("Filename must start with 'PR' or 'SPRD'.")
299
-
300
- # Process and transform data
301
- df_di = _process_di_df(df_di)
302
-
303
- # Standardize column names
304
- df_di = _standardize_column_names(df_di)
305
-
306
- return df_di
307
-
308
- else:
309
- raise ValueError("A file path must be provided.")
310
-
311
-
312
- def read_file(file_path: Path, return_raw: bool = False) -> pd.DataFrame:
313
- """
314
- Reads DI futures data from a file and returns it as a pandas DataFrame.
315
-
316
- This function opens and reads a DI futures data file, returning the contents as a
317
- pandas DataFrame. It supports reading from both XML files provided by B3, wich
318
- are the simplified and complete Price Reports.
319
-
320
- Args:
321
- file_path (Path): The file path to the DI data file. This should be a valid
322
- Path object pointing to the location of the file.
323
- return_raw (bool, optional): If set to True, the function returns the raw data
324
- without applying any transformation or processing. Useful for cases where
325
- raw data inspection or custom processing is needed. Defaults to False.
326
- source_type (Literal["bmf", "b3", "b3s"], optional): Indicates the source of
327
- the data. Defaults to "bmf". Options include:
328
- - "bmf": Fetches data from the old BM&FBOVESPA website. Fastest option.
329
- - "b3": Fetches data from the complete Price Report (XML file) provided
330
- by B3.
331
- - "b3s": Fetches data from the simplified Price Report (XML file)
332
- provided by B3. Faster than "b3" but less detailed.
333
-
334
- Returns:
335
- pd.DataFrame: A DataFrame containing the processed or raw DI futures data,
336
- depending on the `return_raw` flag.
337
-
338
- Examples:
339
- >>> read_di(Path("path/to/di_data_file.xml"))
340
- # returns a DataFrame with the DI futures data
341
-
342
- >>> read_di(Path("path/to/di_data_file.xml"), return_raw=True)
343
- # returns a DataFrame with the raw DI futures data, without processing
344
-
345
- Note:
346
- The ability to process and return raw data is primarily intended for advanced
347
- users who require access to the data in its original form for custom analyses.
348
- """
349
- return read_di(file_path, return_raw=return_raw)
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes