cryptodatapy 0.2.5__py3-none-any.whl → 0.2.6__py3-none-any.whl

cryptodatapy/transform/filter.py

@@ -170,28 +170,31 @@ class Filter:
 
         return self.filtered_df
 
-    def remove_delisted(self, field: str = 'close', n_unch_vals: int = 30) -> pd.DataFrame:
+    def delisted_tickers(self, method: str = 'replace') -> pd.DataFrame:
         """
-        Removes delisted tickers from dataframe.
+        Repairs delisted tickers by either removing them or replacing them with NaNs.
 
         Parameters
         ----------
-        field: str, default 'close'
-            Field/column to use for detecting delisted tickers.
-        n_unch_vals: int, default 30
-            Number of consecutive unchanged values to consider a ticker as delisted.
+        method: str, {'replace', 'remove'}, default 'replace'
+            Method to repair delisted tickers. Can be 'remove' or 'replace'.
 
         Returns
         -------
         filtered_df: pd.DataFrame - MultiIndex
             Filtered dataFrame with DatetimeIndex (level 0), tickers (level 1) and fields (cols).
         """
+        # unchanged rows
+        unch_rows = (self.df.subtract(self.df.iloc[:, :4].mean(axis=1), axis=0) == 0).any(axis=1)
+
         # delisted tickers
-        delisted_tickers = self.df[field].unstack()[self.df[field].unstack().pct_change().iloc[-n_unch_vals:] == 0].\
-            dropna(how='all', axis=0).dropna(thresh=n_unch_vals, axis=1).columns
+        delisted_tickers = unch_rows.unstack().iloc[-1][unch_rows.unstack().iloc[-1]].index.to_list()
 
-        # drop delisted tickers
-        self.filtered_df = self.df.drop(delisted_tickers, level=1)
+        # repair
+        if method == 'remove':
+            self.filtered_df = self.df.drop(delisted_tickers, level=1)
+        else:
+            self.filtered_df = self.df.loc[~unch_rows].reindex(self.df.index)
 
         return self.filtered_df
 

cryptodatapy/transform/od.py

@@ -1,6 +1,7 @@
 from typing import Dict, Optional, Union
 
 import numpy as np
+np.float_ = np.float64
 import pandas as pd
 from prophet import Prophet
 from statsmodels.tsa.seasonal import STL, seasonal_decompose

cryptodatapy/transform/rates_data.ipynb

@@ -0,0 +1,465 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 461,
+   "id": "9fea9fae",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The history saving thread hit an unexpected error (OperationalError('attempt to write a readonly database')).History will not be written to the database.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "\n",
+    "from cryptodatapy.extract.datarequest import DataRequest\n",
+    "from cryptodatapy.util.datacatalog import DataCatalog \n",
+    "from cryptodatapy.extract.getdata import GetData\n",
+    "from cryptodatapy.transform.od import OutlierDetection\n",
+    "from cryptodatapy.transform.impute import Impute\n",
+    "from cryptodatapy.transform.filter import Filter\n",
+    "from cryptodatapy.transform.clean import CleanData, stitch_dataframes\n",
+    "from cryptodatapy.transform.impute import Impute"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "fc533553-649e-4479-8a4b-b3db5cb77d76",
+   "metadata": {},
+   "source": [
+    "# Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "2ad72bc7-5fdd-4ae5-8d9e-e90118efcc26",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dc = DataCatalog()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "e7fe6fda-f0cb-420a-a892-b7f73eba48a8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "us_rates_tickers = dc.get_tickers_metadata(cat='rates', subcat='yield', mkt_type='spot').loc[:, 'fred_id'].dropna().index.tolist()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 450,
+   "id": "d92d80db-bf8c-4e8a-87c5-aaebb298bdb5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_req = DataRequest(source='yahoo', source_tickers=['^MOVE'], cat='rates')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 452,
+   "id": "c36cda5a-6406-49b5-84f2-ce83932f423e",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[*********************100%***********************]  1 of 1 completed\n"
+     ]
+    }
+   ],
+   "source": [
+    "move_df = GetData(data_req).get_series()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 456,
+   "id": "c2080432-892e-44e8-9671-45f0ddebb28b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "move_df = move_df.rename(index={'^MOVE': 'US_Rates_Vol'})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "cf537011-9837-46e9-8715-b7cd4c21e226",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data_req = DataRequest(source='fred', tickers=us_rates_tickers, cat='rates')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "c5e4ca96-302b-4ba7-bd4b-010d891d1804",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "us_rates_df = GetData(data_req).get_series()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 457,
+   "id": "f3a6829d-ea74-4495-b792-6f5af054d604",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['US_Rates_10Y', 'US_Rates_1M', 'US_Rates_1Y', 'US_Rates_20Y',\n",
+       "       'US_Rates_2Y', 'US_Rates_30Y', 'US_Rates_3M', 'US_Rates_3Y',\n",
+       "       'US_Rates_5Y', 'US_Rates_6M', 'US_Rates_7Y'],\n",
+       "      dtype='object', name='ticker')"
+      ]
+     },
+     "execution_count": 457,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "us_rates_df.unstack().close.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 458,
+   "id": "f6e4298d-a46c-4efd-b888-8ebe947607f9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "us_rates_df = pd.concat([us_rates_df, move_df]).sort_index()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "id": "ee827d6c-31b4-40a9-927f-2e27f3d9db1e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# us_rates_df.to_csv('us_rates_daily.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 460,
+   "id": "885955b2-6e25-44d2-8f16-d60b79c00c5b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "us_rates_df.to_parquet('s3://factorlab-data/us_yields_daily.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 428,
+   "id": "ede2c7be-9061-4a9f-866e-6070472e195d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "er_df = pd.read_csv('../../../../rose_excess_returns_daily.csv', index_col=['date'], parse_dates=['date'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 429,
+   "id": "d133fd38-0202-41f5-b034-885ba4e4fa12",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prices_df = pd.read_csv('../../../../rose_asset_prices_daily.csv', index_col=['date'], parse_dates=['date'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 430,
+   "id": "1357a5c6-85ca-4ba0-8952-dd0334f5cf27",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "iso3_to_iso2 = {\n",
+    "     'arg': 'AR',\n",
+    "     'aus': 'AU',\n",
+    "     'aut': 'AT',\n",
+    "     'bel': 'BE',\n",
+    "     'bra': 'BR',\n",
+    "     'can': 'CA',\n",
+    "     'che': 'CH',\n",
+    "     'chl': 'CL',\n",
+    "     'chn': 'CN',\n",
+    "     'col': 'CO',\n",
+    "     'cze': 'CZ',\n",
+    "     'deu': 'DE',\n",
+    "     'dnk': 'DK', \n",
+    "     'esp': 'ES',\n",
+    "     'eur': 'EU',\n",
+    "     'fin': 'FI',\n",
+    "     'fra': 'FR',\n",
+    "     'gbr': 'GB',\n",
+    "     'grc': 'GR',\n",
+    "     'hkg': 'HK',\n",
+    "     'hun': 'HU',\n",
+    "     'idn': 'ID',\n",
+    "     'ind': 'IN',\n",
+    "     'irl': 'IE',\n",
+    "     'ita': 'IT',\n",
+    "     'isl': 'IS',\n",
+    "     'isr': 'IL',\n",
+    "     'jpn': 'JP',\n",
+    "     'mex': 'MX',\n",
+    "     'mys': 'MY',\n",
+    "     'nld': 'NL',\n",
+    "     'nor': 'NO',\n",
+    "     'nzl': 'NZ',\n",
+    "     'per': 'PE',\n",
+    "     'phl': 'PH',\n",
+    "     'pol': 'PL',\n",
+    "     'prt': 'PT',\n",
+    "     'rus': 'RU',\n",
+    "     'sgp': 'SG',\n",
+    "     'swe': 'SE',\n",
+    "     'kor': 'KR',\n",
+    "     'swe': 'SE',\n",
+    "     'twn': 'TW',\n",
+    "     'tha': 'TH',\n",
+    "     'tur': 'TR',\n",
+    "     'usa': 'US',\n",
+    "     'zaf': 'ZA'\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 431,
+   "id": "f21d322f-1b3b-4c62-9599-3bdf15b8a1af",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "asset_mapping = {\n",
+    "    'lr': 'Rates_Long', \n",
+    "    'sr': 'Rates_Short', \n",
+    "    'eq': 'Eqty',\n",
+    "    'fx': 'FX'\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 432,
+   "id": "05a96e73-c6f7-4599-bcac-090700f3e3eb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def map_columns(col_name):\n",
+    "    country_code, asset_code = col_name.split('.')\n",
+    "    new_country = iso3_to_iso2.get(country_code, country_code)  # Handle missing mappings gracefully\n",
+    "    new_asset = asset_mapping.get(asset_code, asset_code)  # Handle missing mappings gracefully\n",
+    "    \n",
+    "    if new_asset:  # Only add underscore if the asset name is not empty\n",
+    "        return f'{new_country}_{new_asset}'\n",
+    "    else:\n",
+    "        return new_country"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 433,
+   "id": "cde06474-83d5-4d6d-a321-d0293141a557",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def map_columns_ret(col_name):\n",
+    "    country_code, asset_code, ret = col_name.split('.')\n",
+    "    new_country = iso3_to_iso2.get(country_code, country_code)  # Handle missing mappings gracefully\n",
+    "    new_asset = asset_mapping.get(asset_code, asset_code)  # Handle missing mappings gracefully\n",
+    "    \n",
+    "    if new_asset:  # Only add underscore if the asset name is not empty\n",
+    "        return f'{new_country}_{new_asset}'\n",
+    "    else:\n",
+    "        return new_country"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 434,
+   "id": "618933c5-252e-4c9e-a020-bcb1207b00be",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# asset dfs\n",
+    "eqty_df = prices_df.loc[:, prices_df.columns[:41]]\n",
+    "eqty_er_df = er_df.loc[:, er_df.columns[:41]]\n",
+    "rates_df = prices_df.loc[:, prices_df.columns[41:123]]\n",
+    "rates_er_df = er_df.loc[:, er_df.columns[42:83]]\n",
+    "fx_df = prices_df.loc[:, prices_df.columns[123:164]]\n",
+    "fx_er_df = er_df.loc[:, er_df.columns[83:124]]\n",
+    "cmdty_df = prices_df.loc[:, prices_df.columns[164:]]\n",
+    "cmdty_er_df = er_df.loc[:, er_df.columns[124:]]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 435,
+   "id": "a2e94a95-8e9b-4949-b3ed-d20db0cd093b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# convert price cols\n",
+    "eqty_df.columns = [map_columns(col) for col in eqty_df.columns]\n",
+    "rates_df.columns = [map_columns(col) for col in rates_df.columns]\n",
+    "fx_df.columns = [map_columns(col) for col in fx_df.columns]\n",
+    "cmdty_df.columns = [col.title() for col in cmdty_df.columns]\n",
+    "cmdty_df.columns = ['Gold', 'Silver', 'Platinum', 'Palladium', 'Oil_WTI', 'Oil_Brent', 'Natgas',\n",
+    "       'Rbob', 'Corn', 'Wheat', 'Soybeans', 'Coffee', 'Sugar', 'Cotton',\n",
+    "       'Orange_Juice', 'Cocoa', 'Pork', 'Beef', 'Aluminum', 'Copper', 'Lead',\n",
+    "       'Nickel', 'Tin', 'Zinc']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 436,
+   "id": "9054c525-7fd2-4ca9-86a3-e2dd01c3ac22",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# convert ret cols\n",
+    "eqty_er_df.columns = [map_columns_ret(col) for col in eqty_er_df.columns]\n",
+    "rates_er_df.columns = [map_columns_ret(col) for col in rates_er_df.columns]\n",
+    "fx_er_df.columns = [map_columns_ret(col) for col in fx_er_df.columns]\n",
+    "cmdty_er_df.columns = [col.title().split('.')[0] for col in cmdty_er_df.columns]\n",
+    "cmdty_er_df.columns = ['Gold', 'Silver', 'Platinum', 'Palladium', 'Oil_WTI', 'Oil_Brent', 'Natgas',\n",
+    "       'Rbob', 'Corn', 'Wheat', 'Soybeans', 'Coffee', 'Sugar', 'Cotton',\n",
+    "       'Orange_Juice', 'Cocoa', 'Pork', 'Beef', 'Aluminum', 'Copper', 'Lead',\n",
+    "       'Nickel', 'Tin', 'Zinc']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 437,
+   "id": "2dd75dbd-b287-4aca-af65-3f770d7dd20b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# stack dfs\n",
+    "eqty_df = eqty_df.stack().to_frame('close')\n",
+    "eqty_er_df = eqty_er_df.stack().to_frame('er')\n",
+    "rates_df = rates_df.stack().to_frame('close')\n",
+    "rates_er_df = rates_er_df.stack().to_frame('er')\n",
+    "fx_df = fx_df.stack().to_frame('close')\n",
+    "fx_er_df = fx_er_df.stack().to_frame('er')\n",
+    "cmdty_df = cmdty_df.stack().to_frame('close')\n",
+    "cmdty_er_df = cmdty_er_df.stack().to_frame('er')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 438,
+   "id": "5fb80d6e-f98d-4143-9449-9c6c2ea68369",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# set indexes\n",
+    "eqty_df.index.names = ['date', 'ticker']\n",
+    "eqty_er_df.index.names = ['date', 'ticker']\n",
+    "rates_df.index.names = ['date', 'ticker']\n",
+    "rates_er_df.index.names = ['date', 'ticker']\n",
+    "fx_df.index.names = ['date', 'ticker']\n",
+    "fx_er_df.index.names = ['date', 'ticker']\n",
+    "cmdty_df.index.names = ['date', 'ticker']\n",
+    "cmdty_er_df.index.names = ['date', 'ticker']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 439,
+   "id": "cdfceea4-af45-4bd9-be59-1de835196138",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eqty_df = pd.concat([eqty_df, eqty_er_df], axis=1).sort_index()\n",
+    "rates_df = pd.concat([rates_df, rates_er_df], axis=1).sort_index()\n",
+    "fx_df = pd.concat([fx_df, fx_er_df], axis=1).sort_index()\n",
+    "cmdty_df = pd.concat([cmdty_df, cmdty_er_df], axis=1).sort_index()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 441,
+   "id": "15a544cf-233b-4ea0-9e4a-9d6dc3e63d3b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eqty_df.to_parquet('s3://factorlab-data/global_eqty_data_daily.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 442,
+   "id": "a2b5599b-afff-4241-8c18-06486b1968fa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "rates_df.to_parquet('s3://factorlab-data/global_rates_data_daily.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 443,
+   "id": "87b4f132-6f87-4770-9892-97e069d7f3c0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fx_df.to_parquet('s3://factorlab-data/global_fx_data_daily.parquet')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 444,
+   "id": "5f21be89-a92d-4458-bca6-9a79f79b9d65",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cmdty_df.to_parquet('s3://factorlab-data/commodity_data_daily.parquet')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "cryptodatapy",
+   "language": "python",
+   "name": "cryptodatapy"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}