cryptodatapy 0.2.8__py3-none-any.whl → 0.2.9__py3-none-any.whl

cryptodatapy/transform/rates_data.ipynb

@@ -1,465 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 461,
-   "id": "9fea9fae",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The history saving thread hit an unexpected error (OperationalError('attempt to write a readonly database')).History will not be written to the database.\n"
-     ]
-    }
-   ],
-   "source": [
-    "import pandas as pd\n",
-    "import numpy as np\n",
-    "\n",
-    "from cryptodatapy.extract.datarequest import DataRequest\n",
-    "from cryptodatapy.util.datacatalog import DataCatalog \n",
-    "from cryptodatapy.extract.getdata import GetData\n",
-    "from cryptodatapy.transform.od import OutlierDetection\n",
-    "from cryptodatapy.transform.impute import Impute\n",
-    "from cryptodatapy.transform.filter import Filter\n",
-    "from cryptodatapy.transform.clean import CleanData, stitch_dataframes\n",
-    "from cryptodatapy.transform.impute import Impute"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "fc533553-649e-4479-8a4b-b3db5cb77d76",
-   "metadata": {},
-   "source": [
-    "# Data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 2,
-   "id": "2ad72bc7-5fdd-4ae5-8d9e-e90118efcc26",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dc = DataCatalog()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 24,
-   "id": "e7fe6fda-f0cb-420a-a892-b7f73eba48a8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "us_rates_tickers = dc.get_tickers_metadata(cat='rates', subcat='yield', mkt_type='spot').loc[:, 'fred_id'].dropna().index.tolist()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 450,
-   "id": "d92d80db-bf8c-4e8a-87c5-aaebb298bdb5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "data_req = DataRequest(source='yahoo', source_tickers=['^MOVE'], cat='rates')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 452,
-   "id": "c36cda5a-6406-49b5-84f2-ce83932f423e",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "[*********************100%***********************]  1 of 1 completed\n"
-     ]
-    }
-   ],
-   "source": [
-    "move_df = GetData(data_req).get_series()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 456,
-   "id": "c2080432-892e-44e8-9671-45f0ddebb28b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "move_df = move_df.rename(index={'^MOVE': 'US_Rates_Vol'})"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 27,
-   "id": "cf537011-9837-46e9-8715-b7cd4c21e226",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "data_req = DataRequest(source='fred', tickers=us_rates_tickers, cat='rates')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "id": "c5e4ca96-302b-4ba7-bd4b-010d891d1804",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "us_rates_df = GetData(data_req).get_series()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 457,
-   "id": "f3a6829d-ea74-4495-b792-6f5af054d604",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "Index(['US_Rates_10Y', 'US_Rates_1M', 'US_Rates_1Y', 'US_Rates_20Y',\n",
-       "       'US_Rates_2Y', 'US_Rates_30Y', 'US_Rates_3M', 'US_Rates_3Y',\n",
-       "       'US_Rates_5Y', 'US_Rates_6M', 'US_Rates_7Y'],\n",
-       "      dtype='object', name='ticker')"
-      ]
-     },
-     "execution_count": 457,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "us_rates_df.unstack().close.columns"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 458,
-   "id": "f6e4298d-a46c-4efd-b888-8ebe947607f9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "us_rates_df = pd.concat([us_rates_df, move_df]).sort_index()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 31,
-   "id": "ee827d6c-31b4-40a9-927f-2e27f3d9db1e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# us_rates_df.to_csv('us_rates_daily.csv')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 460,
-   "id": "885955b2-6e25-44d2-8f16-d60b79c00c5b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "us_rates_df.to_parquet('s3://factorlab-data/us_yields_daily.parquet')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 428,
-   "id": "ede2c7be-9061-4a9f-866e-6070472e195d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "er_df = pd.read_csv('../../../../rose_excess_returns_daily.csv', index_col=['date'], parse_dates=['date'])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 429,
-   "id": "d133fd38-0202-41f5-b034-885ba4e4fa12",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "prices_df = pd.read_csv('../../../../rose_asset_prices_daily.csv', index_col=['date'], parse_dates=['date'])"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 430,
-   "id": "1357a5c6-85ca-4ba0-8952-dd0334f5cf27",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "iso3_to_iso2 = {\n",
-    "     'arg': 'AR',\n",
-    "     'aus': 'AU',\n",
-    "     'aut': 'AT',\n",
-    "     'bel': 'BE',\n",
-    "     'bra': 'BR',\n",
-    "     'can': 'CA',\n",
-    "     'che': 'CH',\n",
-    "     'chl': 'CL',\n",
-    "     'chn': 'CN',\n",
-    "     'col': 'CO',\n",
-    "     'cze': 'CZ',\n",
-    "     'deu': 'DE',\n",
-    "     'dnk': 'DK', \n",
-    "     'esp': 'ES',\n",
-    "     'eur': 'EU',\n",
-    "     'fin': 'FI',\n",
-    "     'fra': 'FR',\n",
-    "     'gbr': 'GB',\n",
-    "     'grc': 'GR',\n",
-    "     'hkg': 'HK',\n",
-    "     'hun': 'HU',\n",
-    "     'idn': 'ID',\n",
-    "     'ind': 'IN',\n",
-    "     'irl': 'IE',\n",
-    "     'ita': 'IT',\n",
-    "     'isl': 'IS',\n",
-    "     'isr': 'IL',\n",
-    "     'jpn': 'JP',\n",
-    "     'mex': 'MX',\n",
-    "     'mys': 'MY',\n",
-    "     'nld': 'NL',\n",
-    "     'nor': 'NO',\n",
-    "     'nzl': 'NZ',\n",
-    "     'per': 'PE',\n",
-    "     'phl': 'PH',\n",
-    "     'pol': 'PL',\n",
-    "     'prt': 'PT',\n",
-    "     'rus': 'RU',\n",
-    "     'sgp': 'SG',\n",
-    "     'swe': 'SE',\n",
-    "     'kor': 'KR',\n",
-    "     'swe': 'SE',\n",
-    "     'twn': 'TW',\n",
-    "     'tha': 'TH',\n",
-    "     'tur': 'TR',\n",
-    "     'usa': 'US',\n",
-    "     'zaf': 'ZA'\n",
-    "}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 431,
-   "id": "f21d322f-1b3b-4c62-9599-3bdf15b8a1af",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "asset_mapping = {\n",
-    "    'lr': 'Rates_Long', \n",
-    "    'sr': 'Rates_Short', \n",
-    "    'eq': 'Eqty',\n",
-    "    'fx': 'FX'\n",
-    "}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 432,
-   "id": "05a96e73-c6f7-4599-bcac-090700f3e3eb",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def map_columns(col_name):\n",
-    "    country_code, asset_code = col_name.split('.')\n",
-    "    new_country = iso3_to_iso2.get(country_code, country_code)  # Handle missing mappings gracefully\n",
-    "    new_asset = asset_mapping.get(asset_code, asset_code)  # Handle missing mappings gracefully\n",
-    "    \n",
-    "    if new_asset:  # Only add underscore if the asset name is not empty\n",
-    "        return f'{new_country}_{new_asset}'\n",
-    "    else:\n",
-    "        return new_country"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 433,
-   "id": "cde06474-83d5-4d6d-a321-d0293141a557",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def map_columns_ret(col_name):\n",
-    "    country_code, asset_code, ret = col_name.split('.')\n",
-    "    new_country = iso3_to_iso2.get(country_code, country_code)  # Handle missing mappings gracefully\n",
-    "    new_asset = asset_mapping.get(asset_code, asset_code)  # Handle missing mappings gracefully\n",
-    "    \n",
-    "    if new_asset:  # Only add underscore if the asset name is not empty\n",
-    "        return f'{new_country}_{new_asset}'\n",
-    "    else:\n",
-    "        return new_country"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 434,
-   "id": "618933c5-252e-4c9e-a020-bcb1207b00be",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# asset dfs\n",
-    "eqty_df = prices_df.loc[:, prices_df.columns[:41]]\n",
-    "eqty_er_df = er_df.loc[:, er_df.columns[:41]]\n",
-    "rates_df = prices_df.loc[:, prices_df.columns[41:123]]\n",
-    "rates_er_df = er_df.loc[:, er_df.columns[42:83]]\n",
-    "fx_df = prices_df.loc[:, prices_df.columns[123:164]]\n",
-    "fx_er_df = er_df.loc[:, er_df.columns[83:124]]\n",
-    "cmdty_df = prices_df.loc[:, prices_df.columns[164:]]\n",
-    "cmdty_er_df = er_df.loc[:, er_df.columns[124:]]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 435,
-   "id": "a2e94a95-8e9b-4949-b3ed-d20db0cd093b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# convert price cols\n",
-    "eqty_df.columns = [map_columns(col) for col in eqty_df.columns]\n",
-    "rates_df.columns = [map_columns(col) for col in rates_df.columns]\n",
-    "fx_df.columns = [map_columns(col) for col in fx_df.columns]\n",
-    "cmdty_df.columns = [col.title() for col in cmdty_df.columns]\n",
-    "cmdty_df.columns = ['Gold', 'Silver', 'Platinum', 'Palladium', 'Oil_WTI', 'Oil_Brent', 'Natgas',\n",
-    "       'Rbob', 'Corn', 'Wheat', 'Soybeans', 'Coffee', 'Sugar', 'Cotton',\n",
-    "       'Orange_Juice', 'Cocoa', 'Pork', 'Beef', 'Aluminum', 'Copper', 'Lead',\n",
-    "       'Nickel', 'Tin', 'Zinc']"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 436,
-   "id": "9054c525-7fd2-4ca9-86a3-e2dd01c3ac22",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# convert ret cols\n",
-    "eqty_er_df.columns = [map_columns_ret(col) for col in eqty_er_df.columns]\n",
-    "rates_er_df.columns = [map_columns_ret(col) for col in rates_er_df.columns]\n",
-    "fx_er_df.columns = [map_columns_ret(col) for col in fx_er_df.columns]\n",
-    "cmdty_er_df.columns = [col.title().split('.')[0] for col in cmdty_er_df.columns]\n",
-    "cmdty_er_df.columns = ['Gold', 'Silver', 'Platinum', 'Palladium', 'Oil_WTI', 'Oil_Brent', 'Natgas',\n",
-    "       'Rbob', 'Corn', 'Wheat', 'Soybeans', 'Coffee', 'Sugar', 'Cotton',\n",
-    "       'Orange_Juice', 'Cocoa', 'Pork', 'Beef', 'Aluminum', 'Copper', 'Lead',\n",
-    "       'Nickel', 'Tin', 'Zinc']"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 437,
-   "id": "2dd75dbd-b287-4aca-af65-3f770d7dd20b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# stack dfs\n",
-    "eqty_df = eqty_df.stack().to_frame('close')\n",
-    "eqty_er_df = eqty_er_df.stack().to_frame('er')\n",
-    "rates_df = rates_df.stack().to_frame('close')\n",
-    "rates_er_df = rates_er_df.stack().to_frame('er')\n",
-    "fx_df = fx_df.stack().to_frame('close')\n",
-    "fx_er_df = fx_er_df.stack().to_frame('er')\n",
-    "cmdty_df = cmdty_df.stack().to_frame('close')\n",
-    "cmdty_er_df = cmdty_er_df.stack().to_frame('er')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 438,
-   "id": "5fb80d6e-f98d-4143-9449-9c6c2ea68369",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# set indexes\n",
-    "eqty_df.index.names = ['date', 'ticker']\n",
-    "eqty_er_df.index.names = ['date', 'ticker']\n",
-    "rates_df.index.names = ['date', 'ticker']\n",
-    "rates_er_df.index.names = ['date', 'ticker']\n",
-    "fx_df.index.names = ['date', 'ticker']\n",
-    "fx_er_df.index.names = ['date', 'ticker']\n",
-    "cmdty_df.index.names = ['date', 'ticker']\n",
-    "cmdty_er_df.index.names = ['date', 'ticker']"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 439,
-   "id": "cdfceea4-af45-4bd9-be59-1de835196138",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "eqty_df = pd.concat([eqty_df, eqty_er_df], axis=1).sort_index()\n",
-    "rates_df = pd.concat([rates_df, rates_er_df], axis=1).sort_index()\n",
-    "fx_df = pd.concat([fx_df, fx_er_df], axis=1).sort_index()\n",
-    "cmdty_df = pd.concat([cmdty_df, cmdty_er_df], axis=1).sort_index()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 441,
-   "id": "15a544cf-233b-4ea0-9e4a-9d6dc3e63d3b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "eqty_df.to_parquet('s3://factorlab-data/global_eqty_data_daily.parquet')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 442,
-   "id": "a2b5599b-afff-4241-8c18-06486b1968fa",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "rates_df.to_parquet('s3://factorlab-data/global_rates_data_daily.parquet')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 443,
-   "id": "87b4f132-6f87-4770-9892-97e069d7f3c0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "fx_df.to_parquet('s3://factorlab-data/global_fx_data_daily.parquet')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 444,
-   "id": "5f21be89-a92d-4458-bca6-9a79f79b9d65",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "cmdty_df.to_parquet('s3://factorlab-data/commodity_data_daily.parquet')"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "cryptodatapy",
-   "language": "python",
-   "name": "cryptodatapy"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.12"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}