cryptodatapy 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

cryptodatapy/extract/web/Untitled.ipynb

@@ -1,559 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "id": "777df641",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import logging\n",
-    "from typing import Dict, List, Optional, Union\n",
-    "\n",
-    "import pandas as pd\n",
-    "\n",
-    "from cryptodatapy.extract.datarequest import DataRequest\n",
-    "from cryptodatapy.extract.web.web import Web\n",
-    "from cryptodatapy.transform.convertparams import ConvertParams\n",
-    "from cryptodatapy.transform.wrangle import WrangleData\n",
-    "from cryptodatapy.util.datacredentials import DataCredentials\n",
-    "\n",
-    "# data credentials\n",
-    "data_cred = DataCredentials()\n",
-    "\n",
-    "\n",
-    "class AQR(Web):\n",
-    "    \"\"\"\n",
-    "    Retrieves data from AQR data sets.\n",
-    "    \"\"\"\n",
-    "\n",
-    "    def __init__(\n",
-    "            self,\n",
-    "            categories=None,\n",
-    "            indexes: Optional[Dict[str, List[str]]] = None,\n",
-    "            assets: Optional[Dict[str, List[str]]] = None,\n",
-    "            markets: Optional[Dict[str, List[str]]] = None,\n",
-    "            market_types=None,\n",
-    "            fields: Optional[Dict[str, List[str]]] = None,\n",
-    "            frequencies=None,\n",
-    "            base_url: str = data_cred.aqr_base_url,\n",
-    "            file_formats: Optional[Union[str, List[str]]] = 'xlsx'\n",
-    "    ):\n",
-    "        \"\"\"\n",
-    "        Constructor\n",
-    "\n",
-    "        Parameters\n",
-    "        ----------\n",
-    "        categories: list or str, {'crypto', 'fx', 'rates', 'eqty', 'cmdty', 'credit', 'macro', 'alt'}\n",
-    "            List or string of available categories, e.g. ['crypto', 'fx', 'alt'].\n",
-    "        indexes: dictionary, optional, default None\n",
-    "            Dictionary of available indexes, by cat-indexes key-value pairs,  e.g. [{'eqty': ['SPX', 'N225'],\n",
-    "            'rates': [.... , ...}.\n",
-    "        assets: dictionary, optional, default None\n",
-    "            Dictionary of available assets, by cat-assets key-value pairs,  e.g. {'rates': ['Germany 2Y', 'Japan 10Y',\n",
-    "            ...], 'eqty: ['SPY', 'TLT', ...], ...}.\n",
-    "        markets: dictionary, optional, default None\n",
-    "            Dictionary of available markets, by cat-markets key-value pairs,  e.g. [{'fx': ['EUR/USD', 'USD/JPY', ...],\n",
-    "            'crypto': ['BTC/ETH', 'ETH/USDT', ...}.\n",
-    "        market_types: list\n",
-    "            List of available market types e.g. [spot', 'perpetual_future', 'future', 'option'].\n",
-    "        fields: dictionary, optional, default None\n",
-    "            Dictionary of available fields, by cat-fields key-value pairs,  e.g. {'cmdty': ['date', 'open', 'high',\n",
-    "            'low', 'close', 'volume'], 'macro': ['actual', 'previous', 'expected', 'surprise']}\n",
-    "        frequencies: dictionary\n",
-    "            Dictionary of available frequencies, by cat-frequencies key-value pairs, e.g. {'fx':\n",
-    "            ['d', 'w', 'm', 'q', 'y'], 'rates': ['d', 'w', 'm', 'q', 'y'], 'eqty': ['d', 'w', 'm', 'q', 'y'], ...}.\n",
-    "        base_url: str, optional, default None\n",
-    "            Base url used for GET requests. If not provided, default is set to base_url stored in DataCredentials.\n",
-    "        file_formats: list or str, {'xlsx', 'xls'}, default 'xlsx'\n",
-    "            List of available file formats.\n",
-    "        \"\"\"\n",
-    "        Web.__init__(\n",
-    "            self,\n",
-    "            categories,\n",
-    "            indexes,\n",
-    "            assets,\n",
-    "            markets,\n",
-    "            market_types,\n",
-    "            fields,\n",
-    "            frequencies,\n",
-    "            base_url,\n",
-    "            file_formats\n",
-    "        )\n",
-    "\n",
-    "        if categories is None:\n",
-    "            self.categories = [\"fx\", \"rates\", \"eqty\", \"cmdty\", \"credit\"]\n",
-    "        if frequencies is None:\n",
-    "            self.frequencies = {\n",
-    "                \"fx\": [\"m\", \"q\", \"y\"],\n",
-    "                \"rates\": [\"m\", \"q\", \"y\"],\n",
-    "                \"cmdty\": [\"m\", \"q\", \"y\"],\n",
-    "                \"eqty\": [\"d\", \"w\", \"m\", \"q\", \"y\"],\n",
-    "                \"credit\": [\"m\", \"q\", \"y\"],\n",
-    "            }\n",
-    "        if market_types is None:\n",
-    "            self.market_types = [\"spot\", \"future\"]\n",
-    "        if fields is None:\n",
-    "            self.fields = self.get_fields_info()\n",
-    "\n",
-    "    def get_indexes_info(self) -> None:\n",
-    "        \"\"\"\n",
-    "        Get indexes info.\n",
-    "        \"\"\"\n",
-    "        eqty_idxs = ['AUS', 'AUT', 'BEL', 'CAN', 'CHE', 'DEU', 'DNK', 'ESP', 'FIN', 'FRA', 'GBR', 'GRC', 'HKG', 'IRL',\n",
-    "                     'ISR', 'ITA', 'JPN', 'NLD', 'NOR', 'NZL', 'PRT', 'SGP', 'SWE', 'USA', 'WLD']\n",
-    "        print(\n",
-    "            f\"AQR publishes excess returns data for the following equity market indexes: {eqty_idxs}\"\n",
-    "        )\n",
-    "\n",
-    "    def get_assets_info(self) -> None:\n",
-    "        \"\"\"\n",
-    "        Get assets info.\n",
-    "        \"\"\"\n",
-    "        print(\n",
-    "            f\"AQR does not publish data for individual assets.\"\n",
-    "        )\n",
-    "\n",
-    "    def get_markets_info(self) -> None:\n",
-    "        \"\"\"\n",
-    "        Get markets info.\n",
-    "        \"\"\"\n",
-    "        print(\n",
-    "            f\"AQR does not publish data for individual markets.\"\n",
-    "        )\n",
-    "\n",
-    "    @staticmethod\n",
-    "    def get_fields_info(\n",
-    "            data_type: Optional[str] = \"market\", cat: Optional[str] = None\n",
-    "    ) -> Dict[str, List[str]]:\n",
-    "        \"\"\"\n",
-    "        Get fields info.\n",
-    "\n",
-    "        Parameters\n",
-    "        ----------\n",
-    "        data_type: str, {'market', 'on-chain', 'off-chain'}, default 'market'\n",
-    "            Type of data.\n",
-    "        cat: str, {'crypto', 'eqty', 'fx', 'rates', 'cmdty', 'macro'}, optional, default None\n",
-    "            Asset class or time series category.\n",
-    "\n",
-    "        Returns\n",
-    "        -------\n",
-    "        fields: dictionary\n",
-    "            Dictionary with info on available fields, by category.\n",
-    "        \"\"\"\n",
-    "        if data_type == \"on-chain\" or data_type == 'off-chain':\n",
-    "            raise ValueError(\n",
-    "                \"AQR only publishes total and excess return series used in their research papers.\"\n",
-    "            )\n",
-    "\n",
-    "        # list of fields\n",
-    "        market_fields_list = ['ret', 'tr', 'er']\n",
-    "\n",
-    "        # fields dict\n",
-    "        fields = {\n",
-    "            \"fx\": market_fields_list,\n",
-    "            \"rates\": market_fields_list,\n",
-    "            \"eqty\": market_fields_list,\n",
-    "            \"cmdty\": market_fields_list,\n",
-    "            \"credit\": market_fields_list,\n",
-    "        }\n",
-    "\n",
-    "        # fields obj\n",
-    "        if cat is not None:\n",
-    "            fields = fields[cat]\n",
-    "\n",
-    "        return fields\n",
-    "\n",
-    "    def set_excel_params(self, data_req: DataRequest, ticker: str) -> Dict[str, Union[str, int]]:\n",
-    "        \"\"\"\n",
-    "        Sets excel parameters for reading excel files.\n",
-    "\n",
-    "        Parameters\n",
-    "        ----------\n",
-    "        data_req: DataRequest\n",
-    "            Parameters of data request in CryptoDataPy format.\n",
-    "        ticker: str\n",
-    "            Ticker symbol.\n",
-    "\n",
-    "        Returns\n",
-    "        -------\n",
-    "        dict: dictionary\n",
-    "            Dictionary with params to read excel file.\n",
-    "\n",
-    "        \"\"\"\n",
-    "        # convert data request parameters to aqr format\n",
-    "        conv_data_req = ConvertParams(data_req).to_aqr()\n",
-    "\n",
-    "        # param dict\n",
-    "        params = {\n",
-    "            'file': conv_data_req['tickers'][ticker][0],  # file name\n",
-    "            'freq': conv_data_req['freq'],  # freq\n",
-    "            'format': self.file_formats[0],  # file format\n",
-    "            'sheet': conv_data_req['tickers'][ticker][1],  # sheet name\n",
-    "            'url': None,  # url\n",
-    "            'parse_dates': True,  # parsing dates\n",
-    "            'index_col': None,      # index col\n",
-    "            'header': None      # header row\n",
-    "        }\n",
-    "        # set index url, col and header\n",
-    "        params['url'] = self.base_url + params['file'] + params['freq'] + \".\" + params['format']\n",
-    "        if params['file'] == 'Century-of-Factor-Premia-':\n",
-    "            params['index_col'] = 'Unnamed: 0'\n",
-    "            params['header'] = 18\n",
-    "        elif params['file'] == 'Time-Series-Momentum-Factors-':\n",
-    "            params['index_col'] = 'Unnamed: 0'\n",
-    "            params['header'] = 17\n",
-    "        elif params['file'] == 'Commodities-for-the-Long-Run-Index-Level-Data-':\n",
-    "            params['index_col'] = 'Unnamed: 0'\n",
-    "            params['header'] = 10\n",
-    "        elif params['file'] == 'Credit-Risk-Premium-Preliminary-Paper-Data':\n",
-    "            params['url'] = self.base_url + params['file'] + \".\" + params['format']\n",
-    "            params['index_col'] = 'Date'\n",
-    "            params['header'] = 10\n",
-    "        else:\n",
-    "            params['index_col'] = 'DATE'\n",
-    "            params['header'] = 18\n",
-    "\n",
-    "        return params\n",
-    "\n",
-    "    def get_series(self, data_req: DataRequest) -> Dict[str, pd.DataFrame]:\n",
-    "        \"\"\"\n",
-    "        Gets series from AQR data file.\n",
-    "\n",
-    "        Parameters\n",
-    "        ----------\n",
-    "        data_req: DataRequest\n",
-    "            Parameters of data request in CryptoDataPy format.\n",
-    "\n",
-    "        Returns\n",
-    "        -------\n",
-    "        dfs_dict: dictionary\n",
-    "            Dictionary with ticker-dataframe key-value pairs.\n",
-    "\n",
-    "        \"\"\"\n",
-    "        # convert data request parameters to aqr format\n",
-    "        conv_data_req = ConvertParams(data_req).to_aqr()\n",
-    "\n",
-    "        try:\n",
-    "            # fetch data\n",
-    "            df_dicts = {}\n",
-    "\n",
-    "            for ticker in conv_data_req['tickers']:\n",
-    "                # set excel params\n",
-    "                params = self.set_excel_params(data_req, ticker)\n",
-    "                # fetch excel file\n",
-    "                df1 = pd.read_excel(params['url'], sheet_name=params['sheet'], index_col=params['index_col'],\n",
-    "                                    parse_dates=params['parse_dates'], header=params['header'])\n",
-    "                # add df to dicts\n",
-    "                df_dicts[ticker] = df1\n",
-    "\n",
-    "        except Exception as e:\n",
-    "            logging.warning(e)\n",
-    "            logging.warning(f\"Failed to get data for: {conv_data_req['tickers']}.\")\n",
-    "\n",
-    "        else:\n",
-    "            return df_dicts\n",
-    "\n",
-    "    @staticmethod\n",
-    "    def wrangle_data_resp(data_req: DataRequest, data_resp: Dict[str, pd.DataFrame]) -> pd.DataFrame:\n",
-    "        \"\"\"\n",
-    "        Wrangle data response.\n",
-    "\n",
-    "        Parameters\n",
-    "        ----------\n",
-    "        data_req: DataRequest\n",
-    "            Parameters of data request in CryptoDataPy format.\n",
-    "        data_resp: Dictionary\n",
-    "            Dictionary with ticker-dataframe key-value pairs.\n",
-    "\n",
-    "        Returns\n",
-    "        -------\n",
-    "        df: pd.DataFrame\n",
-    "            Wrangled dataframe with DatetimeIndex (level 0), ticker (level 1), and values for market or macro series\n",
-    "            for selected fields (cols), in tidy format.\n",
-    "        \"\"\"\n",
-    "        # wrangle data resp\n",
-    "        df = WrangleData(data_req, data_resp).aqr()\n",
-    "\n",
-    "        return df\n",
-    "\n",
-    "    def get_tidy_data(self, data_req: DataRequest) -> pd.DataFrame:\n",
-    "        \"\"\"\n",
-    "        Gets data from FRED and wrangles the data response into tidy data format.\n",
-    "\n",
-    "        Parameters\n",
-    "        ----------\n",
-    "        data_req: DataRequest\n",
-    "            Parameters of data request in CryptoDataPy format.\n",
-    "\n",
-    "        Returns\n",
-    "        -------\n",
-    "        df: pd.DataFrame - MultiIndex\n",
-    "            Dataframe with DatetimeIndex (level 0), tickers (level 1) and actual values (cols),\n",
-    "            in tidy data format.\n",
-    "        \"\"\"\n",
-    "        # change to get series\n",
-    "        data_resp = self.get_series(data_req)\n",
-    "        # wrangle data resp\n",
-    "        df = self.wrangle_data_resp(data_req, data_resp)\n",
-    "\n",
-    "        return df\n",
-    "\n",
-    "    def check_params(self, data_req: DataRequest) -> None:\n",
-    "        \"\"\"\n",
-    "        Checks the data request parameters before requesting data to reduce API calls\n",
-    "        and improve efficiency.\n",
-    "\n",
-    "        \"\"\"\n",
-    "        # check cat\n",
-    "        if data_req.cat not in self.categories:\n",
-    "            raise ValueError(\n",
-    "                f\"Select a valid category. Valid categories are: {self.categories}.\"\n",
-    "            )\n",
-    "        # check freq\n",
-    "        if data_req.freq not in self.frequencies[data_req.cat]:\n",
-    "            raise ValueError(\n",
-    "                f\"Invalid data frequency. Valid data frequencies are: {self.frequencies}.\"\n",
-    "            )\n",
-    "        # check fields\n",
-    "        if not any(field in self.fields[data_req.cat] for field in data_req.fields):\n",
-    "            raise ValueError(f\"Invalid fields. Valid data fields are: {self.fields}.\")\n",
-    "\n",
-    "    def get_data(self, data_req: DataRequest) -> pd.DataFrame:\n",
-    "        \"\"\"\n",
-    "        Get data.\n",
-    "\n",
-    "        Parameters\n",
-    "        data_req: DataRequest\n",
-    "            Parameters of data request in CryptoDataPy format.\n",
-    "\n",
-    "        Returns\n",
-    "        -------\n",
-    "        df: pd.DataFrame - MultiIndex\n",
-    "            DataFrame with DatetimeIndex (level 0), ticker (level 1), and values for selected fields (cols),\n",
-    "            in tidy format.\n",
-    "        \"\"\"\n",
-    "        # check params\n",
-    "        self.check_params(data_req)\n",
-    "\n",
-    "        # get tidy data\n",
-    "        df = self.get_tidy_data(data_req)\n",
-    "\n",
-    "        # check if df empty\n",
-    "        if df.empty:\n",
-    "            raise Exception(\n",
-    "                \"No data returned. Check data request parameters and try again.\"\n",
-    "            )\n",
-    "\n",
-    "        return df.sort_index()\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 11,
-   "id": "67d71194",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "aqr = AQR()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 12,
-   "id": "b2d605c3",
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "AQR publishes excess returns data for the following equity market indexes: ['AUS', 'AUT', 'BEL', 'CAN', 'CHE', 'DEU', 'DNK', 'ESP', 'FIN', 'FRA', 'GBR', 'GRC', 'HKG', 'IRL', 'ISR', 'ITA', 'JPN', 'NLD', 'NOR', 'NZL', 'PRT', 'SGP', 'SWE', 'USA', 'WLD']\n"
-     ]
-    }
-   ],
-   "source": [
-    "aqr.get_indexes_info()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 26,
-   "id": "b508d7eb",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dr = DataRequest(source='aqr', tickers=['US_Eqty_Qual', 'US_Eqty_Beta'], freq='m', cat='eqty', fields='tr')"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 27,
-   "id": "ccc66aad",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df = aqr.get_data(dr)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 28,
-   "id": "66db393a",
-   "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/html": [
-       "<div>\n",
-       "<style scoped>\n",
-       "    .dataframe tbody tr th:only-of-type {\n",
-       "        vertical-align: middle;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe tbody tr th {\n",
-       "        vertical-align: top;\n",
-       "    }\n",
-       "\n",
-       "    .dataframe thead th {\n",
-       "        text-align: right;\n",
-       "    }\n",
-       "</style>\n",
-       "<table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       "    <tr style=\"text-align: right;\">\n",
-       "      <th></th>\n",
-       "      <th></th>\n",
-       "      <th>er</th>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>date</th>\n",
-       "      <th>ticker</th>\n",
-       "      <th></th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "    <tr>\n",
-       "      <th>1930-12-31</th>\n",
-       "      <th>US_Eqty_Beta</th>\n",
-       "      <td>-0.000558</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1931-01-31</th>\n",
-       "      <th>US_Eqty_Beta</th>\n",
-       "      <td>-0.022446</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1931-02-28</th>\n",
-       "      <th>US_Eqty_Beta</th>\n",
-       "      <td>-0.077423</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1931-03-31</th>\n",
-       "      <th>US_Eqty_Beta</th>\n",
-       "      <td>0.029235</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>1931-04-30</th>\n",
-       "      <th>US_Eqty_Beta</th>\n",
-       "      <td>-0.012986</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>...</th>\n",
-       "      <th>...</th>\n",
-       "      <td>...</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>2023-04-30</th>\n",
-       "      <th>US_Eqty_Qual</th>\n",
-       "      <td>0.010278</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th rowspan=\"2\" valign=\"top\">2023-05-31</th>\n",
-       "      <th>US_Eqty_Beta</th>\n",
-       "      <td>-0.053566</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>US_Eqty_Qual</th>\n",
-       "      <td>-0.039877</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th rowspan=\"2\" valign=\"top\">2023-06-30</th>\n",
-       "      <th>US_Eqty_Beta</th>\n",
-       "      <td>-0.01</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <th>US_Eqty_Qual</th>\n",
-       "      <td>-0.013823</td>\n",
-       "    </tr>\n",
-       "  </tbody>\n",
-       "</table>\n",
-       "<p>1903 rows × 1 columns</p>\n",
-       "</div>"
-      ],
-      "text/plain": [
-       "                               er\n",
-       "date       ticker                \n",
-       "1930-12-31 US_Eqty_Beta -0.000558\n",
-       "1931-01-31 US_Eqty_Beta -0.022446\n",
-       "1931-02-28 US_Eqty_Beta -0.077423\n",
-       "1931-03-31 US_Eqty_Beta  0.029235\n",
-       "1931-04-30 US_Eqty_Beta -0.012986\n",
-       "...                           ...\n",
-       "2023-04-30 US_Eqty_Qual  0.010278\n",
-       "2023-05-31 US_Eqty_Beta -0.053566\n",
-       "           US_Eqty_Qual -0.039877\n",
-       "2023-06-30 US_Eqty_Beta     -0.01\n",
-       "           US_Eqty_Qual -0.013823\n",
-       "\n",
-       "[1903 rows x 1 columns]"
-      ]
-     },
-     "execution_count": 28,
-     "metadata": {},
-     "output_type": "execute_result"
-    }
-   ],
-   "source": [
-    "df"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3bde6a95",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "dd8784b5",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "cryptodatapy",
-   "language": "python",
-   "name": "cryptodatapy"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.9.12"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}