oxtapus 0.2.2__py3-none-any.whl

oxtapus/econ/__init__.py

@@ -0,0 +1 @@
+from .tgju import TGJU

oxtapus/econ/tgju.py

@@ -0,0 +1,58 @@
+import pandas as pd
+from ..utils import get
+
+
+class TGJU:
+    """
+    دریافتِ داده‌هایِ گذشته‌یِ سایتِ www.econ.org
+    """
+
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def get_hist_price(item):
+        url = f"https://api.tgju.org/v1/market/indicator/summary-table-data/{item}"
+        main = get(url=url, timeout=(2, 6), verify=True).json()
+        df = pd.DataFrame().from_records(main["data"]).drop([4, 5], axis=1)
+        df.columns = ["open", "low", "high", "close", "date", "jdate"]
+        df[["open", "low", "high", "close"]] = df[
+            ["open", "low", "high", "close"]
+        ].applymap(lambda x: float(x.replace(",", "")))
+        df["date"] = pd.to_datetime(df.date)
+        return df.set_index("date").sort_index()
+
+    def usd_irr(self):
+        """
+        دلار/ریال
+        :return:
+        """
+        return self.get_hist_price("price_dollar_rl")
+
+    def sekke_emami(self):
+        """
+        سکه‌یِ امامی
+        :return:
+        """
+        return self.get_hist_price("sekee")
+
+    def nim_sekke(self):
+        """
+        نیم-سکه
+        :return:
+        """
+        return self.get_hist_price("nim")
+
+    def rob_sekke(self):
+        """
+        ربعِ-سکه
+        :return:
+        """
+        return self.get_hist_price("rob")
+
+    def ons(self):
+        """
+        اونس طلا
+        :return:
+        """
+        return self.get_hist_price("ons")

oxtapus/ise/__init__.py

@@ -0,0 +1,3 @@
+from .tsetmc import TSETMC
+from .rahavard import Rahavard
+from .codal import Codal

oxtapus/ise/codal.py

@@ -0,0 +1,86 @@
+import pandas as pd
+
+from .codal_utils import URL, QueryParameters, ced, cols
+from ..utils import get
+
+
+class Codal(URL):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def letters(self):
+        main = get(self.search_url, verify=True, timeout=(2, 7)).json()
+        df = pd.DataFrame(main.get("Letters")).rename(columns=cols.letters.rename)[
+            cols.letters.rep
+        ]
+        page = main.get("Page")
+        if page > 1:
+            for i in range(1, page + 1):
+                self.page_number = i
+                main_ = get(self.search_url).json()
+                df_ = pd.DataFrame(main_.get("Letters")).rename(
+                    columns=cols.letters.rename
+                )[cols.letters.rep]
+                df = pd.concat([df, df_])
+        df.replace({"\u200c": " "}, regex=True)
+        return df
+
+    def income_statements(self):
+        letters = self.letters()
+        df = pd.DataFrame()
+        for i, row in letters.iterrows():
+            url = self.financial_statements(letter_url=row.url, sheet_id=1)
+            try:
+                r = get(url, verify=True, timeout=(2, 10))
+                df_ = ced.income_statements(r.text)
+                df_ = df_.join(
+                    pd.DataFrame.from_dict(
+                        [
+                            row[
+                                [
+                                    "symbol",
+                                    "company_name",
+                                    "title",
+                                    "issue_datetime",
+                                    "url",
+                                ]
+                            ].to_dict()
+                        ]
+                    )
+                )
+                df = pd.concat([df, df_])
+            except Exception as e:
+                print(e)
+                print(url)
+
+        return df[cols.income_statements.rep]
+
+    def balance_sheet(self):
+        letters = self.letters()
+        df = pd.DataFrame()
+        for i, row in letters.iterrows():
+            url = self.financial_statements(letter_url=row.url, sheet_id=0)
+            try:
+                r = get(url, verify=True, timeout=(2, 10))
+                df_ = ced.balance_sheet(r.text)
+                df_ = df_.join(
+                    pd.DataFrame.from_dict(
+                        [
+                            row[
+                                [
+                                    "symbol",
+                                    "company_name",
+                                    "title",
+                                    "issue_datetime",
+                                    "url",
+                                ]
+                            ].to_dict()
+                        ]
+                    )
+                )
+                df = pd.concat([df, df_])
+            except Exception as e:
+                print(e)
+                print(url)
+        return df[cols.balance_sheet.rep]
+

oxtapus/ise/codal_utils/__init__.py

@@ -0,0 +1,3 @@
+from .columns import cols
+from .constants import URL, QueryParameters
+from .clean_and_extend_data import ced

oxtapus/ise/codal_utils/clean_and_extend_data.py

@@ -0,0 +1,127 @@
+from collections import namedtuple
+import re
+import json
+import pandas as pd
+from hazm import Normalizer
+
+from .columns import cols
+from .fs_items import fs_items
+
+normalizer = Normalizer()
+Ced = namedtuple("Ced", ["letters", "balance_sheet", "income_statements", "cash_flows"])
+
+
+# ------------- ****** -------------
+def codal_parser(text):
+    t = text.split("var datasource =")[1].split("};")[0].strip() + "}"
+    return json.loads(t)
+
+
+def arabic_char(s: str):
+    return s.translate(str.maketrans({"ي": "ی", "ك": "ک"}))
+
+
+def match_date(t):
+    t = t.replace("/", "-")
+    try:
+        return re.findall("\d{1,4}-\d{1,2}-\d{2,4}", t)[0]
+    except:
+        print(("enter valid text to match date"))
+
+
+# ------------- letters -------------
+def letters(url):
+    pass
+
+
+# ------------- balance_sheet -------------
+def clean_balance_sheet(df):
+    df.item = (
+        df.item.fillna("").map(normalizer.normalize).map(fs_items.balance_sheet)
+    )
+    df.dropna(inplace=True)
+    df.iloc[:, 1] = pd.to_numeric(df.iloc[:, 1], errors="raise")
+    df.dropna(inplace=True)
+    df.drop_duplicates(inplace=True)
+    df = df.groupby("item").sum().reset_index()
+    df = (
+        df.pivot_table(columns=["item"], aggfunc=sum)
+        .reset_index()
+        .rename(columns={"index": "period_end_to_date"})
+    )
+    df.columns.set_names(None, inplace=True)
+    return df
+
+
+def balance_sheet(r):
+    dict_ = codal_parser(r)
+    records = dict_["sheets"][0]["tables"][0]["cells"]
+    try:
+        if dict_["sheets"][0]["tables"][1]["aliasName"] == "BalanceSheet":
+            records = dict_["sheets"][0]["tables"][1]["cells"]
+    except :
+        records = dict_["sheets"][0]["tables"][0]["cells"]
+    cols_ = ["item", dict_["periodEndToDate"]]
+    df = pd.DataFrame()
+    for i in records:
+        df.loc[i.get("rowSequence"), i.get("columnCode")] = i.get("value")
+    len_cols = len(df.columns)
+    df.sort_index(inplace=True)
+    if len_cols > 7:
+        df_right = df.iloc[:, :2]
+        df_right.columns = cols_
+        df_left = df.iloc[:, int(len_cols / 2):int(len_cols / 2 + 2)]
+        df_left.columns = cols_
+        df = pd.concat([df_right, df_left])
+        return clean_balance_sheet(df)
+    else:
+        df = df.iloc[:, :2]
+        df.columns = cols_
+        return clean_balance_sheet(df)
+
+
+# ------------- income_statements -------------
+
+
+def clean_income_statements(df):
+    df = df[[1, 2]]
+    df.columns = ["item", match_date(df[df[1] == "شرح"][2].values[0])]
+    df.item = (
+        df.item.fillna("").map(normalizer.normalize).map(fs_items.income_statements)
+    )
+    df.dropna(inplace=True)
+    df.iloc[:, 1] = pd.to_numeric(df.iloc[:, 1], errors="raise")
+    df.dropna(inplace=True)
+    df.drop_duplicates(inplace=True)
+    df = df.groupby("item").sum().reset_index()
+    df = (
+        df.pivot_table(columns=["item"], aggfunc=sum)
+        .reset_index()
+        .rename(columns={"index": "period_end_to_date"})
+    )
+    df.columns.set_names(None, inplace=True)
+    return df
+
+
+def income_statements(r: str):
+    dict_ = codal_parser(r)
+    records = dict_["sheets"][0]["tables"][0]["cells"]
+    if dict_["sheets"][0]["tables"][1]["aliasName"] == "IncomeStatement":
+        records = dict_["sheets"][0]["tables"][1]["cells"]
+    df = pd.DataFrame()
+    for i in records:
+        df.loc[i.get("rowCode"), i.get("columnCode")] = i.get("value")
+    return clean_income_statements(df.sort_index())
+
+
+# ------------- cash_flows -------------
+def cash_flows():
+    pass
+
+
+ced = Ced(
+    letters=letters,
+    balance_sheet=balance_sheet,
+    income_statements=income_statements,
+    cash_flows=cash_flows,
+)

oxtapus/ise/codal_utils/columns.py

@@ -0,0 +1,128 @@
+from collections import namedtuple
+
+Cols = namedtuple("Cols", ["letters", "income_statements", "balance_sheet"])
+Property = namedtuple("Property", ["rename", "drop", "rep"])
+
+_letters = {
+    "rename": {
+        "Symbol": "symbol",
+        "CompanyName": "company_name",
+        "Title": "title",
+        "SentDateTime": "sent_datetime",
+        "PublishDateTime": "issue_datetime",
+        "HasAttachment": "has_attachment",
+        "HasPdf": "has_pdf",
+        "Url": "url",
+        "AttachmentUrl": "attachment_url",
+        "PdfUrl": "pdf_url",
+        "ExcelUrl": "excel_url",
+    },
+    "rep": [
+        "symbol",
+        "company_name",
+        "title",
+        "sent_datetime",
+        "issue_datetime",
+        "has_attachment",
+        "has_pdf",
+        "url",
+        "attachment_url",
+        "pdf_url",
+        "excel_url",
+    ],
+}
+
+_income_statements = {
+    "rep": [
+        "symbol",
+        "company_name",
+        "title",
+        "issue_datetime",
+        "period_end_to_date",
+        "revenue",
+        "cost_of_goods_sold",
+        "gross_profit",
+        "sales_general_and_administrative_expense",
+        "cost_of_decrease_in_receivables_value(exceptional_cost)",
+        "other_operating_revenue",
+        "other_operating_expenses",
+        "operating_income",
+        "finance_expense",
+        "net_miscellaneous_revenue",
+        "net_income_from_continuing_operations_before_tax",
+        "taxes",
+        "net_income_from_continuing_operation",
+        "discontinued_operations_profit",
+        "net_income",
+        "earning_per_share",
+        "listed_capital",
+        "url",
+    ]
+}
+
+_balance_sheet = {
+    "rep": [
+        "symbol",
+        "company_name",
+        "title",
+        "issue_datetime",
+        "period_end_to_date",
+        "tangible_fixed_assets",
+        "investment_property",
+        "intangible_assets",
+        "long_term_investments",
+        "long_term_receivables",
+        "other_assets",
+        "total_non_current_assets",
+        "prepayments",
+        "inventories",
+        "trade_and_other_receivables",
+        "short_term_investments",
+        "cash_and_equivalents",
+        "asset_for_sale",
+        "total_current_assets",
+        "total_assets",
+        "common_stock",
+        "received_for_capital_advance",
+        "capital_surplus",
+        "treasury_stock_surplus",
+        "legal_reserve",
+        "expansion_reserve",
+        "revaluation_surplus",
+        "exchange_differences_on_translation",
+        "retained_earnings",
+        "treasury_stock",
+        "total_shareholders_equity",
+        "long_term_liabilities",
+        "long_term_debt",
+        "allowance_for_post_retirement",
+        "total_non_current_liabilities",
+        "trade_and_other_liabilities",
+        "deferred_tax_liabilities",
+        "dividends_payable",
+        "loan_payable",
+        "provisions",
+        "deferred_revenue",
+        "liabilities_related_to_assets_for_sale",
+        "total_current_liabilities",
+        "total_liabilities",
+        "total_liabilities_and_shareholders_equities",
+        "url",
+    ]
+}
+
+letters = Property(
+    rename=_letters.get("rename"), drop=_letters.get("drop"), rep=_letters.get("rep")
+)
+income_statements = Property(
+    rename=_income_statements.get("rename"),
+    drop=_income_statements.get("drop"),
+    rep=_income_statements.get("rep"),
+)
+balance_sheet = Property(
+    rename=_balance_sheet.get("rename"),
+    drop=_balance_sheet.get("drop"),
+    rep=_balance_sheet.get("rep"),
+)
+
+cols = Cols(letters=letters, income_statements=income_statements, balance_sheet=balance_sheet)

oxtapus/ise/codal_utils/constants.py

@@ -0,0 +1,148 @@
+from urllib.parse import urlencode
+
+
+class QueryParameters:
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._symbol = "فملی"
+        self._audited = "true"
+        self._category = 1
+        self._childes = "false"
+        self._length = -1
+        self._mains = "true"
+        self._page_number = 1
+
+    @property
+    def symbol(self):
+        return self._symbol
+
+    @symbol.setter
+    def symbol(self, symbol):
+        self._symbol = symbol
+
+    @property
+    def audited(self):
+        return self._audited
+
+    @audited.setter
+    def audited(self, audited: bool):
+        if audited:
+            self._audited = "true"
+        elif not audited:
+            self._audited = "false"
+
+    @property
+    def category(self):
+        return self._category
+
+    @category.setter
+    def category(self, category: int):
+        match category:
+            case 1:
+                self._category = category
+            case 2:
+                self._category = category
+            case 3:
+                self._category = category
+            case _:
+                self._category = -1
+
+    @property
+    def childes(self):
+        return self._childes
+
+    @childes.setter
+    def childes(self, childes: bool):
+        if childes:
+            self._childes = "true"
+        elif not childes:
+            self._childes = "false"
+
+    @property
+    def length(self):
+        return self._length
+
+    @length.setter
+    def length(self, length: int):
+        if self._category == 1:
+            match length:
+                case 3:
+                    self._length = length
+                case 6:
+                    self._length = length
+                case 12:
+                    self._length = length
+                case _:
+                    self._length = -1
+        elif length in range(1, 13):
+            self._length = length
+        else:
+            self._length = -1
+
+    @property
+    def mains(self):
+        return self._mains
+
+    @mains.setter
+    def mains(self, mains: bool):
+        if mains:
+            self._mains = "true"
+        elif not mains:
+            self._mains = "false"
+
+    @property
+    def page_number(self):
+        return self._page_number
+
+    @page_number.setter
+    def page_number(self, page_number):
+        self._page_number = page_number
+
+    @property
+    def parameters(self):
+        return {
+            "Audited": self._audited,  # حسابرسی شده
+            "AuditorRef": -1,  #
+            "Category": self._category,  #
+            "Childs": self._childes,  # زیرمجموعه
+            "CompanyState": 0,  #
+            "CompanyType": 1,  #
+            "Consolidatable": "true",  # تلفیقی
+            "FromDate": "1395/01/01",
+            "IsNotAudited": "false",  # حسابرسی نشده
+            "Isic": 271018,  #
+            "Length": self._length,  # طولِ دوره
+            "LetterType": 6,  # نوعِ اطلاعیه
+            "Mains": self._mains,  # اصلی
+            "NotAudited": "false",  #
+            "NotConsolidatable": "true",  #
+            "PageNumber": self._page_number,  #
+            "Publisher": "false",  #
+            "Symbol": self._symbol,  # نماد
+            # "ToDate": "1402/05/03",
+            "TracingNo": -1,  #
+            "search": "true",
+        }
+
+    @property
+    def query_parameters(self):
+        return urlencode(self.parameters)
+
+
+class URL(QueryParameters):
+    def __init__(
+        self,
+        base_url="https://codal.ir",
+        api="https://search.codal.ir/api/search/v2/q?",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.base_url = base_url
+        self.api = api
+
+    @property
+    def search_url(self):
+        return self.api + self.query_parameters
+
+    def financial_statements(self, letter_url, sheet_id):
+        return self.base_url + f"{letter_url}&sheetId={sheet_id}"