financechatbotkit 2.0.0__py3-none-any.whl

orchestrator/fnguide/workflow.py

@@ -0,0 +1,391 @@
+"""FnGuide workflows for financial statements and ratios.
+- Follow the policies of the target website and use appropriate delays between requests if needed.
+"""
+
+from __future__ import annotations
+
+from html import unescape
+import logging
+import re
+from typing import Any
+
+import httpx
+
+from ..exceptions import DownloadError, InvalidInputError
+
+FN_GUIDE_FINANCE_RATIO_URL = "https://comp.fnguide.com/SVO2/ASP/SVD_FinanceRatio.asp"
+FN_GUIDE_FINANCE_STATEMENT_URL = "https://comp.fnguide.com/SVO2/ASP/SVD_Finance.asp"
+
+_TAG_RE = re.compile(r"<[^>]+>", re.S)
+_WHITESPACE_RE = re.compile(r"\s+")
+_TABLE_SECTION_RE = re.compile(
+    r'<div class="ul_col2wrap pd_t25">.*?<h2>(.*?)</h2>\s*'
+    r'<span class="stxt">(.*?)</span>.*?'
+    r'<span class="txt1">(.*?)</span>.*?'
+    r"(<table\b.*?</table>)",
+    re.S,
+)
+_TABLE_RE = re.compile(r"<table\b.*?</table>", re.S)
+_ROW_RE = re.compile(r"<tr\b([^>]*)>(.*?)</tr>", re.S)
+_CELL_RE = re.compile(r"<(th|td)\b([^>]*)>(.*?)</(?:th|td)>", re.S)
+
+logger = logging.getLogger(__name__)
+
+
+def build_fnguide_params(
+    *,
+    gicode: str,
+    new_menu_id: str,
+    pgb: str = "1",
+    cid: str = "",
+    menu_yn: str = "Y",
+    report_gb: str = "",
+    stk_gb: str = "701",
+) -> dict[str, Any]:
+    return {
+        "pGB": pgb,
+        "gicode": gicode,
+        "cID": cid,
+        "MenuYn": menu_yn,
+        "ReportGB": report_gb,
+        "NewMenuID": new_menu_id,
+        "stkGb": stk_gb,
+    }
+
+
+def build_finance_ratio_params(
+    *,
+    gicode: str,
+    report_gb: str = "",
+) -> dict[str, Any]:
+    return build_fnguide_params(gicode=gicode, new_menu_id="104", report_gb=report_gb)
+
+
+def build_finance_statement_params(
+    *,
+    gicode: str,
+    report_gb: str = "D",
+) -> dict[str, Any]:
+    return build_fnguide_params(gicode=gicode, new_menu_id="103", report_gb=report_gb)
+
+
+def fetch_fnguide_page(
+    *,
+    url: str,
+    params: dict[str, Any],
+    timeout: float = 30.0,
+    user_agent: str | None = None,
+    headers: dict[str, str] | None = None,
+) -> str:
+    request_headers: dict[str, str] = {}
+    if user_agent is not None:
+        request_headers["User-Agent"] = user_agent
+    if headers:
+        request_headers.update(headers)
+
+    try:
+        response = httpx.get(
+            url,
+            params=params,
+            headers=request_headers if request_headers else None,
+            timeout=timeout,
+            follow_redirects=True,
+        )
+        response.raise_for_status()
+    except httpx.HTTPError as exc:
+        logger.warning("FnGuide request failed for %s: %s", url, exc)
+        raise DownloadError(f"Failed to fetch FnGuide page: {url}") from exc
+
+    return response.text
+
+
+def _normalize_text(text: str) -> str:
+    text = unescape(text).replace("\xa0", " ")
+    text = _WHITESPACE_RE.sub(" ", text)
+    return text.strip()
+
+
+def _strip_tags(html: str) -> str:
+    cleaned = re.sub(r"<br\s*/?>", "\n", html, flags=re.I)
+    cleaned = _TAG_RE.sub(" ", cleaned)
+    return _normalize_text(cleaned)
+
+
+def _extract_attr(tag_attrs: str, name: str) -> str | None:
+    match = re.search(rf'{name}=(["\'])(.*?)\1', tag_attrs, re.S)
+    if not match:
+        return None
+    return unescape(match.group(2)).strip()
+
+
+def _clean_label_html(cell_html: str) -> str:
+    cleaned = re.sub(r"<dl[^>]*>.*?</dl>", " ", cell_html, flags=re.S)
+    cleaned = re.sub(
+        r'<a\b[^>]*class=["\']btn_acdopen["\'][^>]*>.*?</a>',
+        " ",
+        cleaned,
+        flags=re.S,
+    )
+    cleaned = re.sub(
+        r'<span\b[^>]*class=["\']blind["\'][^>]*>.*?</span>',
+        " ",
+        cleaned,
+        flags=re.S,
+    )
+    return _strip_tags(cleaned)
+
+
+def _parse_value_cells(cells: list[dict[str, Any]], columns: list[str]) -> list[dict[str, str | None]]:
+    values: list[dict[str, str | None]] = []
+    for column, cell in zip(columns, cells[1:]):
+        values.append(
+            {
+                "period": column,
+                "text": cell["text"] or None,
+                "title": cell["title"] or None,
+            }
+        )
+    return values
+
+
+def _parse_table_row(row_attrs: str, row_html: str, columns: list[str]) -> dict[str, Any]:
+    cells = []
+    for _, cell_attrs, cell_html in _CELL_RE.findall(row_html):
+        cells.append(
+            {
+                "attrs": cell_attrs,
+                "html": cell_html,
+                "text": _strip_tags(cell_html),
+                "title": _extract_attr(cell_attrs, "title"),
+            }
+        )
+
+    label_cell = cells[0] if cells else {"html": "", "text": ""}
+    row_class = _extract_attr(row_attrs, "class") or ""
+
+    return {
+        "_row_class": row_class,
+        "_row_id": _extract_attr(row_attrs, "id"),
+        "label": _clean_label_html(label_cell["html"]) or label_cell["text"],
+        "values": _parse_value_cells(cells, columns),
+    }
+
+
+def _infer_table_period_kind(subtitle: str, columns: list[str]) -> str:
+    lower_subtitle = subtitle.lower()
+    if "연간" in subtitle:
+        return "annual"
+    if "분기" in subtitle:
+        return "quarterly"
+    if "누적" in subtitle:
+        return "annual_cumulative"
+    if "3개월" in subtitle:
+        return "quarter_3m"
+
+    period_columns = [str(column) for column in columns if re.match(r"^\d{4}/\d{2}$", str(column))]
+    years = [column.split("/")[0] for column in period_columns]
+    if len(set(years)) < len(years):
+        return "quarterly"
+    return "annual"
+
+
+def _parse_finance_table(
+    *,
+    title: str,
+    subtitle: str,
+    unit: str,
+    table_html: str,
+) -> dict[str, Any]:
+    caption_match = re.search(r"<caption[^>]*>(.*?)</caption>", table_html, re.S)
+    caption = _strip_tags(caption_match.group(1)) if caption_match else None
+
+    header_match = re.search(r"<thead>(.*?)</thead>", table_html, re.S)
+    header_cells = _CELL_RE.findall(header_match.group(1)) if header_match else []
+    header_texts = [_strip_tags(cell_html) for _, _, cell_html in header_cells]
+    basis = header_texts[0] if header_texts else None
+    columns = header_texts[1:]
+
+    body_match = re.search(r"<tbody>(.*?)</tbody>", table_html, re.S)
+    sections: list[dict[str, Any]] = []
+    current_section: dict[str, Any] | None = None
+    row_lookup: dict[str, dict[str, Any]] = {}
+
+    for row_attrs, row_html in _ROW_RE.findall(body_match.group(1) if body_match else ""):
+        row_class = _extract_attr(row_attrs, "class") or ""
+
+        if "tbody_tit" in row_class:
+            title_cells = _CELL_RE.findall(row_html)
+            section_title = _strip_tags(title_cells[0][2]) if title_cells else ""
+            current_section = {"title": section_title, "rows": []}
+            sections.append(current_section)
+            continue
+
+        if current_section is None:
+            current_section = {"title": None, "rows": []}
+            sections.append(current_section)
+
+        parsed_row = _parse_table_row(row_attrs, row_html, columns)
+
+        if "acd_dep2_sub" in row_class:
+            parent_keys = [
+                cls[len("c_") :]
+                for cls in row_class.split()
+                if cls.startswith("c_grid")
+            ]
+            parent = next((row_lookup[key] for key in parent_keys if key in row_lookup), None)
+            parsed_row.pop("_row_class", None)
+            parsed_row.pop("_row_id", None)
+            if parent is not None:
+                parent.setdefault("details", []).append(parsed_row)
+            else:
+                current_section["rows"].append(parsed_row)
+            continue
+
+        row_id = parsed_row.pop("_row_id", None)
+        parsed_row.pop("_row_class", None)
+        current_section["rows"].append(parsed_row)
+        key = row_id
+        if key and key.startswith("p_"):
+            key = key[2:]
+        if key:
+            row_lookup[key] = parsed_row
+
+    return {
+        "title": _normalize_text(title),
+        "subtitle": _normalize_text(subtitle),
+        "period_kind": _infer_table_period_kind(_normalize_text(subtitle), columns),
+        "unit": _normalize_text(unit),
+        "caption": caption,
+        "basis": basis,
+        "columns": columns,
+        "sections": sections,
+    }
+
+
+def parse_fnguide_html_tables(html: str) -> dict[str, Any]:
+    title_match = re.search(r"<title>(.*?)</title>", html, re.S)
+    page_title = _strip_tags(title_match.group(1)) if title_match else None
+
+    hidden_keys = ("gicode", "giname", "stkGb", "NewMenuID", "ReportGb", "MenuYn")
+    hidden_inputs = {
+        key: value
+        for key in hidden_keys
+        if (
+            value := _extract_attr(
+                re.search(rf'<input[^>]*id=["\']{key}["\'][^>]*', html).group(0),
+                "value",
+            )
+            if re.search(rf'<input[^>]*id=["\']{key}["\'][^>]*', html)
+            else None
+        )
+    }
+
+    tables = [
+        _parse_finance_table(
+            title=title,
+            subtitle=subtitle,
+            unit=unit,
+            table_html=table_html,
+        )
+        for title, subtitle, unit, table_html in _TABLE_SECTION_RE.findall(html)
+    ]
+
+    if not tables:
+        tables = [
+            _parse_finance_table(
+                title=f"table_{index}",
+                subtitle="",
+                unit="",
+                table_html=table_html,
+            )
+            for index, table_html in enumerate(_TABLE_RE.findall(html), start=1)
+        ]
+
+    return {
+        "page_title": page_title,
+        "page_meta": hidden_inputs,
+        "tables": tables,
+    }
+
+
+def parse_fnguide_finance_ratio_html(html: str) -> dict[str, Any]:
+    return parse_fnguide_html_tables(html)
+
+
+def parse_fnguide_finance_statement_html(html: str) -> dict[str, Any]:
+    return parse_fnguide_html_tables(html)
+
+
+def _normalize_stock_code(stock_code: str) -> str:
+    normalized = str(stock_code or "").strip()
+    if not normalized:
+        raise InvalidInputError("stock_code is required.")
+    return normalized
+
+
+def _normalize_statement_type(statement_type: str) -> tuple[str, str]:
+    normalized = str(statement_type or "").strip().lower()
+    if normalized in {"consolidated", "connected", "d"}:
+        return "consolidated", "D"
+    if normalized in {"separate", "standalone", "single", "b"}:
+        return "separate", "B"
+    raise InvalidInputError("statement_type must be 'consolidated' or 'separate'.")
+
+
+class FnGuideWorkflow:
+    """Fetch FnGuide ratio and financial statement pages as normalized JSON."""
+
+    def run(
+        self,
+        *,
+        stock_code: str,
+        statement_type: str = "consolidated",
+        timeout: float = 30.0,
+        user_agent: str | None = None,
+    ) -> dict[str, Any]:
+        normalized_code = _normalize_stock_code(stock_code)
+        normalized_statement_type, report_gb = _normalize_statement_type(statement_type)
+        gicode = normalized_code if normalized_code.startswith("A") else f"A{normalized_code}"
+
+        ratio_html = fetch_fnguide_page(
+            url=FN_GUIDE_FINANCE_RATIO_URL,
+            params=build_finance_ratio_params(gicode=gicode, report_gb=report_gb),
+            timeout=timeout,
+            user_agent=user_agent,
+        )
+        statement_html = fetch_fnguide_page(
+            url=FN_GUIDE_FINANCE_STATEMENT_URL,
+            params=build_finance_statement_params(gicode=gicode, report_gb=report_gb),
+            timeout=timeout,
+            user_agent=user_agent,
+        )
+
+        return {
+            "input": {
+                "stock_code": normalized_code.removeprefix("A"),
+                "statement_type": normalized_statement_type,
+                "timeout": timeout,
+            },
+            "data": {
+                "financials": parse_fnguide_finance_statement_html(statement_html),
+                "ratios": parse_fnguide_finance_ratio_html(ratio_html),
+            },
+        }
+
+
+_DEFAULT_WORKFLOW = FnGuideWorkflow()
+
+
+def run_fnguide_workflow(
+    *,
+    stock_code: str,
+    statement_type: str = "consolidated",
+    timeout: float = 30.0,
+    user_agent: str | None = None,
+) -> dict[str, Any]:
+    return _DEFAULT_WORKFLOW.run(
+        stock_code=stock_code,
+        statement_type=statement_type,
+        timeout=timeout,
+        user_agent=user_agent,
+    )

orchestrator/mapping/__init__.py

@@ -0,0 +1,22 @@
+"""Mapping workflows for FinanceChatbot.
+
+For development:
+    run `orchestrator-update-corp-codes --api-key "$DART_API_KEY"` to update the raw corp codes file
+"""
+
+from .workflow import MappingWorkflow, download_latest_raw_entries, run_mapping_workflow
+
+__all__ = [
+    "MappingWorkflow",
+    "download_latest_raw_entries",
+    "run_mapping_workflow",
+    "update_raw_corp_codes_file",
+]
+
+
+def __getattr__(name: str):
+    if name == "update_raw_corp_codes_file":
+        from .update_raw_data import update_raw_corp_codes_file
+
+        return update_raw_corp_codes_file
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

orchestrator/mapping/data/__init__.py

@@ -0,0 +1 @@
+"""Bundled corp-code data for mapping workflows."""