jfinqa-helm 0.1.0__tar.gz

jfinqa_helm-0.1.0/.gitignore

@@ -0,0 +1,9 @@
+__pycache__/
+*.pyc
+*.pyo
+dist/
+build/
+*.egg-info/
+.venv/
+.env
+.env.*

jfinqa_helm-0.1.0/PKG-INFO

@@ -0,0 +1,50 @@
+Metadata-Version: 2.4
+Name: jfinqa-helm
+Version: 0.1.0
+Summary: HELM plugin for JFinQA: Japanese Financial Numerical Reasoning QA Benchmark
+Project-URL: Homepage, https://github.com/ajtgjmdjp/jfinqa
+Project-URL: Dataset, https://huggingface.co/datasets/ajtgjmdjp/jfinqa
+Author: ajtgjmdjp
+License: Apache-2.0
+Keywords: benchmark,finance,helm,japanese,nlp
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.9
+Requires-Dist: crfm-helm>=0.5.0
+Requires-Dist: datasets>=2.0
+Description-Content-Type: text/markdown
+
+# jfinqa-helm
+
+HELM plugin for [JFinQA](https://github.com/ajtgjmdjp/jfinqa): Japanese Financial Numerical Reasoning QA Benchmark.
+
+## Installation
+
+```bash
+pip install crfm-helm jfinqa-helm
+```
+
+## Usage
+
+```bash
+helm-run --run-entries jfinqa:model=openai/gpt-4o
+```
+
+## About JFinQA
+
+JFinQA contains 1,000 questions across three subtasks:
+
+- **numerical_reasoning** (550 questions): Calculate financial ratios, growth rates, etc.
+- **consistency_checking** (200 questions): Verify whether a statement is consistent with financial data
+- **temporal_reasoning** (250 questions): Reason about changes over multiple fiscal years
+
+Questions are drawn from 68 companies' EDINET filings covering J-GAAP, IFRS, and US-GAAP.
+
+**Dataset**: [ajtgjmdjp/jfinqa](https://huggingface.co/datasets/ajtgjmdjp/jfinqa)
+
+## License
+
+Apache-2.0

jfinqa_helm-0.1.0/README.md

@@ -0,0 +1,31 @@
+# jfinqa-helm
+
+HELM plugin for [JFinQA](https://github.com/ajtgjmdjp/jfinqa): Japanese Financial Numerical Reasoning QA Benchmark.
+
+## Installation
+
+```bash
+pip install crfm-helm jfinqa-helm
+```
+
+## Usage
+
+```bash
+helm-run --run-entries jfinqa:model=openai/gpt-4o
+```
+
+## About JFinQA
+
+JFinQA contains 1,000 questions across three subtasks:
+
+- **numerical_reasoning** (550 questions): Calculate financial ratios, growth rates, etc.
+- **consistency_checking** (200 questions): Verify whether a statement is consistent with financial data
+- **temporal_reasoning** (250 questions): Reason about changes over multiple fiscal years
+
+Questions are drawn from 68 companies' EDINET filings covering J-GAAP, IFRS, and US-GAAP.
+
+**Dataset**: [ajtgjmdjp/jfinqa](https://huggingface.co/datasets/ajtgjmdjp/jfinqa)
+
+## License
+
+Apache-2.0

jfinqa_helm-0.1.0/pyproject.toml

@@ -0,0 +1,37 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "jfinqa-helm"
+version = "0.1.0"
+description = "HELM plugin for JFinQA: Japanese Financial Numerical Reasoning QA Benchmark"
+readme = "README.md"
+license = { text = "Apache-2.0" }
+authors = [{ name = "ajtgjmdjp" }]
+requires-python = ">=3.9"
+dependencies = [
+    "crfm-helm>=0.5.0",
+    "datasets>=2.0",
+]
+keywords = ["helm", "benchmark", "japanese", "finance", "nlp"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[project.urls]
+Homepage = "https://github.com/ajtgjmdjp/jfinqa"
+Dataset = "https://huggingface.co/datasets/ajtgjmdjp/jfinqa"
+
+[project.entry-points.helm]
+jfinqa = "jfinqa_helm.run_specs"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/jfinqa_helm"]
+
+[tool.hatch.build.targets.sdist]
+exclude = [".env", ".env.*", ".claude/"]

jfinqa_helm-0.1.0/src/jfinqa_helm/__init__.py

@@ -0,0 +1 @@
+"""HELM plugin for JFinQA: Japanese Financial Numerical Reasoning QA Benchmark."""

jfinqa_helm-0.1.0/src/jfinqa_helm/run_specs.py

@@ -0,0 +1,30 @@
+"""Run spec for JFinQA HELM plugin."""
+
+from helm.benchmark.adaptation.common_adapter_specs import get_generation_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_basic_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("jfinqa")
+def get_jfinqa_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="jfinqa_helm.scenario.JFinQAScenario", args={}
+    )
+    adapter_spec = get_generation_adapter_spec(
+        instructions=(
+            "以下の財務データを読み、質問に正確な数値で答えてください。\n"
+            "Read the following financial data and answer the question with the exact numeric value.\n"
+        ),
+        input_noun=None,
+        output_noun="Answer",
+        max_tokens=50,
+    )
+    metric_specs = get_basic_metric_specs([])
+    return RunSpec(
+        name="jfinqa",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=["jfinqa"],
+    )

jfinqa_helm-0.1.0/src/jfinqa_helm/scenario.py

@@ -0,0 +1,97 @@
+"""JFinQA: Japanese Financial Numerical Reasoning QA Benchmark.
+
+Data source:
+https://huggingface.co/datasets/ajtgjmdjp/jfinqa
+
+JFinQA is a benchmark for numerical reasoning over Japanese corporate
+financial disclosures. It contains 1,000 questions across three subtasks
+—numerical reasoning (550), consistency checking (200), and temporal
+reasoning (250)—drawn from 68 companies' EDINET filings covering
+J-GAAP, IFRS, and US-GAAP.
+"""
+
+import os
+from typing import Any, Dict, List
+
+from datasets import load_dataset
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Instance,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class JFinQAScenario(Scenario):
+    """Japanese Financial Numerical Reasoning QA."""
+
+    name = "jfinqa"
+    description = (
+        "JFinQA: Japanese Financial Numerical Reasoning QA — "
+        "1,000 questions across numerical reasoning, consistency checking, "
+        "and temporal reasoning from 68 companies' EDINET filings."
+    )
+    tags = ["question_answering", "finance", "japanese"]
+
+    HF_DATASET_ID = "ajtgjmdjp/jfinqa"
+    SUBSETS = ("numerical_reasoning", "consistency_checking", "temporal_reasoning")
+
+    @staticmethod
+    def _format_table(headers: List[str], rows: List[List[str]]) -> str:
+        header_line = "| " + " | ".join(str(h) for h in headers) + " |"
+        sep_line = "| " + " | ".join("---" for _ in headers) + " |"
+        row_lines = ["| " + " | ".join(str(c) for c in row) + " |" for row in rows]
+        return "\n".join([header_line, sep_line, *row_lines])
+
+    @staticmethod
+    def _build_input(row: Dict[str, Any]) -> str:
+        parts: List[str] = []
+
+        pre_text = row.get("pre_text", [])
+        if pre_text:
+            parts.append("\n".join(pre_text))
+
+        headers = row.get("table_headers", [])
+        rows = row.get("table_rows", [])
+        if headers:
+            parts.append(JFinQAScenario._format_table(headers, rows))
+
+        post_text = row.get("post_text", [])
+        if post_text:
+            parts.append("\n".join(post_text))
+
+        question = row.get("question", "")
+        parts.append(f"Question: {question}")
+
+        return "\n\n".join(parts)
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        instances: List[Instance] = []
+        for subset in self.SUBSETS:
+            dataset = load_dataset(
+                self.HF_DATASET_ID,
+                subset,
+                split="test",
+                cache_dir=cache_dir,
+                trust_remote_code=True,
+            )
+            for row in dataset:
+                input_text = self._build_input(row)
+                answer = str(row["answer"])
+                instances.append(
+                    Instance(
+                        input=Input(text=input_text),
+                        references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                        split=TEST_SPLIT,
+                        id=str(row.get("id", "")),
+                    )
+                )
+        return instances