hermes-benchmark 0.0.5__tar.gz

hermes_benchmark-0.0.5/.github/workflows/docs.yml

@@ -0,0 +1,82 @@
+name: Documentation
+
+on:
+  pull_request:
+    paths:
+      - 'docs/**'
+      - 'scripts/**'
+      - 'src/**'
+      - 'tests/**'
+      - 'configs/**'
+      - 'README.md'
+      - 'pyproject.toml'
+      - 'mkdocs.yml'
+      - '.github/workflows/docs.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'docs/**'
+      - 'scripts/**'
+      - 'src/**'
+      - 'tests/**'
+      - 'configs/**'
+      - 'README.md'
+      - 'pyproject.toml'
+      - 'mkdocs.yml'
+      - '.github/workflows/docs.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: docs-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install project and documentation tools
+        run: python -m pip install '.[dev,docs]'
+      - name: Check generated references
+        run: python scripts/generate_docs_reference.py --check
+      - name: Validate documentation examples
+        run: python scripts/validate_docs.py
+      - name: Run software tests
+        run: python -m pytest -q --junitxml=output/documentation-validation/tests.xml
+      - name: Build with strict link and navigation checks
+        run: python -m mkdocs build --strict
+      - name: Keep validation evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: documentation-validation
+          path: output/documentation-validation/
+          if-no-files-found: ignore
+      - name: Upload Pages site
+        if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: site
+
+  deploy:
+    if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main'
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      pages: write
+      id-token: write
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Configure Pages
+        uses: actions/configure-pages@v5
+      - name: Deploy validated site
+        id: deployment
+        uses: actions/deploy-pages@v4

hermes_benchmark-0.0.5/.gitignore

@@ -0,0 +1,20 @@
+__pycache__/
+*.py[cod]
+.pytest_cache/
+.coverage
+htmlcov/
+.venv/
+.venv-*/
+venv/
+dist/
+build/
+*.egg-info/
+.DS_Store
+output/
+*.xml
+*.xml.bz2
+*.xml.gz
+!tests/fixtures/*.xml
+config.*.yaml
+.secrets/
+site/

hermes_benchmark-0.0.5/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2018 The Python Packaging Authority
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

hermes_benchmark-0.0.5/PKG-INFO

@@ -0,0 +1,137 @@
+Metadata-Version: 2.4
+Name: hermes_benchmark
+Version: 0.0.5
+Summary: Wikipedia based benchmark creation package for low-resource languages.
+Project-URL: Homepage, https://github.com/sasalya98/hermes-benchmark
+Project-URL: Documentation, https://sasalya98.github.io/hermes-benchmark/
+Project-URL: Repository, https://github.com/sasalya98/hermes-benchmark
+Project-URL: Issues, https://github.com/sasalya98/hermes-benchmark/issues
+Author-email: Doruk Demirci <ddemirci3606@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.9
+Requires-Dist: mwparserfromhell>=0.7
+Requires-Dist: mwxml>=0.3.8
+Requires-Dist: numpy>=1.24
+Requires-Dist: pyarrow
+Requires-Dist: pydantic>=2
+Requires-Dist: pyyaml>=6
+Requires-Dist: tqdm
+Requires-Dist: typer
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs<2,>=1.6; extra == 'docs'
+Provides-Extra: embeddings
+Requires-Dist: sentence-transformers; extra == 'embeddings'
+Provides-Extra: gemini
+Requires-Dist: google-genai; extra == 'gemini'
+Provides-Extra: huggingface
+Requires-Dist: datasets; extra == 'huggingface'
+Requires-Dist: huggingface-hub; extra == 'huggingface'
+Provides-Extra: vertex
+Requires-Dist: google-genai; extra == 'vertex'
+Description-Content-Type: text/markdown
+
+# Hermes Benchmark
+
+Hermes Benchmark generates Wikipedia-based NLP benchmark datasets through a
+Python API and the `hermes-data` command line. It supports eight task families,
+localized configuration templates, cached external services, and artifact
+validation.
+
+**Documentation:** [Documentation site](https://sasalya98.github.io/hermes-benchmark/) ·
+[Getting started](https://sasalya98.github.io/hermes-benchmark/getting-started/) ·
+[Configuration](https://sasalya98.github.io/hermes-benchmark/configuration/) ·
+[Tasks and outputs](https://sasalya98.github.io/hermes-benchmark/tasks/) ·
+[Python API](https://sasalya98.github.io/hermes-benchmark/python-api/) ·
+[Methodology and limitations](https://sasalya98.github.io/hermes-benchmark/methodology/)
+
+## Installation
+
+Python 3.9 or newer is required by the package metadata. Use a maintained Python
+release for a new environment; optional model/provider dependencies may impose
+additional version requirements.
+
+Install from PyPI:
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+python -m pip install --upgrade pip
+python -m pip install "hermes-benchmark[embeddings,gemini]"
+```
+
+For source installation, wheel installation, and optional dependencies, see the
+[installation guide](https://sasalya98.github.io/hermes-benchmark/installation/).
+
+## Quick start
+
+Place an Azerbaijani `pages-articles` XML dump in the working directory, then run:
+
+```bash
+hermes-data languages
+hermes-data init-config --language az --output config.az.yaml
+hermes-data info --config config.az.yaml
+hermes-data pilot --config config.az.yaml
+hermes-data generate --config config.az.yaml
+hermes-data validate --config config.az.yaml
+```
+
+Before generation, edit the dump path, output directory, and targets in the YAML
+file. The Azerbaijani starter expects
+`azwiki-latest-pages-articles-multistream.xml.bz2`. It requests **100 samples per
+output variant**, enables seven task families, and leaves classification disabled.
+NER and template QA still require Wikidata access or a complete local cache.
+Generation also requires the configured embedding model.
+
+For 1,000 examples per variant, use the reviewed
+[Azerbaijani configuration](https://sasalya98.github.io/hermes-benchmark/examples/az_1000.yaml). Gemini QA and a small Gemini
+classification test have separate examples in [the guide](https://sasalya98.github.io/hermes-benchmark/getting-started/).
+Reranking defaults to **20 titles: one gold title and 19 hard negatives**.
+
+## Supported tasks
+
+| Configuration task | Output variants | Services |
+|---|---|---|
+| `paragraph_coherence` | Four-way outlier selection | Embeddings |
+| `paragraph_ordering` | Four-way paragraph-order selection | Embeddings through shared enrichment |
+| `article_membership` | Same-article binary decision | Embeddings |
+| `next_paragraph` | Four-way continuation selection | Embeddings |
+| `reranking` | Title ranking with one known gold title | Embeddings |
+| `ner` | Extraction, type assignment, type verification | Wikidata |
+| `classification` | Label prediction, multiple choice, verification | Embeddings and Gemini/Vertex AI |
+| `extractive_qa` | Answer-span extraction | Wikidata; Gemini/Vertex AI in `hybrid` mode |
+
+A `target` of 1,000 produces 1,000 records **in each enabled variant**: NER, for
+example, produces three files with 1,000 records each.
+
+## Validation and research use
+
+`validate` checks file integrity, corpus alignment, sample structure/counts, QA
+spans, and contradictory QA annotations. It does **not** establish expert-level
+annotation accuracy, question uniqueness for every sample, or contamination-free
+splits. NER labels come from Wikipedia links and a configurable Wikidata ontology;
+classification labels may be model-generated. Read the
+[methodology](https://sasalya98.github.io/hermes-benchmark/methodology/) before reporting benchmark results.
+
+## Development and documentation
+
+```bash
+python -m pip install ".[dev,docs]"
+python scripts/generate_docs_reference.py --check
+python scripts/validate_docs.py
+python -m pytest -q
+python -m mkdocs build --strict
+python -m mkdocs serve
+```
+
+Documentation is maintained in `docs/`. The
+[publishing guide](https://sasalya98.github.io/hermes-benchmark/contributing/) describes PyPI releases, GitHub Pages deployment, and
+how to keep examples and references synchronized with the implementation.
+
+The software is [MIT-licensed](https://github.com/sasalya98/hermes-benchmark/blob/main/LICENSE). Wikipedia and Wikidata source material
+have their own [licensing and attribution requirements](https://sasalya98.github.io/hermes-benchmark/methodology/#licensing-and-attribution).

hermes_benchmark-0.0.5/README.md

@@ -0,0 +1,99 @@
+# Hermes Benchmark
+
+Hermes Benchmark generates Wikipedia-based NLP benchmark datasets through a
+Python API and the `hermes-data` command line. It supports eight task families,
+localized configuration templates, cached external services, and artifact
+validation.
+
+**Documentation:** [Documentation site](https://sasalya98.github.io/hermes-benchmark/) ·
+[Getting started](https://sasalya98.github.io/hermes-benchmark/getting-started/) ·
+[Configuration](https://sasalya98.github.io/hermes-benchmark/configuration/) ·
+[Tasks and outputs](https://sasalya98.github.io/hermes-benchmark/tasks/) ·
+[Python API](https://sasalya98.github.io/hermes-benchmark/python-api/) ·
+[Methodology and limitations](https://sasalya98.github.io/hermes-benchmark/methodology/)
+
+## Installation
+
+Python 3.9 or newer is required by the package metadata. Use a maintained Python
+release for a new environment; optional model/provider dependencies may impose
+additional version requirements.
+
+Install from PyPI:
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+python -m pip install --upgrade pip
+python -m pip install "hermes-benchmark[embeddings,gemini]"
+```
+
+For source installation, wheel installation, and optional dependencies, see the
+[installation guide](https://sasalya98.github.io/hermes-benchmark/installation/).
+
+## Quick start
+
+Place an Azerbaijani `pages-articles` XML dump in the working directory, then run:
+
+```bash
+hermes-data languages
+hermes-data init-config --language az --output config.az.yaml
+hermes-data info --config config.az.yaml
+hermes-data pilot --config config.az.yaml
+hermes-data generate --config config.az.yaml
+hermes-data validate --config config.az.yaml
+```
+
+Before generation, edit the dump path, output directory, and targets in the YAML
+file. The Azerbaijani starter expects
+`azwiki-latest-pages-articles-multistream.xml.bz2`. It requests **100 samples per
+output variant**, enables seven task families, and leaves classification disabled.
+NER and template QA still require Wikidata access or a complete local cache.
+Generation also requires the configured embedding model.
+
+For 1,000 examples per variant, use the reviewed
+[Azerbaijani configuration](https://sasalya98.github.io/hermes-benchmark/examples/az_1000.yaml). Gemini QA and a small Gemini
+classification test have separate examples in [the guide](https://sasalya98.github.io/hermes-benchmark/getting-started/).
+Reranking defaults to **20 titles: one gold title and 19 hard negatives**.
+
+## Supported tasks
+
+| Configuration task | Output variants | Services |
+|---|---|---|
+| `paragraph_coherence` | Four-way outlier selection | Embeddings |
+| `paragraph_ordering` | Four-way paragraph-order selection | Embeddings through shared enrichment |
+| `article_membership` | Same-article binary decision | Embeddings |
+| `next_paragraph` | Four-way continuation selection | Embeddings |
+| `reranking` | Title ranking with one known gold title | Embeddings |
+| `ner` | Extraction, type assignment, type verification | Wikidata |
+| `classification` | Label prediction, multiple choice, verification | Embeddings and Gemini/Vertex AI |
+| `extractive_qa` | Answer-span extraction | Wikidata; Gemini/Vertex AI in `hybrid` mode |
+
+A `target` of 1,000 produces 1,000 records **in each enabled variant**: NER, for
+example, produces three files with 1,000 records each.
+
+## Validation and research use
+
+`validate` checks file integrity, corpus alignment, sample structure/counts, QA
+spans, and contradictory QA annotations. It does **not** establish expert-level
+annotation accuracy, question uniqueness for every sample, or contamination-free
+splits. NER labels come from Wikipedia links and a configurable Wikidata ontology;
+classification labels may be model-generated. Read the
+[methodology](https://sasalya98.github.io/hermes-benchmark/methodology/) before reporting benchmark results.
+
+## Development and documentation
+
+```bash
+python -m pip install ".[dev,docs]"
+python scripts/generate_docs_reference.py --check
+python scripts/validate_docs.py
+python -m pytest -q
+python -m mkdocs build --strict
+python -m mkdocs serve
+```
+
+Documentation is maintained in `docs/`. The
+[publishing guide](https://sasalya98.github.io/hermes-benchmark/contributing/) describes PyPI releases, GitHub Pages deployment, and
+how to keep examples and references synchronized with the implementation.
+
+The software is [MIT-licensed](https://github.com/sasalya98/hermes-benchmark/blob/main/LICENSE). Wikipedia and Wikidata source material
+have their own [licensing and attribution requirements](https://sasalya98.github.io/hermes-benchmark/methodology/#licensing-and-attribution).

hermes_benchmark-0.0.5/configs/az_1000.yaml

@@ -0,0 +1,116 @@
+# Azərbaycan dili: LLM tələb etməyən yeddi tapşırıq, hər çıxış növünə 1000 nümunə.
+dump:
+  path: azwiki-latest-pages-articles-multistream.xml.bz2
+  language: az
+  dump_id: azwiki-latest
+seed: 42
+output_dir: output/azwiki-1000
+inspection:
+  sample_pages: 10000
+  pilot_articles: 100
+  pilot_max_pages: 10000
+corpus:
+  article_limit: 2000
+  max_pages: 100000
+  min_paragraphs: 4
+  max_paragraphs: 8
+  min_paragraph_chars: 150
+  max_paragraph_chars: 900
+  cleaning_mode: structured
+  excluded_title_suffixes: ["(dəqiqləşdirmə)"]
+  excluded_paragraph_prefixes: ["istinadlar", "həmçinin bax", "xarici keçidlər", "qeydlər"]
+  additional_excluded_link_prefixes: ["Şəkil"]
+tasks:
+  paragraph_coherence:
+    target: 1000
+    options:
+      instruction: >-
+        Məqalənin məna axınını pozan abzası seçin. Yalnız düzgün seçim
+        identifikatorunu yazın.
+  paragraph_ordering:
+    target: 1000
+    options:
+      instruction: >-
+        Qarışdırılmış abzasları məqalədəki təbii ardıcıllığa salan seçimi edin.
+        Yalnız düzgün seçim identifikatorunu yazın.
+  article_membership:
+    target: 1000
+    options:
+      instruction: >-
+        İki abzas eyni Wikipedia məqaləsinə aiddirsə yalnız True, əks halda
+        yalnız False yazın.
+  next_paragraph:
+    target: 1000
+    options:
+      instruction: >-
+        Başlığa və əvvəlki abzaslara əsasən düzgün növbəti abzası seçin.
+        Yalnız düzgün seçim identifikatorunu yazın.
+  reranking:
+    target: 1000
+    options:
+      candidate_count: 20
+      hard_negative_count: 19
+      instruction: >-
+        Namizəd Wikipedia başlıqlarını verilmiş abzası ehtiva etmə ehtimalına
+        görə ən yüksəkdən ən aşağıya sıralayın. Hər identifikatoru bir dəfə
+        işlədərək yalnız vergüllə ayrılmış namizəd identifikatorlarını yazın.
+  ner:
+    target: 1000
+    options:
+      instruction_type1: >-
+        Mətndəki adlı varlıqları tapın. Hər sətirdə entity_text<TAB>TYPE
+        formatında bir varlıq yazın; varlıq yoxdursa EMPTY yazın. Etibarlı növlər:
+        {ner_labels}.
+      instruction_type2: >-
+        Hər varlıq identifikatoruna bir NER növü təyin edin. Yalnız E1=TYPE,
+        E2=TYPE formatında cavab verin. Yalnız seçimlərdəki etibarlı növlərdən istifadə edin.
+      instruction_type3: >-
+        Təklif olunan varlıq/növ cütü düzgündürsə yalnız True, əks halda yalnız
+        False yazın.
+  extractive_qa:
+    target: 1000
+    options:
+      generation_method: template
+      question_template: >-
+        Mətnə görə {title} məqaləsində hansı ad və ya ifadə qeyd olunur?
+      question_without_title: >-
+        Mətnə görə hansı ad və ya ifadə qeyd olunur?
+      instruction: >-
+        Suala kontekstə əsasən cavab verin. Cavabı kontekstdən dəqiq mətn
+        parçası kimi çıxarın və izah vermədən yalnız cavab mətnini yazın.
+  classification:
+    enabled: false
+    target: 1000
+services:
+  embeddings:
+    provider: sentence_transformers
+    model: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+    batch_size: 128
+    local_files_only: true
+  knowledge_base:
+    provider: wikidata
+    # Editable NER ontology shared by NER and extractive QA.
+    ner_ontology_by_label:
+      PER: [Q5, Q15632617, Q12737077]
+      GROUP: [Q41710, Q6266, Q16334295, Q8436, Q164950]
+      FACILITY: [Q13226383, Q41176, Q1248784, Q55488, Q483110, Q12280]
+      ORG: [Q43229, Q4830453, Q7210356, Q327333, Q783794, Q31855, Q38723, Q3918, Q476028]
+      LOC: [Q515, Q6256, Q3957, Q486972, Q82794, Q23397, Q4022, Q8502, Q46831, Q14763, Q1549591, Q3624078, Q5107, Q23442]
+      WORK: [Q386724, Q17537576, Q571, Q11424, Q482994, Q7725634, Q2188189, Q13406463, Q732577, Q5398426, Q7366, Q7889, Q41298, Q11032]
+      EVENT: [Q1190554, Q1656682, Q198, Q40231, Q27020041, Q16510064, Q47566]
+      PRODUCT: [Q2424752, Q15401930, Q7397]
+      LANGUAGE: [Q315, Q33742, Q9143, Q33215, Q25295, Q33384]
+      LAW: [Q7748, Q820655, Q7755, Q131569]
+      AWARD: [Q618779, Q203535, Q208045, Q7191]
+      MEDICAL: [Q12136, Q2057971, Q3505845, Q169872, Q1441305, Q18123741]
+      TAXON: [Q16521, Q7432]
+      CHEMICAL: [Q11173, Q79529, Q11344, Q25276, Q407595]
+      CELESTIAL: [Q6999, Q634, Q318, Q523, Q728, Q3863, Q44559]
+      FOOD: [Q2095, Q40050, Q746549, Q19861951, Q25403900]
+      VEHICLE: [Q42889, Q11436, Q11446, Q1420, Q870, Q35676]
+    ner_labels: [PER, GROUP, FACILITY, ORG, LOC, WORK, EVENT, PRODUCT, LANGUAGE, LAW, AWARD, MEDICAL, TAXON, CHEMICAL, CELESTIAL, FOOD, VEHICLE]
+    ner_label_priority: [PER, GROUP, FACILITY, ORG, LANGUAGE, LAW, AWARD, MEDICAL, TAXON, CHEMICAL, CELESTIAL, FOOD, VEHICLE, LOC, WORK, EVENT, PRODUCT]
+    ontology_batch_size: 10
+    request_delay_seconds: 0.25
+    timeout_seconds: 60
+    max_retries: 3

hermes_benchmark-0.0.5/configs/az_1000_gemini.yaml

@@ -0,0 +1,123 @@
+# Azərbaycan dili: hər çıxış növünə 1000 nümunə; QA sualları Gemini ilə yazılır.
+dump:
+  path: azwiki-latest-pages-articles-multistream.xml.bz2
+  language: az
+  dump_id: azwiki-latest
+seed: 42
+output_dir: output/azwiki-1000
+inspection:
+  sample_pages: 10000
+  pilot_articles: 100
+  pilot_max_pages: 10000
+corpus:
+  article_limit: 2000
+  max_pages: 100000
+  min_paragraphs: 4
+  max_paragraphs: 8
+  min_paragraph_chars: 150
+  max_paragraph_chars: 900
+  cleaning_mode: structured
+  excluded_title_suffixes: ["(dəqiqləşdirmə)"]
+  excluded_paragraph_prefixes: ["istinadlar", "həmçinin bax", "xarici keçidlər", "qeydlər"]
+  additional_excluded_link_prefixes: ["Şəkil"]
+tasks:
+  paragraph_coherence:
+    target: 1000
+    options:
+      instruction: >-
+        Məqalənin məna axınını pozan abzası seçin. Yalnız düzgün seçim
+        identifikatorunu yazın.
+  paragraph_ordering:
+    target: 1000
+    options:
+      instruction: >-
+        Qarışdırılmış abzasları məqalədəki təbii ardıcıllığa salan seçimi edin.
+        Yalnız düzgün seçim identifikatorunu yazın.
+  article_membership:
+    target: 1000
+    options:
+      instruction: >-
+        İki abzas eyni Wikipedia məqaləsinə aiddirsə yalnız True, əks halda
+        yalnız False yazın.
+  next_paragraph:
+    target: 1000
+    options:
+      instruction: >-
+        Başlığa və əvvəlki abzaslara əsasən düzgün növbəti abzası seçin.
+        Yalnız düzgün seçim identifikatorunu yazın.
+  reranking:
+    target: 1000
+    options:
+      candidate_count: 20
+      hard_negative_count: 19
+      instruction: >-
+        Namizəd Wikipedia başlıqlarını verilmiş abzası ehtiva etmə ehtimalına
+        görə ən yüksəkdən ən aşağıya sıralayın. Hər identifikatoru bir dəfə
+        işlədərək yalnız vergüllə ayrılmış namizəd identifikatorlarını yazın.
+  ner:
+    target: 1000
+    options:
+      instruction_type1: >-
+        Mətndəki adlı varlıqları tapın. Hər sətirdə entity_text<TAB>TYPE
+        formatında bir varlıq yazın; varlıq yoxdursa EMPTY yazın. Etibarlı növlər:
+        {ner_labels}.
+      instruction_type2: >-
+        Hər varlıq identifikatoruna bir NER növü təyin edin. Yalnız E1=TYPE,
+        E2=TYPE formatında cavab verin. Yalnız seçimlərdəki etibarlı növlərdən istifadə edin.
+      instruction_type3: >-
+        Təklif olunan varlıq/növ cütü düzgündürsə yalnız True, əks halda yalnız
+        False yazın.
+  extractive_qa:
+    target: 1000
+    options:
+      generation_method: hybrid
+      question_template: >-
+        Mətnə görə {title} məqaləsində hansı ad və ya ifadə qeyd olunur?
+      question_without_title: >-
+        Mətnə görə hansı ad və ya ifadə qeyd olunur?
+      instruction: >-
+        Suala kontekstə əsasən cavab verin. Cavabı kontekstdən dəqiq mətn
+        parçası kimi çıxarın və izah vermədən yalnız cavab mətnini yazın.
+  classification:
+    enabled: false
+    target: 1000
+services:
+  llm:
+    provider: gemini
+    model: gemini-3.1-flash-lite
+    api_key: ""  # Paste your Gemini API key between the quotes.
+    api_version: v1beta
+    batch_size: 10
+    temperature: 0
+  embeddings:
+    provider: sentence_transformers
+    model: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+    batch_size: 128
+    local_files_only: true
+  knowledge_base:
+    provider: wikidata
+    # Editable NER ontology shared by NER and extractive QA.
+    ner_ontology_by_label:
+      PER: [Q5, Q15632617, Q12737077]
+      GROUP: [Q41710, Q6266, Q16334295, Q8436, Q164950]
+      FACILITY: [Q13226383, Q41176, Q1248784, Q55488, Q483110, Q12280]
+      ORG: [Q43229, Q4830453, Q7210356, Q327333, Q783794, Q31855, Q38723, Q3918, Q476028]
+      LOC: [Q515, Q6256, Q3957, Q486972, Q82794, Q23397, Q4022, Q8502, Q46831, Q14763, Q1549591, Q3624078, Q5107, Q23442]
+      WORK: [Q386724, Q17537576, Q571, Q11424, Q482994, Q7725634, Q2188189, Q13406463, Q732577, Q5398426, Q7366, Q7889, Q41298, Q11032]
+      EVENT: [Q1190554, Q1656682, Q198, Q40231, Q27020041, Q16510064, Q47566]
+      PRODUCT: [Q2424752, Q15401930, Q7397]
+      LANGUAGE: [Q315, Q33742, Q9143, Q33215, Q25295, Q33384]
+      LAW: [Q7748, Q820655, Q7755, Q131569]
+      AWARD: [Q618779, Q203535, Q208045, Q7191]
+      MEDICAL: [Q12136, Q2057971, Q3505845, Q169872, Q1441305, Q18123741]
+      TAXON: [Q16521, Q7432]
+      CHEMICAL: [Q11173, Q79529, Q11344, Q25276, Q407595]
+      CELESTIAL: [Q6999, Q634, Q318, Q523, Q728, Q3863, Q44559]
+      FOOD: [Q2095, Q40050, Q746549, Q19861951, Q25403900]
+      VEHICLE: [Q42889, Q11436, Q11446, Q1420, Q870, Q35676]
+    ner_labels: [PER, GROUP, FACILITY, ORG, LOC, WORK, EVENT, PRODUCT, LANGUAGE, LAW, AWARD, MEDICAL, TAXON, CHEMICAL, CELESTIAL, FOOD, VEHICLE]
+    ner_label_priority: [PER, GROUP, FACILITY, ORG, LANGUAGE, LAW, AWARD, MEDICAL, TAXON, CHEMICAL, CELESTIAL, FOOD, VEHICLE, LOC, WORK, EVENT, PRODUCT]
+    ontology_batch_size: 10
+    request_delay_seconds: 0.25
+    timeout_seconds: 60
+    max_retries: 3

hermes_benchmark-0.0.5/configs/az_classification_smoke.yaml

@@ -0,0 +1,62 @@
+# Small live Gemini classification test; existing azwiki corpus is reused.
+dump:
+  path: azwiki-latest-pages-articles-multistream.xml.bz2
+  language: az
+  dump_id: azwiki-latest
+seed: 42
+output_dir: output/azwiki-classification-smoke
+corpus:
+  article_limit: 2000
+  max_pages: 100000
+  min_paragraphs: 4
+  max_paragraphs: 8
+  min_paragraph_chars: 150
+  max_paragraph_chars: 900
+  cleaning_mode: structured
+  excluded_title_suffixes:
+  - (dəqiqləşdirmə)
+  excluded_paragraph_prefixes:
+  - istinadlar
+  - həmçinin bax
+  - xarici keçidlər
+  - qeydlər
+  additional_excluded_link_prefixes:
+  - Şəkil
+tasks:
+  classification:
+    target: 8
+    options:
+      labels:
+      - Coğrafiya
+      - Mədəniyyət
+      - Elm və texnologiya
+      - Digər
+      candidate_total: 16
+      cluster_count: 22
+      llm_batch_size: 8
+      manual_review_total: 8
+      instruction_type1: Mətn üçün ən uyğun mövzu kateqoriyasını seçin. Yalnız kateqoriya
+        adını yazın.
+      instruction_type2: Mətn üçün düzgün mövzu kateqoriyasını seçimlərdən seçin.
+        Yalnız seçim identifikatorunu yazın.
+      instruction_type3: Mətn və kateqoriya uyğun gəlirsə True, əks halda False yazın.
+services:
+  llm:
+    provider: gemini
+    model: gemini-3.1-flash-lite
+    batch_size: 8
+    temperature: 0.0
+    project: null
+    location: us-central1
+    cache_path: null
+    api_key_file: .secrets/gemini_api_key.txt
+    timeout_seconds: 60
+    max_retries: 2
+    max_output_tokens: 4096
+    api_version: v1beta
+    api_key: ''
+  embeddings:
+    provider: sentence_transformers
+    model: sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+    batch_size: 128
+    local_files_only: true