lurebench 0.9.1__tar.gz

lurebench-0.9.1/.editorconfig

@@ -0,0 +1,18 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+
+[*.py]
+indent_size = 4
+max_line_length = 100
+
+[*.{yml,yaml,json,toml,cff}]
+indent_size = 2
+
+[*.md]
+trim_trailing_whitespace = false

lurebench-0.9.1/.env.example

@@ -0,0 +1,18 @@
+# LureBench provider keys — copy to `.env` and fill in. `.env` is gitignored.
+# Only fill the providers you have keys for; the rest can stay blank.
+# Load into a shell with:  set -a; source .env; set +a
+
+# DeepSeek — https://platform.deepseek.com
+DEEPSEEK_API_KEY=
+
+# Qwen (Alibaba DashScope) — https://dashscope.console.aliyun.com
+DASHSCOPE_API_KEY=
+
+# GLM (Zhipu / Z.ai) — https://open.bigmodel.cn  or  https://z.ai
+ZHIPUAI_API_KEY=
+
+# Kimi (Moonshot) — https://platform.moonshot.ai
+MOONSHOT_API_KEY=
+
+# Mistral — https://console.mistral.ai
+MISTRAL_API_KEY=

lurebench-0.9.1/.github/DISCUSSION_TEMPLATE/q-a.yml

@@ -0,0 +1,34 @@
+title: "[Question] "
+body:
+  - type: dropdown
+    id: area
+    attributes:
+      label: Area
+      options:
+        - Installation or CLI
+        - Dataset or licensing
+        - Evaluation methodology
+        - Adding a detector
+        - Reproducing a published result
+        - Other
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Version or commit
+      placeholder: v0.9.1 or a commit SHA
+    validations:
+      required: true
+  - type: textarea
+    id: question
+    attributes:
+      label: Question
+      description: Include the outcome you need and enough context to reproduce the situation.
+    validations:
+      required: true
+  - type: textarea
+    id: attempted
+    attributes:
+      label: What have you tried?
+      description: Redact provider keys, personal data, and live malicious infrastructure.

lurebench-0.9.1/.github/ISSUE_TEMPLATE/benchmark_submission.yml

@@ -0,0 +1,63 @@
+name: Benchmark submission
+description: Propose a detector or published result for reproducible evaluation.
+title: "[Benchmark]: "
+labels: [benchmark-submission]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Thanks for contributing a result. Do not include live credentials, working
+        malicious links, private messages, or data you are not allowed to share.
+  - type: input
+    id: detector
+    attributes:
+      label: Detector and version
+      placeholder: example-detector 1.2.0
+    validations:
+      required: true
+  - type: dropdown
+    id: task
+    attributes:
+      label: Evaluation task
+      options:
+        - Fraud detection
+        - AI-authorship provenance
+        - Adversarial robustness
+        - Multilingual generalization
+        - Calibration or risk control
+        - Other
+    validations:
+      required: true
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Reproduction command or code
+      description: Include the exact dataset version, split, detector configuration, and command.
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: results
+    attributes:
+      label: Results and uncertainty
+      description: Report denominators and confidence intervals or replicate variation where available.
+    validations:
+      required: true
+  - type: input
+    id: implementation
+    attributes:
+      label: Public implementation or paper
+      placeholder: https://...
+    validations:
+      required: true
+  - type: checkboxes
+    id: integrity
+    attributes:
+      label: Submission checks
+      options:
+        - label: I evaluated on the documented frozen split and did not tune on the test labels.
+          required: true
+        - label: I have disclosed provider/model versions, abstentions, failures, and paid API use.
+          required: true
+        - label: The submission contains no secrets, personal data, or actionable fraud infrastructure.
+          required: true

lurebench-0.9.1/.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,38 @@
+name: Bug report
+description: Something behaves differently from what the docs say
+labels: ["bug"]
+body:
+  - type: markdown
+    attributes:
+      value: |
+        For a **security vulnerability**, do not file here. See
+        [SECURITY.md](../blob/main/SECURITY.md) and email the maintainer.
+  - type: textarea
+    id: what
+    attributes:
+      label: What happened
+      description: What you expected, and what you got instead.
+    validations:
+      required: true
+  - type: textarea
+    id: repro
+    attributes:
+      label: How to reproduce
+      description: The exact command or code. A failing snippet beats a description.
+      render: shell
+    validations:
+      required: true
+  - type: input
+    id: version
+    attributes:
+      label: Version or commit
+      placeholder: "0.9.1, or a commit SHA"
+    validations:
+      required: true
+  - type: input
+    id: python
+    attributes:
+      label: Python version and OS
+      placeholder: "3.11 on macOS 14"
+    validations:
+      required: true

lurebench-0.9.1/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: true
+contact_links:
+  - name: Security vulnerability
+    url: https://github.com/immu4989/lurebench/security/advisories/new
+    about: Report privately through GitHub. Please do not open a public issue.
+  - name: Question or idea
+    url: https://github.com/immu4989/lurebench/discussions
+    about: Open a discussion for usage questions and proposals.

lurebench-0.9.1/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,20 @@
+## What this changes
+
+<!-- One or two sentences. What is different after this PR? -->
+
+## Why
+
+<!-- The problem being solved. Link an issue if there is one. -->
+
+## How it was verified
+
+<!-- Commands you ran, tests you added, numbers you observed. If this changes a
+     published result, say which one and by how much. -->
+
+## Checklist
+
+- [ ] `pytest` passes
+- [ ] `ruff check` passes
+- [ ] New behaviour has a test
+- [ ] Docs updated if behaviour or results changed
+- [ ] Any fraud samples are defanged (no real URLs, contacts, payment rails, or PII)

lurebench-0.9.1/.github/dependabot.yml

@@ -0,0 +1,25 @@
+version: 2
+updates:
+  - package-ecosystem: pip
+    directory: "/"
+    schedule:
+      interval: monthly
+    open-pull-requests-limit: 5
+    groups:
+      python-dependencies:
+        patterns: ["*"]
+    # The bundled classifier is serialized with scikit-learn 1.6.1. Minor/major
+    # deserializer upgrades require an explicit model-compatibility validation.
+    # Dependabot security updates are unaffected by version-update ignores.
+    ignore:
+      - dependency-name: scikit-learn
+        update-types:
+          - version-update:semver-minor
+          - version-update:semver-major
+  - package-ecosystem: github-actions
+    directory: "/"
+    schedule:
+      interval: monthly
+    groups:
+      github-actions:
+        patterns: ["*"]

lurebench-0.9.1/.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: ${{ matrix.python-version }}
+      - uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+      - name: Install
+        run: uv sync --frozen --python "${{ matrix.python-version }}" --extra dev --extra train
+      - name: Lint
+        run: uv run --frozen ruff check lurebench tests scripts
+      - name: Tests
+        run: uv run --frozen pytest -q
+      - name: Build distributions
+        if: matrix.python-version == '3.12'
+        run: uv build

lurebench-0.9.1/.github/workflows/codeql.yml

@@ -0,0 +1,37 @@
+name: CodeQL
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  schedule:
+    - cron: "17 4 * * 2"
+
+permissions:
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze (python)
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      packages: read
+      security-events: write
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
+        with:
+          languages: python
+          build-mode: none
+          queries: security-extended
+      - name: Analyze
+        uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
+        with:
+          category: /language:python

lurebench-0.9.1/.github/workflows/release.yml

@@ -0,0 +1,72 @@
+name: Release artifacts
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Build and attest Python distributions
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      id-token: write
+      attestations: write
+    steps:
+      - name: Checkout release source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: "3.12"
+      - name: Set up uv
+        uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0
+        with:
+          version: "0.12.3"
+      - name: Verify protected release source and metadata
+        run: |
+          git merge-base --is-ancestor "$GITHUB_SHA" origin/main
+          python scripts/verify_release.py "$GITHUB_REF_NAME"
+      - name: Build and verify distributions
+        run: |
+          uv build
+          uvx --from twine==7.0.0 twine check dist/*
+      - name: Attest distributions
+        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+        with:
+          subject-path: dist/*
+      - name: Store distributions
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: python-package-distributions
+          path: dist/
+          if-no-files-found: error
+      - name: Attach distributions to the GitHub release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh release upload "$GITHUB_REF_NAME" dist/* --clobber
+
+  publish-pypi:
+    name: Publish to PyPI with OIDC
+    if: ${{ vars.PYPI_PUBLISH == 'true' }}
+    needs: build
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/p/lurebench
+    permissions:
+      id-token: write
+    steps:
+      - name: Download distributions
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: python-package-distributions
+          path: dist/
+      - name: Publish distributions
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

lurebench-0.9.1/.github/workflows/scorecard.yml

@@ -0,0 +1,38 @@
+name: OpenSSF Scorecard
+
+on:
+  push:
+    branches: [main]
+  schedule:
+    - cron: "31 5 * * 6"
+
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      id-token: write
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - name: Run Scorecard
+        uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          publish_results: true
+      - name: Upload SARIF artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: scorecard-sarif
+          path: results.sarif
+          retention-days: 5
+      - name: Upload to code scanning
+        uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6
+        with:
+          sarif_file: results.sarif

lurebench-0.9.1/.gitignore

@@ -0,0 +1,29 @@
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+.venv/
+venv/
+env/
+.pytest_cache/
+.ruff_cache/
+.mypy_cache/
+.DS_Store
+*.log
+# Secrets — API keys never get committed.
+.env
+.env.*
+!.env.example
+*.key
+secrets/
+# Generated staging (review-pending lures), trained model artifacts, and large data.
+staging/
+models/
+data/full/
+data/restricted/
+*.parquet
+
+# Cached detector scores from scripts/llm_experiments.py (regenerable, large)
+.cache/

lurebench-0.9.1/.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: '\.md$'
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-added-large-files
+        args: ["--maxkb=2048"]
+      - id: check-merge-conflict
+      - id: detect-private-key
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.15.20
+    hooks:
+      - id: ruff
+        args: [--fix]

lurebench-0.9.1/.zenodo.json

@@ -0,0 +1,35 @@
+{
+  "title": "LureBench: A Benchmark and Evaluation Harness for Detecting AI-Generated Fraud Lures",
+  "description": "<p>LureBench is a Python benchmark and evaluation harness for detecting AI-generated fraud lures across phishing, business email compromise, romance and pig-butchering typologies. It ships a distribution-matched corpus in nine languages, detector baselines from keyword rules through a trained classifier to LLM-as-classifier judges, and an adversarial layer of character-level and LLM-driven evasion attacks. Detectors are scored on two tasks: <em>fraud</em> (is this a lure?) and <em>provenance</em> (did a machine write it?).</p><p>The project documents several measurement results, including a corpus confound that makes AI-vs-human fraud detection look trivially easy until the classes are distribution-matched, a defang artifact that inflates apparent multilingual recall, and a robustness ordering that inverts depending on whether the attack changes spelling or meaning.</p><p>Everything runs without model downloads or API keys; provider keys are needed only to generate new lures or to run LLM-backed detectors and attacks.</p>",
+  "upload_type": "software",
+  "license": "Apache-2.0",
+  "access_right": "open",
+  "creators": [
+    {
+      "name": "Ahamed, Imran",
+      "orcid": "0009-0002-7717-7480"
+    }
+  ],
+  "keywords": [
+    "fraud detection",
+    "phishing",
+    "business email compromise",
+    "machine-generated text detection",
+    "adversarial robustness",
+    "large language models",
+    "AI security",
+    "benchmark"
+  ],
+  "related_identifiers": [
+    {
+      "identifier": "https://github.com/immu4989/lurebench",
+      "relation": "isSupplementTo",
+      "scheme": "url"
+    },
+    {
+      "identifier": "https://github.com/immu4989/lurescope",
+      "relation": "isSourceOf",
+      "scheme": "url"
+    }
+  ]
+}