reasonwork 0.4.0__py3-none-any.whl

reasonwork_exchange/analyzer.py

@@ -0,0 +1,908 @@
+"""Streaming, content-safe checks for Codex and Claude Code JSONL sessions."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from collections import Counter
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any
+
+MAX_TRAJECTORY_BYTES = 100 * 1024 * 1024
+HARD_QUALITY_REJECTION_CODES = frozenset(
+    {"unrecognized_source", "invalid_json_lines", "too_few_messages", "no_agentic_tool_use"}
+)
+
+
+def _luhn_valid(candidate: str) -> bool:
+    """Checksum-validate a digit string shaped like a card number.
+
+    The credit_card pattern below only matches BIN prefix + length, which
+    also matches plenty of incidental digit runs (UUIDs, hashes, session
+    ids) that are not card numbers. Luhn rejects the vast majority of those.
+    """
+    digits = [int(d) for d in candidate if d.isdigit()]
+    if not (13 <= len(digits) <= 19):
+        return False
+    total = 0
+    for i, digit in enumerate(reversed(digits)):
+        if i % 2 == 1:
+            digit *= 2
+            if digit > 9:
+                digit -= 9
+        total += digit
+    return total % 10 == 0
+
+_SECRET_PATTERNS = {
+    # Matches the whole PEM block through its matching END marker so the base64 body
+    # isn't left in plaintext after redaction; falls back to the header alone if no
+    # END marker follows (e.g. a truncated key), which is why the alternative is kept.
+    "private_key": re.compile(
+        r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"
+        r"|-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"
+    ),
+    "aws_access_key": re.compile(r"\b(?:AKIA|ASIA)[A-Z0-9]{16}\b"),
+    "openai_key": re.compile(r"\bsk-(?!ant-)(?:proj-)?[A-Za-z0-9_-]{20,}\b"),
+    "anthropic_key": re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}\b"),
+    "github_token": re.compile(r"\bgh[opusr]_[A-Za-z0-9]{20,}\b"),
+    "bearer_token": re.compile(r"\bBearer\s+[A-Za-z0-9._~-]{24,}\b", re.IGNORECASE),
+    "slack_token": re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b"),
+    "slack_webhook": re.compile(r"\bhooks\.slack\.com/services/[A-Za-z0-9/]+\b"),
+    "stripe_key": re.compile(r"\b(?:sk|rk)_live_[A-Za-z0-9]{20,}\b"),
+    "google_api_key": re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b"),
+    "npm_token": re.compile(r"\bnpm_[A-Za-z0-9]{36}\b"),
+    "db_connection_password": re.compile(
+        r"\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqp)://[^\s\"'/:@]+:[^\s\"'/@]+@[^\s\"']+",
+        re.IGNORECASE,
+    ),
+    "jwt": re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
+    "ssn": re.compile(r"\b\d{3}-\d{2}-\d{4}\b"),
+    "credit_card": re.compile(
+        r"\b(?:4[0-9]{3}|5[1-5][0-9]{2}|3[47][0-9]{2}|6011)[- ]?[0-9]{4}[- ]?[0-9]{4}[- ]?[0-9]{1,4}\b"
+    ),
+    "twilio_api_key": re.compile(r"\bSK[a-f0-9]{32}\b"),
+    "twilio_account_sid": re.compile(r"\bAC[a-f0-9]{32}\b"),
+    "sendgrid_key": re.compile(r"\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b"),
+    "mailgun_key": re.compile(r"\bkey-[a-f0-9]{32}\b"),
+    "digitalocean_token": re.compile(r"\bdop_v1_[a-f0-9]{64}\b"),
+    "azure_storage_key": re.compile(r"\bAccountKey=[A-Za-z0-9+/=]{60,}"),
+    "generic_api_secret": re.compile(
+        r"\b(?:api[_-]?key|api[_-]?secret|access[_-]?token|client[_-]?secret|secret[_-]?key)"
+        r"\s*[=:]\s*\\?[\"']?[A-Za-z0-9_\-.]{16,}\\?[\"']?",
+        re.IGNORECASE,
+    ),
+    "generic_password_assignment": re.compile(
+        r"\b(?:password|passwd|pwd)\s*[=:]\s*\\?[\"'][^\"'\s]{6,}\\?[\"']", re.IGNORECASE
+    ),
+    # --- Fixed-prefix service tokens (documented public formats) ---
+    "github_fine_grained_pat": re.compile(r"\bgithub_pat_[A-Za-z0-9_]{20,}\b"),
+    "gitlab_personal_access_token": re.compile(r"\bglpat-[A-Za-z0-9_-]{20}\b"),
+    "gitlab_pipeline_trigger_token": re.compile(r"\bglptt-[A-Za-z0-9]{20,}\b"),
+    "gitlab_deploy_token": re.compile(r"\bgldt-[A-Za-z0-9_-]{20,}\b"),
+    "gitlab_runner_registration_token": re.compile(r"\bGR1348941[A-Za-z0-9_-]{20,}\b"),
+    "dockerhub_personal_access_token": re.compile(r"\bdckr_pat_[A-Za-z0-9_-]{20,}\b"),
+    "vault_service_token": re.compile(r"\bhvs\.[A-Za-z0-9_-]{24,}\b"),
+    "vault_batch_token": re.compile(r"\bhvb\.[A-Za-z0-9_-]{24,}\b"),
+    "digitalocean_oauth_token": re.compile(r"\bdoo_v1_[a-f0-9]{64}\b"),
+    "digitalocean_refresh_token": re.compile(r"\bdor_v1_[a-f0-9]{64}\b"),
+    "pypi_upload_token": re.compile(r"\bpypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{50,}\b"),
+    "mailchimp_api_key": re.compile(r"\b[a-f0-9]{32}-us\d{1,2}\b"),
+    "facebook_access_token": re.compile(r"\bEAA[A-Za-z0-9]{20,}\b"),
+    "telegram_bot_token": re.compile(r"\b\d{8,10}:[A-Za-z0-9_-]{35}\b"),
+    "discord_bot_token": re.compile(r"\b[MN][A-Za-z0-9_-]{23,25}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27,38}\b"),
+    "shopify_admin_api_token": re.compile(r"\bshpat_[a-fA-F0-9]{32}\b"),
+    "shopify_shared_secret": re.compile(r"\bshpss_[a-fA-F0-9]{32}\b"),
+    "shopify_custom_app_token": re.compile(r"\bshpca_[a-fA-F0-9]{32}\b"),
+    "contentful_personal_access_token": re.compile(r"\bCFPAT-[A-Za-z0-9_-]{20,}\b"),
+    "square_access_token": re.compile(r"\bEAAA[A-Za-z0-9_-]{20,}\b"),
+    "square_oauth_secret": re.compile(r"\bsq0(?:csp|atp)-[A-Za-z0-9_-]{22,43}\b"),
+    "razorpay_key": re.compile(r"\brzp_(?:live|test)_[A-Za-z0-9]{14,}\b"),
+    "adyen_api_key": re.compile(r"\bAQE[A-Za-z0-9+/=_-]{20,}\b"),
+    "huggingface_token": re.compile(r"\bhf_[A-Za-z0-9]{34}\b"),
+    "replicate_token": re.compile(r"\br8_[A-Za-z0-9]{40}\b"),
+    "perplexity_api_key": re.compile(r"\bpplx-[A-Za-z0-9]{40,}\b"),
+    "groq_api_key": re.compile(r"\bgsk_[A-Za-z0-9]{52}\b"),
+    "fireworks_api_key": re.compile(r"\bfw_[A-Za-z0-9]{20,}\b"),
+    "cerebras_api_key": re.compile(r"\bcsk-[A-Za-z0-9]{20,}\b"),
+    "openrouter_api_key": re.compile(r"\bsk-or-v1-[a-f0-9]{40,}\b"),
+    "xai_api_key": re.compile(r"\bxai-[A-Za-z0-9]{20,}\b"),
+    "new_relic_user_api_key": re.compile(r"\bNRAK-[A-Z0-9]{27}\b"),
+    "sentry_dsn": re.compile(r"https://[a-f0-9]{32}@[a-z0-9.-]+\.ingest(?:\.us)?\.sentry\.io/\d+"),
+    "launchdarkly_mobile_key": re.compile(r"\bmob-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b"),
+    "hubspot_private_app_token": re.compile(r"\bpat-(?:na1|eu1)-[a-f0-9-]{36}\b"),
+    "slack_app_level_token": re.compile(r"\bxapp-1-[A-Za-z0-9-]{20,}\b"),
+    "slack_config_token": re.compile(r"\bxoxe\.xox[a-z]-1-[A-Za-z0-9-]{20,}\b"),
+    "netlify_personal_access_token": re.compile(r"\bnfp_[A-Za-z0-9]{40,}\b"),
+    "render_api_key": re.compile(r"\brnd_[A-Za-z0-9]{20,}\b"),
+    "planetscale_service_token": re.compile(r"\bpscale_tkn_[A-Za-z0-9]{20,}\b"),
+    "planetscale_oauth_token": re.compile(r"\bpscale_oauth_[A-Za-z0-9]{20,}\b"),
+    "figma_personal_access_token": re.compile(r"\bfigd_[A-Za-z0-9_-]{20,}\b"),
+    "linear_api_key": re.compile(r"\blin_api_[A-Za-z0-9]{20,}\b"),
+    "notion_api_key": re.compile(r"\bntn_[A-Za-z0-9]{20,}\b"),
+    "notion_api_key_legacy": re.compile(r"\bsecret_[A-Za-z0-9]{43}\b"),
+    "airtable_personal_access_token": re.compile(r"\bpat[A-Za-z0-9]{14}\.[a-f0-9]{64}\b"),
+    "postman_api_key": re.compile(r"\bPMAK-[a-f0-9]{24}-[a-f0-9]{34}\b"),
+    "sourcegraph_access_token": re.compile(r"\bsgp_[A-Za-z0-9_]{20,}\b"),
+    "grafana_service_account_token": re.compile(r"\bglsa_[A-Za-z0-9_]{32,}\b"),
+    "cloudinary_url": re.compile(r"\bcloudinary://[A-Za-z0-9]+:[A-Za-z0-9]+@[A-Za-z0-9_-]+\b"),
+    "amazon_mws_auth_token": re.compile(r"\bamzn\.mws\.[0-9a-fA-F-]{36}\b"),
+    "alibaba_cloud_access_key_id": re.compile(r"\bLTAI[A-Za-z0-9]{12,20}\b"),
+    "tencent_cloud_secret_id": re.compile(r"\bAKID[A-Za-z0-9]{32,36}\b"),
+    "firebase_cloud_messaging_legacy_key": re.compile(r"\bAAAA[A-Za-z0-9_-]{7}:[A-Za-z0-9_-]{140,}\b"),
+    "mapbox_secret_token": re.compile(r"\bsk\.[A-Za-z0-9]{60,}\.[A-Za-z0-9_-]{20,}\b"),
+    "mapbox_public_token": re.compile(r"\bpk\.[A-Za-z0-9]{60,}\.[A-Za-z0-9_-]{20,}\b"),
+    # --- Context-gated secrets (no fixed prefix; require a nearby key name) ---
+    "algolia_admin_key_assignment": re.compile(
+        r"\balgolia[_-]?(?:admin)?[_-]?api[_-]?key\s*[=:]\s*[\"']?[A-Za-z0-9]{32}[\"']?", re.IGNORECASE
+    ),
+    "okta_api_token_assignment": re.compile(
+        r"\bokta[_-]?api[_-]?token\s*[=:]\s*[\"']?00[A-Za-z0-9_-]{40}[\"']?", re.IGNORECASE
+    ),
+    "auth0_client_secret_assignment": re.compile(
+        r"\bauth0[_-]?client[_-]?secret\s*[=:]\s*[\"']?[A-Za-z0-9_-]{32,}[\"']?", re.IGNORECASE
+    ),
+    "datadog_api_key_assignment": re.compile(
+        r"\bdd[_-]?api[_-]?key\s*[=:]\s*[\"']?[a-f0-9]{32}[\"']?", re.IGNORECASE
+    ),
+    "pagerduty_api_key_assignment": re.compile(
+        r"\bpagerduty[_-]?api[_-]?key\s*[=:]\s*[\"']?[A-Za-z0-9+_-]{20,}[\"']?", re.IGNORECASE
+    ),
+    "segment_write_key_assignment": re.compile(
+        r"\bsegment[_-]?write[_-]?key\s*[=:]\s*[\"']?[A-Za-z0-9]{20,}[\"']?", re.IGNORECASE
+    ),
+    "amplitude_api_key_assignment": re.compile(
+        r"\bamplitude[_-]?api[_-]?key\s*[=:]\s*[\"']?[a-f0-9]{32}[\"']?", re.IGNORECASE
+    ),
+    "cloudflare_api_token_assignment": re.compile(
+        r"\bcloudflare[_-]?(?:api)?[_-]?token\s*[=:]\s*[\"']?[A-Za-z0-9_-]{40}[\"']?", re.IGNORECASE
+    ),
+    "heroku_api_key_assignment": re.compile(
+        r"\bheroku[_-]?api[_-]?key\s*[=:]\s*[\"']?[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}[\"']?",
+        re.IGNORECASE,
+    ),
+    "jira_api_token_assignment": re.compile(
+        r"\bjira[_-]?api[_-]?token\s*[=:]\s*[\"']?[A-Za-z0-9]{24}[\"']?", re.IGNORECASE
+    ),
+    "zendesk_api_token_assignment": re.compile(
+        r"\bzendesk[_-]?api[_-]?token\s*[=:]\s*[\"']?[A-Za-z0-9]{20,}[\"']?", re.IGNORECASE
+    ),
+    # --- Structurally-standardized government/financial identifiers (PII) ---
+    "iban": re.compile(r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b"),
+    "uk_national_insurance_number": re.compile(r"\b[A-CEGHJ-PR-TW-Z]{1}[A-CEGHJ-NPR-TW-Z]{1}\d{6}[A-D]\b"),
+    "india_pan": re.compile(r"\b[A-Z]{5}\d{4}[A-Z]\b"),
+    "india_aadhaar": re.compile(r"\b\d{4}\s\d{4}\s\d{4}\b"),
+    "us_itin": re.compile(r"\b9\d{2}-\d{2}-\d{4}\b"),
+    "passport_mrz_line": re.compile(r"\bP<[A-Z<]{43}\b"),
+    "drivers_license_assignment": re.compile(
+        r"driver'?s?\s*licen[cs]e\s*(?:number|#|no\.?)?\s*[:#]?\s*[A-Z0-9]{6,20}", re.IGNORECASE
+    ),
+    "medical_record_number": re.compile(r"\bMRN\s*[:#]?\s*\d{6,10}\b", re.IGNORECASE),
+    "date_of_birth": re.compile(
+        r"\b(?:DOB|date of birth)\s*[:\-]?\s*(?:\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|\d{4}-\d{2}-\d{2})\b", re.IGNORECASE
+    ),
+    "us_bank_routing_number": re.compile(
+        r"\brouting\s*(?:number|no\.?|#)?\s*[:#]?\s*\d{9}\b", re.IGNORECASE
+    ),
+    "swift_bic_code": re.compile(
+        r"\bswift\s*(?:code)?\s*[:#]?\s*[A-Z]{6}[A-Z0-9]{2}(?:[A-Z0-9]{3})?\b", re.IGNORECASE
+    ),
+    "brazil_cpf": re.compile(r"\b\d{3}\.\d{3}\.\d{3}-\d{2}\b"),
+    "brazil_cnpj": re.compile(r"\b\d{2}\.\d{3}\.\d{3}/\d{4}-\d{2}\b"),
+    "south_africa_id_number": re.compile(r"\b\d{6}[01]\d{3}[01]\d{2}\b"),
+    "spain_dni_nie": re.compile(r"\b(?:[XYZ]\d{7}|\d{8})[A-Za-z]\b"),
+    "italy_codice_fiscale": re.compile(r"\b[A-Z]{6}\d{2}[A-EHLMPRST]\d{2}[A-Z]\d{3}[A-Z]\b"),
+    "sweden_personnummer": re.compile(r"\b\d{6}[-+]\d{4}\b"),
+    "singapore_nric_fin": re.compile(r"\b[STFG]\d{7}[A-Z]\b"),
+    "south_korea_resident_registration_number": re.compile(r"\b\d{6}-[1-4]\d{6}\b"),
+    "mexico_curp": re.compile(r"\b[A-Z]{4}\d{6}[HM][A-Z]{5}[A-Z0-9]\d\b"),
+    "russia_snils": re.compile(r"\b\d{3}-\d{3}-\d{3}\s\d{2}\b"),
+    "france_insee_number": re.compile(r"\b[12]\d{2}(?:0[1-9]|1[0-2])\d{2}\d{3}\d{3}\d{2}\b"),
+    "netherlands_bsn_assignment": re.compile(r"\bBSN\s*[:#]?\s*\d{8,9}\b", re.IGNORECASE),
+    "australia_tfn_assignment": re.compile(r"\b(?:TFN|tax file number)\s*[:#]?\s*\d{8,9}\b", re.IGNORECASE),
+    "germany_tax_id_assignment": re.compile(r"\b(?:steuer-?id|tax\s*id)\s*[:#]?\s*\d{11}\b", re.IGNORECASE),
+    "japan_my_number_assignment": re.compile(r"\bmy\s*number\s*[:#]?\s*\d{12}\b", re.IGNORECASE),
+    "passport_number_assignment": re.compile(
+        r"\bpassport\s*(?:number|no\.?|#)?\s*[:#]?\s*[A-Z0-9]{6,9}\b", re.IGNORECASE
+    ),
+    # --- Generic secret shapes (not tied to a specific vendor) ---
+    "generic_basic_auth_url": re.compile(r"\b[a-z][a-z0-9+.-]*://[^\s\"'/:@]+:[^\s\"'/@]+@[^\s\"']+"),
+    "generic_basic_auth_header": re.compile(r"Authorization:\s*Basic\s+[A-Za-z0-9+/=]{16,}", re.IGNORECASE),
+    "generic_authorization_header_token": re.compile(
+        r"Authorization:\s*(?:Token|ApiKey|Api-Key)\s+[A-Za-z0-9._~-]{16,}", re.IGNORECASE
+    ),
+    "generic_token_assignment": re.compile(
+        r"\b(?:auth|session|refresh|private|signing|webhook)[_-]?token"
+        r"\s*[=:]\s*\\?[\"']?[A-Za-z0-9_\-.]{16,}\\?[\"']?",
+        re.IGNORECASE,
+    ),
+    "generic_encryption_key_assignment": re.compile(
+        r"\b(?:encryption|master|signing)[_-]?(?:key|secret)"
+        r"\s*[=:]\s*\\?[\"']?[A-Za-z0-9_\-+/=]{16,}\\?[\"']?",
+        re.IGNORECASE,
+    ),
+    "odbc_jdbc_connection_string_password": re.compile(r"\bPassword\s*=\s*[^;\s\"']{4,};", re.IGNORECASE),
+    "generic_api_key_in_query_string": re.compile(
+        r"[?&](?:api_key|apikey|access_token|token|key)=[A-Za-z0-9_\-]{16,}", re.IGNORECASE
+    ),
+    "pgp_private_key_block": re.compile(
+        r"-----BEGIN PGP PRIVATE KEY BLOCK-----[\s\S]*?-----END PGP PRIVATE KEY BLOCK-----"
+        r"|-----BEGIN PGP PRIVATE KEY BLOCK-----"
+    ),
+    "encrypted_private_key_block": re.compile(
+        r"-----BEGIN ENCRYPTED PRIVATE KEY-----[\s\S]*?-----END ENCRYPTED PRIVATE KEY-----"
+        r"|-----BEGIN ENCRYPTED PRIVATE KEY-----"
+    ),
+    "putty_private_key": re.compile(r"PuTTY-User-Key-File-\d+"),
+    # --- Additional vendor secrets (documented formats, cf. TruffleHog detectors) ---
+    "stripe_webhook_secret": re.compile(r"\bwhsec_[A-Za-z0-9]{32,}\b"),
+    "stripe_test_key": re.compile(r"\b(?:sk|rk)_test_[A-Za-z0-9]{20,}\b"),
+    "databricks_token": re.compile(r"\bdapi[a-f0-9]{32}(?:-\d)?\b"),
+    "artifactory_api_key": re.compile(r"\bAKCp[A-Za-z0-9]{69,73}\b"),
+    "dropbox_short_lived_token": re.compile(r"\bsl\.[A-Za-z0-9_-]{60,}\b"),
+    "wakatime_api_key": re.compile(r"\bwaka_[a-f0-9-]{36}\b"),
+    "microsoft_teams_webhook": re.compile(
+        r"\bhttps://[a-z0-9.-]+\.webhook\.office\.com/webhookb2/[A-Za-z0-9@/_-]+\b"
+    ),
+    "gcp_service_account_json_marker": re.compile(
+        r"\\?\"type\\?\"\s*:\s*\\?\"service_account\\?\""
+    ),
+    "aws_secret_access_key_assignment": re.compile(
+        r"aws_secret_access_key\s*[=:]\s*\\?[\"']?[A-Za-z0-9/+=]{40}\\?[\"']?", re.IGNORECASE
+    ),
+    # --- Azure ---
+    "azure_shared_access_key": re.compile(r"SharedAccessKey\s*=\s*[A-Za-z0-9+/=]{20,}"),
+    "azure_service_bus_connection_string": re.compile(
+        r"Endpoint=sb://[^;\s\"']+;SharedAccessKeyName=[^;\s\"']+;SharedAccessKey=[A-Za-z0-9+/=]{20,}"
+    ),
+    "azure_sas_token": re.compile(
+        r"[?&]sv=\d{4}-\d{2}-\d{2}-?[^\s\"']*&[^\s\"']*sig=[A-Za-z0-9%]{20,}"
+    ),
+    "azure_openai_api_key_assignment": re.compile(
+        r"azure_openai_api_key\s*[=:]\s*\\?[\"']?[a-f0-9]{32}\\?[\"']?", re.IGNORECASE
+    ),
+    # --- Supabase ---
+    "supabase_service_role_key_assignment": re.compile(
+        r"SUPABASE_SERVICE_ROLE_KEY\s*[=:]\s*\\?[\"']?eyJ[A-Za-z0-9_-]{10,}", re.IGNORECASE
+    ),
+    "supabase_anon_key_assignment": re.compile(
+        r"SUPABASE_ANON_KEY\s*[=:]\s*\\?[\"']?eyJ[A-Za-z0-9_-]{10,}", re.IGNORECASE
+    ),
+    # --- More fixed-prefix service tokens ---
+    "fly_io_api_token": re.compile(r"\bfm(?:1[ab]?|2)_[A-Za-z0-9+/_=-]{20,}\b"),
+    "cloudflare_origin_ca_key": re.compile(r"\bv1\.0-[a-f0-9-]{60,}\b"),
+    "onepassword_service_account_token": re.compile(r"\bops_[A-Za-z0-9+/=_-]{40,}\b"),
+    "doppler_token": re.compile(r"\bdp\.(?:st|ct|pt|sa)\.[A-Za-z0-9]{40,50}\b"),
+    "sentry_internal_integration_token": re.compile(r"\bsntrys_[A-Za-z0-9_]{20,}\b"),
+    "supabase_personal_access_token": re.compile(r"\bsbp_[a-f0-9]{40}\b"),
+    "plaid_access_token": re.compile(r"\baccess-(?:sandbox|development|production)-[a-f0-9-]{36}\b"),
+    "langsmith_api_key": re.compile(r"\blsv2_(?:pt|sk)_[A-Za-z0-9]{20,}\b"),
+    "resend_api_key": re.compile(r"\bre_[A-Za-z0-9]{32,}\b"),
+    # --- More PII / national identifiers ---
+    "us_medicare_beneficiary_identifier": re.compile(r"\b[1-9][A-Z][A-Z0-9]\d[A-Z][A-Z0-9]\d[A-Z]{2}\d{2}\b"),
+    "dea_registration_number": re.compile(r"\b[ABFGHJKLMPRSTUX][A-Z]\d{7}\b"),
+    "uk_nhs_number": re.compile(r"\b\d{3}\s\d{3}\s\d{4}\b"),
+    "ireland_pps_number": re.compile(r"\b\d{7}[A-Z]{1,2}\b"),
+    "switzerland_ahv_number": re.compile(r"\b756\.\d{4}\.\d{4}\.\d{2}\b"),
+    "finland_henkilotunnus": re.compile(r"\b\d{6}[-+A]\d{3}[0-9A-Z]\b"),
+    "imei_number_assignment": re.compile(r"\bIMEI\s*[:#]?\s*\d{15}\b", re.IGNORECASE),
+    "npi_number_assignment": re.compile(r"\bNPI\s*[:#]?\s*\d{10}\b", re.IGNORECASE),
+    "nz_ird_number_assignment": re.compile(r"\bIRD\s*(?:number)?\s*[:#]?\s*\d{8,9}\b", re.IGNORECASE),
+    "bank_account_number_assignment": re.compile(r"\baccount\s*number\s*[:#]?\s*\d{8,17}\b", re.IGNORECASE),
+    "cvv_assignment": re.compile(r"\bCVV2?\s*[:#]?\s*\d{3,4}\b", re.IGNORECASE),
+    "diners_club_card": re.compile(r"\b3(?:0[0-5]|[68][0-9])[0-9]{11}\b"),
+    "jcb_card": re.compile(r"\b(?:2131|1800|35\d{2})\d{12}\b"),
+}
+_EMAIL_PATTERN = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
+_PHONE_PATTERN = re.compile(
+    r"\b(?:\+?1[-.\s])?\(?\d{3}\)?[-.\s]\d{3}[-.\s]\d{4}\b"
+    r"|\+[1-9]\d{0,3}(?:[-.\s]?\d{2,4}){2,4}\b"
+)
+_CODEX_TYPES = {"session_meta", "event_msg", "response_item", "turn_context"}
+_CLAUDE_TYPES = {"user", "assistant", "system", "file-history-snapshot", "queue-operation"}
+
+
+@dataclass(frozen=True)
+class Analysis:
+    """Metadata-only analysis result; never contains prompt or tool-output text."""
+
+    source: str | None
+    session_id: str | None
+    agent_id: str | None
+    content_sha256: str
+    byte_size: int
+    line_count: int
+    valid_json_lines: int
+    invalid_json_lines: int
+    event_types: dict[str, int]
+    message_count: int
+    tool_call_count: int
+    tool_result_count: int
+    secret_findings: dict[str, int]
+    email_count: int
+    phone_count: int
+    privacy_risk: str
+    base_score: int
+    anti_gaming_adjustment: float
+    score_flags: list[str]
+    deterministic_score: int
+    estimated_cents: int
+    rejection_codes: list[str]
+    # Per-component breakdown of base_score (tool_diversity, code_edit_presence,
+    # message_density, error_recovery, session_substance) -- kept alongside the
+    # total so a price can be explained/audited component-by-component instead
+    # of just as one opaque number. Defaulted so existing positional/keyword
+    # constructions of Analysis (tests, etc.) don't all need updating.
+    score_components: dict[str, float] = field(default_factory=dict)
+    # True when this file is itself a subagent/delegate transcript rather than
+    # a top-level session. For Claude Code and Cursor this mirrors discover()'s
+    # path-based guess (a file living under a subagents/ folder); for Codex,
+    # which writes subagent rollouts as ordinary sibling files with no path
+    # signal, this is the only source of truth -- derived from the session's
+    # own session_meta record (thread_source == "subagent").
+    is_subagent: bool = False
+    # Raw counters feeding the real scoring formula, which deliberately does
+    # NOT live in this published package anymore (see ui/contributor-ui/
+    # backend_workers/pricing.py). None of these individually reveal the
+    # formula's weights/thresholds -- they're just counts -- so it's safe for
+    # the CLI to compute and report them; only the worker turns them into an
+    # actual score/price.
+    distinct_tool_name_count: int = 0
+    edit_call_count: int = 0
+    text_char_count: int = 0
+    error_result_count: int = 0
+    recovered_error_count: int = 0
+    tool_signature_counts: dict[str, int] = field(default_factory=dict)
+    text_message_count: int = 0
+    duplicate_text_message_count: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Return the stable JSON representation used by CLI and worker."""
+        return asdict(self)
+
+
+def _nested_type(payload: dict[str, Any]) -> str | None:
+    value = payload.get("type")
+    return value if isinstance(value, str) else None
+
+
+def _session_id(record: dict[str, Any], source: str | None) -> str | None:
+    if source == "codex":
+        payload = record.get("payload")
+        if isinstance(payload, dict):
+            # A Codex subagent's own thread id is distinct from its parent's --
+            # unlike Claude Code, where a subagent transcript's records are
+            # tagged with the *same* sessionId as the parent conversation.
+            # Returning parent_thread_id here (when present) instead of the
+            # subagent's own id makes Codex subagents group under their parent
+            # the same way, via the shared client_session_id sent to the
+            # server -- no separate linking field needed downstream.
+            parent = payload.get("parent_thread_id")
+            if isinstance(parent, str) and parent:
+                return parent
+            for key in ("id", "session_id", "thread_id"):
+                value = payload.get(key)
+                if isinstance(value, str) and value:
+                    return value
+    for key in ("sessionId", "session_id", "uuid"):
+        value = record.get(key)
+        if isinstance(value, str) and value:
+            return value
+    return None
+
+
+def _codex_is_subagent(record: dict[str, Any]) -> bool:
+    """True if this record's payload marks the session as a Codex subagent/delegate thread."""
+    payload = record.get("payload")
+    return isinstance(payload, dict) and payload.get("thread_source") == "subagent"
+
+
+def _codex_agent_id(record: dict[str, Any]) -> str | None:
+    payload = record.get("payload")
+    if not isinstance(payload, dict):
+        return None
+    for key in ("agent_nickname", "agent_path", "agent_role"):
+        value = payload.get(key)
+        if isinstance(value, str) and value:
+            return value
+    return None
+
+
+_EDIT_TOOL_NAMES = {"Edit", "Write", "NotebookEdit", "MultiEdit", "apply_patch"}
+
+
+@dataclass
+class _ClaudeEvents:
+    """Per-record tallies feeding the base quality score and anti-gaming checks."""
+
+    tool_calls: int = 0
+    tool_results: int = 0
+    edit_calls: int = 0
+    tool_names: list[str] | None = None
+    tool_signatures: list[str] | None = None
+    error_results: int = 0
+    text_chars: int = 0
+    text_messages: list[str] | None = None
+
+
+def _claude_events(record: dict[str, Any]) -> _ClaudeEvents:
+    events = _ClaudeEvents(tool_names=[], tool_signatures=[], text_messages=[])
+    message = record.get("message")
+    if not isinstance(message, dict):
+        return events
+    content = message.get("content")
+    if not isinstance(content, list):
+        return events
+    for item in content:
+        if not isinstance(item, dict):
+            continue
+        item_type = item.get("type")
+        if item_type == "tool_use":
+            events.tool_calls += 1
+            name = item.get("name")
+            name = name if isinstance(name, str) else "(unknown)"
+            events.tool_names.append(name)
+            events.edit_calls += int(name in _EDIT_TOOL_NAMES)
+            # A cheap same-call-repeated-verbatim signature: tool name plus its
+            # serialized args, truncated so huge payloads don't blow up the tally.
+            try:
+                args_repr = json.dumps(item.get("input"), sort_keys=True, default=str)[:500]
+            except (TypeError, ValueError):
+                args_repr = str(item.get("input"))[:500]
+            events.tool_signatures.append(f"{name}:{args_repr}")
+        elif item_type == "tool_result":
+            events.tool_results += 1
+            events.error_results += int(bool(item.get("is_error")))
+        elif item_type == "text":
+            text = item.get("text")
+            if isinstance(text, str):
+                events.text_chars += len(text)
+                events.text_messages.append(text)
+    if isinstance(content, str):
+        events.text_chars += len(content)
+        events.text_messages.append(content)
+    return events
+
+
+def analyze(path: Path, discovered_source: str | None = None) -> Analysis:
+    """Analyze one JSONL session without retaining raw content in the result.
+
+    discovered_source is a hint from discover() (e.g. "cursor"), used only to
+    relabel the final source when the file's own content is schema-compatible
+    with another source (Cursor CLI transcripts are Claude-Code-compatible
+    JSONL) and would otherwise be misattributed. Content sniffing still drives
+    which parsing branch runs, since that reflects the file's real schema.
+    """
+    size = path.stat().st_size
+    if size <= 0:
+        raise ValueError("trajectory is empty")
+    if size > MAX_TRAJECTORY_BYTES:
+        raise ValueError("trajectory exceeds the 100 MB upload limit")
+
+    digest = hashlib.sha256()
+    event_types: Counter[str] = Counter()
+    secrets: Counter[str] = Counter()
+    valid = 0
+    invalid = 0
+    line_count = 0
+    messages = 0
+    tool_calls = 0
+    tool_results = 0
+    email_count = 0
+    phone_count = 0
+    source: str | None = None
+    session_id: str | None = None
+    agent_id: str | None = None
+    is_subagent = False
+    tool_names: set[str] = set()
+    edit_calls = 0
+    error_results = 0
+    recovered_errors = 0
+    pending_error = False
+    text_chars = 0
+    tool_signatures: Counter[str] = Counter()
+    text_messages: Counter[str] = Counter()
+
+    with path.open("rb") as handle:
+        for raw in handle:
+            digest.update(raw)
+            if not raw.strip():
+                continue
+            line_count += 1
+            text = raw.decode("utf-8", errors="replace")
+            for name, pattern in _SECRET_PATTERNS.items():
+                matches = pattern.findall(text)
+                if name == "credit_card":
+                    matches = [m for m in matches if _luhn_valid(m)]
+                secrets[name] += len(matches)
+            email_count += len(_EMAIL_PATTERN.findall(text))
+            phone_count += len(_PHONE_PATTERN.findall(text))
+            try:
+                record = json.loads(text)
+            except json.JSONDecodeError:
+                invalid += 1
+                continue
+            if not isinstance(record, dict):
+                invalid += 1
+                continue
+            valid += 1
+            event_type = record.get("type")
+            if not isinstance(event_type, str):
+                event_type = "(missing)"
+            event_types[event_type] += 1
+            if source is None:
+                if event_type in _CODEX_TYPES:
+                    source = "codex"
+                elif event_type in _CLAUDE_TYPES:
+                    source = "claude_code"
+            if session_id is None:
+                session_id = _session_id(record, source)
+            if agent_id is None:
+                value = record.get("agentId")
+                if isinstance(value, str) and value:
+                    agent_id = value
+                elif source == "codex":
+                    agent_id = _codex_agent_id(record)
+            if source == "codex" and not is_subagent:
+                is_subagent = _codex_is_subagent(record)
+
+            if source == "codex":
+                payload = record.get("payload")
+                if isinstance(payload, dict):
+                    payload_type = _nested_type(payload)
+                    if event_type == "response_item" and payload_type == "message":
+                        messages += 1
+                        message_text = _content_text(payload.get("content"))
+                        text_chars += len(message_text)
+                        if message_text:
+                            text_messages[message_text] += 1
+                    if event_type == "response_item" and payload_type in {
+                        "function_call",
+                        "custom_tool_call",
+                        "local_shell_call",
+                    }:
+                        tool_calls += 1
+                        name = payload.get("name")
+                        name = name if isinstance(name, str) else payload_type
+                        tool_names.add(name)
+                        edit_calls += int(name in _EDIT_TOOL_NAMES)
+                        try:
+                            args_repr = json.dumps(payload.get("arguments"), sort_keys=True, default=str)[:500]
+                        except (TypeError, ValueError):
+                            args_repr = str(payload.get("arguments"))[:500]
+                        tool_signatures[f"{name}:{args_repr}"] += 1
+                        if pending_error:
+                            recovered_errors += 1
+                            pending_error = False
+                    if event_type == "response_item" and payload_type in {
+                        "function_call_output",
+                        "custom_tool_call_output",
+                        "local_shell_call_output",
+                    }:
+                        tool_results += 1
+                        output = payload.get("output")
+                        is_error = isinstance(output, dict) and bool(output.get("error"))
+                        error_results += int(is_error)
+                        pending_error = pending_error or is_error
+            elif source == "claude_code":
+                messages += int(event_type in {"user", "assistant"})
+                events = _claude_events(record)
+                tool_calls += events.tool_calls
+                tool_results += events.tool_results
+                edit_calls += events.edit_calls
+                text_chars += events.text_chars
+                for msg_text in events.text_messages or []:
+                    text_messages[msg_text] += 1
+                tool_names.update(events.tool_names or [])
+                for sig in events.tool_signatures or []:
+                    tool_signatures[sig] += 1
+                    if pending_error:
+                        recovered_errors += 1
+                        pending_error = False
+                if events.error_results:
+                    error_results += events.error_results
+                    pending_error = True
+
+    secret_total = sum(secrets.values())
+    if secret_total:
+        privacy_risk = "blocked"
+    elif email_count >= 10 or phone_count >= 5:
+        privacy_risk = "high"
+    elif email_count or phone_count:
+        privacy_risk = "medium"
+    else:
+        privacy_risk = "low"
+    text_message_count = sum(text_messages.values())
+    duplicate_text_message_count = sum(count - 1 for count in text_messages.values() if count > 1)
+    # The real deterministic score/price/anti-gaming formula deliberately does
+    # NOT run here anymore -- see ui/contributor-ui/backend_workers/pricing.py.
+    # This package ships publicly (PyPI); that module never does. base_score/
+    # anti_gaming_adjustment/score_flags/deterministic_score/estimated_cents
+    # below are neutral placeholders, not an approximation -- the worker
+    # overwrites them with the real values after re-downloading and
+    # re-analyzing the actual uploaded content, which was always the only
+    # authoritative source of truth regardless of what this function reports.
+    base_score = 0
+    score_components: dict[str, float] = {}
+    adjustment = 1.0
+    score_flags: list[str] = []
+    score = 0
+    rejection_codes: list[str] = []
+    if source is None:
+        rejection_codes.append("unrecognized_source")
+    if invalid:
+        rejection_codes.append("invalid_json_lines")
+    if messages < 2:
+        rejection_codes.append("too_few_messages")
+    if tool_calls < 1:
+        rejection_codes.append("no_agentic_tool_use")
+    if secret_total:
+        rejection_codes.append("secret_detected")
+    if privacy_risk == "high":
+        rejection_codes.append("excess_personal_data")
+    elif privacy_risk == "medium":
+        rejection_codes.append("personal_data_detected")
+    estimated_cents = 0
+    reported_source = source
+    if discovered_source == "cursor" and source in (None, "claude_code"):
+        reported_source = "cursor"
+
+    return Analysis(
+        source=reported_source,
+        session_id=session_id,
+        agent_id=agent_id,
+        content_sha256=digest.hexdigest(),
+        byte_size=size,
+        line_count=line_count,
+        valid_json_lines=valid,
+        invalid_json_lines=invalid,
+        event_types=dict(sorted(event_types.items())),
+        message_count=messages,
+        tool_call_count=tool_calls,
+        tool_result_count=tool_results,
+        secret_findings={key: value for key, value in sorted(secrets.items()) if value},
+        email_count=email_count,
+        phone_count=phone_count,
+        privacy_risk=privacy_risk,
+        base_score=base_score,
+        anti_gaming_adjustment=adjustment,
+        score_flags=score_flags,
+        deterministic_score=score,
+        estimated_cents=estimated_cents,
+        rejection_codes=rejection_codes,
+        score_components=score_components,
+        is_subagent=is_subagent,
+        distinct_tool_name_count=len(tool_names),
+        edit_call_count=edit_calls,
+        text_char_count=text_chars,
+        error_result_count=error_results,
+        recovered_error_count=recovered_errors,
+        tool_signature_counts=dict(tool_signatures),
+        text_message_count=text_message_count,
+        duplicate_text_message_count=duplicate_text_message_count,
+    )
+
+
+def _content_text(value: object) -> str:
+    if isinstance(value, str):
+        return value
+    if not isinstance(value, list):
+        return ""
+    chunks: list[str] = []
+    for item in value:
+        if isinstance(item, str):
+            chunks.append(item)
+        elif isinstance(item, dict):
+            text = item.get("text")
+            if isinstance(text, str):
+                chunks.append(text)
+    return "\n".join(chunks)
+
+
+def _redact(text: str) -> str:
+    for pattern in _SECRET_PATTERNS.values():
+        text = pattern.sub("[REDACTED_SECRET]", text)
+    return _EMAIL_PATTERN.sub("[REDACTED_EMAIL]", text)
+
+
+@dataclass
+class RedactionResult:
+    """Redacted trajectory bytes plus a re-scan confirming no residual PII/secrets remain."""
+
+    redacted_bytes: bytes
+    content_sha256: str
+    byte_size: int
+    line_count: int
+    redacted_categories: list[str]
+    residual_secret_findings: dict[str, int]
+    residual_email_count: int
+    residual_phone_count: int
+
+    @property
+    def clean(self) -> bool:
+        """True once the re-scan of the redacted bytes finds nothing left to redact."""
+        return not self.residual_secret_findings and not self.residual_email_count and not self.residual_phone_count
+
+
+def redact_trajectory(path: Path) -> RedactionResult:
+    """Redact secrets/PII from a flagged trajectory, line by line, preserving JSON structure.
+
+    Substitutions run directly on each line's decoded text rather than parsing JSON: every
+    pattern here matches only within a JSON string's content (none can match a `"`), so an
+    in-place replacement can never break a line's JSON syntax — this avoids the reformatting
+    and key-reordering risk of a parse-and-reserialize approach.
+    """
+    digest = hashlib.sha256()
+    output = bytearray()
+    line_count = 0
+    categories: set[str] = set()
+    residual_secrets: Counter[str] = Counter()
+    residual_email = 0
+    residual_phone = 0
+
+    with path.open("rb") as handle:
+        for raw in handle:
+            if not raw.strip():
+                output.extend(raw)
+                digest.update(raw)
+                continue
+            line_count += 1
+            text = raw.decode("utf-8", errors="replace")
+            for name, pattern in _SECRET_PATTERNS.items():
+                if name == "credit_card":
+
+                    def _sub_card(match: re.Match[str]) -> str:
+                        if _luhn_valid(match.group(0)):
+                            categories.add("credit_card")
+                            return "[REDACTED_CREDIT_CARD]"
+                        return match.group(0)
+
+                    text = pattern.sub(_sub_card, text)
+                    continue
+                if pattern.search(text):
+                    categories.add(name)
+                text = pattern.sub(f"[REDACTED_{name.upper()}]", text)
+            if _EMAIL_PATTERN.search(text):
+                categories.add("email")
+            text = _EMAIL_PATTERN.sub("[REDACTED_EMAIL]", text)
+            if _PHONE_PATTERN.search(text):
+                categories.add("phone")
+            text = _PHONE_PATTERN.sub("[REDACTED_PHONE]", text)
+
+            encoded = text.encode("utf-8")
+            output.extend(encoded)
+            digest.update(encoded)
+
+            for name, pattern in _SECRET_PATTERNS.items():
+                matches = pattern.findall(text)
+                if name == "credit_card":
+                    matches = [m for m in matches if _luhn_valid(m)]
+                residual_secrets[name] += len(matches)
+            residual_email += len(_EMAIL_PATTERN.findall(text))
+            residual_phone += len(_PHONE_PATTERN.findall(text))
+
+    return RedactionResult(
+        redacted_bytes=bytes(output),
+        content_sha256=digest.hexdigest(),
+        byte_size=len(output),
+        line_count=line_count,
+        redacted_categories=sorted(categories),
+        residual_secret_findings={key: value for key, value in sorted(residual_secrets.items()) if value},
+        residual_email_count=residual_email,
+        residual_phone_count=residual_phone,
+    )
+
+
+def review_sample(path: Path, *, max_messages: int = 6, max_chars: int = 2400) -> list[dict[str, str]]:
+    """Extract a tiny redacted message sample for an optional semantic judge.
+
+    This output is intentionally separate from :class:`Analysis`; the CLI never
+    emits it and the worker only uses it after deterministic privacy screening.
+    """
+    sample: list[dict[str, str]] = []
+    used = 0
+    with path.open(encoding="utf-8", errors="replace") as handle:
+        for line in handle:
+            if len(sample) >= max_messages or used >= max_chars:
+                break
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(record, dict):
+                continue
+            role = ""
+            text = ""
+            if record.get("type") == "response_item" and isinstance(record.get("payload"), dict):
+                payload = record["payload"]
+                if payload.get("type") == "message":
+                    role = str(payload.get("role", ""))
+                    text = _content_text(payload.get("content"))
+            elif record.get("type") in {"user", "assistant"} and isinstance(record.get("message"), dict):
+                role = str(record["message"].get("role") or record.get("type"))
+                text = _content_text(record["message"].get("content"))
+            if role not in {"user", "assistant"} or not text.strip():
+                continue
+            redacted = _redact(text).strip()[:600]
+            remaining = max_chars - used
+            if remaining <= 0:
+                break
+            redacted = redacted[:remaining]
+            used += len(redacted)
+            sample.append({"role": role, "text": redacted})
+    return sample
+
+
+_SIMHASH_BITS = 64
+_TIMESTAMP_RE = re.compile(r"\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}(\.\d+)?z?", re.IGNORECASE)
+_UUID_RE = re.compile(r"\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b", re.IGNORECASE)
+
+
+def _shingles(text: str, k: int = 8) -> Counter[str]:
+    """8-character shingles of normalized text, weighted by frequency.
+
+    Normalization strips exactly the kind of per-run noise that would
+    otherwise make two genuinely-the-same conversations hash completely
+    differently -- timestamps and UUIDs, which are unique to every capture
+    even when the surrounding content is identical or near-identical.
+    """
+    normalized = _TIMESTAMP_RE.sub("", text.lower())
+    normalized = _UUID_RE.sub("", normalized)
+    normalized = re.sub(r"\s+", " ", normalized).strip()
+    return Counter(normalized[i : i + k] for i in range(max(0, len(normalized) - k + 1)))
+
+
+def content_simhash(path: Path) -> int:
+    """64-bit SimHash over a session's message text, for fuzzy near-duplicate detection.
+
+    Worker-only -- like review_sample, the CLI never calls this, since
+    comparing against every other contributor's content is meaningless
+    without server-side access to it.
+
+    Like Analysis, the output can't be reversed into readable content --
+    it's a lossy bit vector, not a copy of the text. Two sessions whose
+    message text is identical or near-identical (same conversation resubmitted
+    with edits scattered through it, unlike a byte-exact prefix which the
+    existing containment check already catches) produce fingerprints with a
+    small Hamming distance; unrelated sessions produce a distance close to
+    32 (half the bits, by construction of SimHash).
+    """
+    shingle_weights: Counter[str] = Counter()
+    with path.open(encoding="utf-8", errors="replace") as handle:
+        for line in handle:
+            try:
+                record = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(record, dict):
+                continue
+            text = ""
+            if record.get("type") == "response_item" and isinstance(record.get("payload"), dict):
+                payload = record["payload"]
+                if payload.get("type") == "message":
+                    text = _content_text(payload.get("content"))
+            elif record.get("type") in {"user", "assistant"} and isinstance(record.get("message"), dict):
+                text = _content_text(record["message"].get("content"))
+            if text.strip():
+                shingle_weights.update(_shingles(text))
+
+    if not shingle_weights:
+        return 0
+
+    bit_totals = [0] * _SIMHASH_BITS
+    for shingle, weight in shingle_weights.items():
+        shingle_hash = int(hashlib.sha256(shingle.encode("utf-8")).hexdigest(), 16)
+        for bit in range(_SIMHASH_BITS):
+            bit_totals[bit] += weight if shingle_hash & (1 << bit) else -weight
+
+    fingerprint = 0
+    for bit, total in enumerate(bit_totals):
+        if total > 0:
+            fingerprint |= 1 << bit
+    return fingerprint
+
+
+def hamming_distance(a: int, b: int) -> int:
+    """Number of differing bits between two fingerprints."""
+    return bin(a ^ b).count("1")