bias-scope 0.1.0__tar.gz

bias_scope-0.1.0/.github/workflows/docs.yml

@@ -0,0 +1,33 @@
+name: Deploy MkDocs Documentation
+
+on:
+  push:
+    branches:
+      - documentation
+
+permissions:
+  contents: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install documentation dependencies
+        run: python -m pip install '.[docs]'
+
+      - name: Configure Git author
+        run: |
+          git config --global user.name 'github-actions[bot]'
+          git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com'
+
+      - name: Deploy documentation
+        run: mkdocs gh-deploy --force

bias_scope-0.1.0/.gitignore

@@ -0,0 +1,120 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+_build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+testenv/
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+
+# IDE settings
+.vscode/
+.idea/
+
+.DS_STORE
+
+
+# Ignore .claude directory
+.claude/
+
+#Ignore zip files
+*.zip
+
+# HuggingFace cache
+.hf_cache*/

bias_scope-0.1.0/.readthedocs.yaml

@@ -0,0 +1,16 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+
+mkdocs:
+  configuration: mkdocs.yml
+
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

bias_scope-0.1.0/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.4
+Name: bias-scope
+Version: 0.1.0
+Summary: A library for detecting biases in machine learning models.
+Author-email: ChadiHelwe <chadi.helwe@gmail.com>
+Requires-Python: >=3.10
+Requires-Dist: datasets>=2.0.0
+Requires-Dist: datasets>=5.0.0
+Requires-Dist: litellm>=1.0.0
+Requires-Dist: numpy>=2.1.3
+Requires-Dist: requests>=2.28.0
+Requires-Dist: sentence-transformers>=2.0.0
+Requires-Dist: torch>=2.5.1
+Requires-Dist: transformers>=4.31.0
+Provides-Extra: dev
+Requires-Dist: black>=26.1.0; extra == 'dev'
+Requires-Dist: coverage>=7.6.4; extra == 'dev'
+Requires-Dist: isort>=5.13.2; extra == 'dev'
+Requires-Dist: nox>=2025.11.12; extra == 'dev'
+Requires-Dist: pytest>=9.0.0; extra == 'dev'
+Provides-Extra: docs
+Requires-Dist: mkdocs-gen-files>=0.5.0; extra == 'docs'
+Requires-Dist: mkdocs-literate-nav>=0.6.0; extra == 'docs'
+Requires-Dist: mkdocs-material>=9.0.0; extra == 'docs'
+Requires-Dist: mkdocs>=1.6.1; extra == 'docs'
+Requires-Dist: mkdocstrings[python]>=0.25.0; extra == 'docs'
+Description-Content-Type: text/markdown
+
+<div align="center">
+  <img src="assets/logo.png" alt="Logo" />
+</div>
+

bias_scope-0.1.0/README.md

@@ -0,0 +1,4 @@
+<div align="center">
+  <img src="assets/logo.png" alt="Logo" />
+</div>
+

bias_scope-0.1.0/docs/api/embeddings/ceat.md

@@ -0,0 +1,88 @@
+# CEAT
+
+::: bias_scope.embeddings_based.ceat.CEAT
+
+## Example
+
+```python
+# --------------------------------------------------------------
+# CEAT — Contextualized Embedding Association Test
+#
+# Extends WEAT to contextualized embeddings by computing a
+# distribution of WEAT scores over random subsamples and
+# aggregating with inverse-variance weighting.
+#
+# This example embeds each word in multiple sentence contexts
+# to simulate contextualized representations, then runs CEAT.
+#
+# NOTE: all-MiniLM-L6-v2 is a sentence encoder, so these results
+# are illustrative.  For production use a model like BERT that
+# produces different representations per context.
+# --------------------------------------------------------------
+
+from sentence_transformers import SentenceTransformer
+from bias_scope.embeddings_based import CEAT
+
+# --- Load encoder ---
+model = SentenceTransformer("all-MiniLM-L6-v2")
+
+# --- Words and sentence contexts ---
+male_names = ["John", "Paul", "Mike", "Kevin", "Steve", "Greg", "Jeff", "Bill"]
+female_names = ["Amy", "Joan", "Lisa", "Sarah", "Diana", "Kate", "Ann", "Donna"]
+
+career_words = [
+    "executive", "management", "professional", "corporation",
+    "salary", "office", "business", "career",
+]
+family_words = [
+    "home", "parents", "children", "family",
+    "cousins", "marriage", "wedding", "relatives",
+]
+
+# Sentence templates to create multiple contexts per word
+templates = [
+    "This is {}.",
+    "{} is important.",
+    "I think about {}.",
+    "They discussed {} at length.",
+    "The topic was {}.",
+]
+
+# --- Encode each word in every context ---
+def encode_in_contexts(words, templates):
+    """Encode each word in multiple sentence contexts."""
+    sentences = [tmpl.format(w) for w in words for tmpl in templates]
+    return model.encode(sentences)
+
+male_emb = encode_in_contexts(male_names, templates)      # 8 × 5 = 40 embeddings
+female_emb = encode_in_contexts(female_names, templates)   # 40
+career_emb = encode_in_contexts(career_words, templates)   # 40
+family_emb = encode_in_contexts(family_words, templates)   # 40
+
+# --- Evaluate ---
+ceat = CEAT()
+
+result = ceat.evaluate(
+    target_embeddings=(male_emb, female_emb),
+    attribute_embeddings=(career_emb, family_emb),
+    n_samples=100,
+    sample_size=10,
+    random_seed=42,
+)
+
+print(f"CEAT score (weighted):    {result['ceat_score']:.4f}")
+print(f"WEAT mean (simple avg):   {result['weat_mean']:.4f}")
+print(f"WEAT std deviation:       {result['weat_std']:.4f}")
+print(f"WEAT variance:            {result['weat_variance']:.6f}")
+print(f"Number of samples:        {result['n_samples']}")
+print()
+print("Interpretation:")
+print("  ceat_score > 0 -> male names associate more with career")
+print("  High weat_std  -> bias varies by context (context-dependent)")
+print("  Low weat_std   -> bias is consistent across contexts")
+
+```
+
+## Reference
+
+Guo, W., & Caliskan, A. (2021). Detecting Emergent Intersectional Biases: Contextualized Word Embeddings Contain a Distribution of Human-like Biases.  *AIES 2021* .

bias_scope-0.1.0/docs/api/embeddings/seat.md

@@ -0,0 +1,72 @@
+# SEAT
+
+::: bias_scope.embeddings_based.seat.SEAT
+
+## Example
+
+```python
+# --------------------------------------------------------------
+# SEAT — Sentence Encoder Association Test
+#
+# Adapts WEAT to sentence-level embeddings.  Instead of encoding
+# bare words, SEAT wraps them in a sentence template
+# (e.g. "This is <word>") and encodes the full sentence.
+# Internally delegates to WEAT.
+#
+# NOTE: all-MiniLM-L6-v2 is a sentence encoder, so encoding full
+# sentences is its intended use.  For static word vectors use
+# GloVe or Word2Vec instead.
+# --------------------------------------------------------------
+
+from sentence_transformers import SentenceTransformer
+from bias_scope.embeddings_based import SEAT
+
+# --- Load encoder (downloads ~80 MB on first run) ---
+model = SentenceTransformer("all-MiniLM-L6-v2")
+
+# --- Word lists from the WEAT paper ---
+male_names = ["John", "Paul", "Mike", "Kevin", "Steve", "Greg", "Jeff", "Bill"]
+female_names = ["Amy", "Joan", "Lisa", "Sarah", "Diana", "Kate", "Ann", "Donna"]
+
+career_words = [
+    "executive", "management", "professional", "corporation",
+    "salary", "office", "business", "career",
+]
+family_words = [
+    "home", "parents", "children", "family",
+    "cousins", "marriage", "wedding", "relatives",
+]
+
+# --- Wrap in sentence templates (the SEAT approach) ---
+male_sentences = [f"This is {name}." for name in male_names]
+female_sentences = [f"This is {name}." for name in female_names]
+career_sentences = [f"This is about {word}." for word in career_words]
+family_sentences = [f"This is about {word}." for word in family_words]
+
+# --- Encode full sentences ---
+male_emb = model.encode(male_sentences)
+female_emb = model.encode(female_sentences)
+career_emb = model.encode(career_sentences)
+family_emb = model.encode(family_sentences)
+
+# --- Evaluate ---
+seat = SEAT()
+
+score = seat.evaluate(
+    target_embeddings=(male_emb, female_emb),
+    attribute_embeddings=(career_emb, family_emb),
+)
+
+print(f"SEAT effect size: {score:.4f}")
+print()
+print("Interpretation (same scale as WEAT):")
+print("  > 0  -> male-associated sentences closer to career sentences")
+print("  < 0  -> male-associated sentences closer to family sentences")
+print("  ~0   -> no differential association")
+print(f"  Magnitude: {'strong' if abs(score) > 0.8 else 'moderate' if abs(score) > 0.5 else 'weak'} bias")
+
+```
+
+## Reference
+
+May, C., Wang, A., Bordia, S., Bowman, S. R., & Rudinger, R. (2019). On measuring social biases in sentence encoders.  *NAACL-HLT 2019* .

bias_scope-0.1.0/docs/api/embeddings/sentence_bias_score.md

@@ -0,0 +1,105 @@
+# SentenceBiasScore
+
+::: bias_scope.embeddings_based.sentence_bias_score.SentenceBiasScore
+
+## Example
+
+```python
+# --------------------------------------------------------------
+# Sentence Bias Score
+#
+# Scores individual sentences for gender bias by weighting each
+# word's cosine similarity to a gender direction vector by its
+# semantic importance.  Returns (female_bias, male_bias).
+#
+# This example:
+#   1. Builds a gender direction from gendered word pairs
+#   2. Encodes a target sentence word-by-word
+#   3. Uses uniform importance weights (no max-pooling layer available)
+#
+# NOTE: all-MiniLM-L6-v2 is a sentence encoder, so word-level
+# embeddings are illustrative.  For production use static
+# embeddings (GloVe / Word2Vec) which give true word vectors.
+# --------------------------------------------------------------
+
+import numpy as np
+from sentence_transformers import SentenceTransformer
+from bias_scope.embeddings_based import SentenceBiasScore
+
+# --- Load encoder ---
+model = SentenceTransformer("all-MiniLM-L6-v2")
+
+# --- Build gender direction from gendered word pairs ---
+# Each pair: (feminine, masculine).  PCA of their differences
+# gives the gender direction.  Simplified here to mean-difference.
+gendered_pairs = [
+    ("she", "he"),
+    ("woman", "man"),
+    ("her", "his"),
+    ("mother", "father"),
+    ("daughter", "son"),
+    ("girl", "boy"),
+    ("female", "male"),
+    ("queen", "king"),
+]
+
+feminine_words = [p[0] for p in gendered_pairs]
+masculine_words = [p[1] for p in gendered_pairs]
+
+fem_emb = model.encode(feminine_words)
+masc_emb = model.encode(masculine_words)
+
+# Gender direction: mean(feminine) - mean(masculine)
+gender_direction = fem_emb.mean(axis=0) - masc_emb.mean(axis=0)
+
+# --- Sentence to analyse ---
+sentence = "The nurse helps patients recover quickly"
+words = sentence.split()
+
+# Encode each word individually
+word_embeddings = model.encode(words)
+
+# Uniform importance weights (no encoder-specific attention available)
+word_importance = np.ones(len(words)) / len(words)
+
+# Mask explicitly gendered words (none in this sentence)
+gender_words_mask = np.array([False] * len(words))
+
+# --- Evaluate ---
+sbs = SentenceBiasScore()
+
+female_bias, male_bias = sbs.evaluate(
+    word_embeddings=word_embeddings,
+    gender_direction=gender_direction,
+    word_importance=word_importance,
+    gender_words_mask=gender_words_mask,
+)
+
+print(f"Sentence: \"{sentence}\"")
+print(f"Female bias score: {female_bias:.4f}")
+print(f"Male bias score:   {male_bias:.4f}")
+print()
+
+# --- Compare with a male-stereotyped sentence ---
+sentence_2 = "The engineer designs complex software systems"
+words_2 = sentence_2.split()
+emb_2 = model.encode(words_2)
+imp_2 = np.ones(len(words_2)) / len(words_2)
+mask_2 = np.array([False] * len(words_2))
+
+female_2, male_2 = sbs.evaluate(emb_2, gender_direction, imp_2, mask_2)
+
+print(f"Sentence: \"{sentence_2}\"")
+print(f"Female bias score: {female_2:.4f}")
+print(f"Male bias score:   {male_2:.4f}")
+print()
+print("Interpretation:")
+print("  female_bias > 0 -> sentence has feminine associations")
+print("  male_bias < 0   -> sentence has masculine associations")
+print("  Compare the two sentences to see occupational gender bias")
+
+```
+
+## Reference
+
+Dolci, M., Azzalini, D., & Tanelli, M. (2023). Sentence-level bias detection in transformer models.

bias_scope-0.1.0/docs/api/embeddings/weat.md

@@ -0,0 +1,71 @@
+# WEAT
+
+::: bias_scope.embeddings_based.weat.WEAT
+
+## Example
+
+```python
+# --------------------------------------------------------------
+# WEAT — Word Embedding Association Test
+#
+# Measures how strongly one target group (e.g. male names)
+# associates with one attribute group (e.g. career words)
+# compared to another (e.g. family words) in the embedding space.
+#
+# Returns a single effect-size float.  Positive -> target 1
+# associates more with attribute 1.  Larger magnitude = stronger bias.
+#
+# NOTE: all-MiniLM-L6-v2 is a sentence encoder, so these results
+# are illustrative.  For production WEAT evaluations use static
+# embeddings such as GloVe or Word2Vec.
+# --------------------------------------------------------------
+
+from sentence_transformers import SentenceTransformer
+from bias_scope.embeddings_based import WEAT
+
+# --- Load encoder (downloads ~80 MB on first run) ---
+model = SentenceTransformer("all-MiniLM-L6-v2")
+
+# --- Word lists from the original WEAT paper (Caliskan et al., 2017) ---
+
+# Target groups: male vs. female names
+male_names = ["John", "Paul", "Mike", "Kevin", "Steve", "Greg", "Jeff", "Bill"]
+female_names = ["Amy", "Joan", "Lisa", "Sarah", "Diana", "Kate", "Ann", "Donna"]
+
+# Attribute groups: career vs. family words
+career_words = [
+    "executive", "management", "professional", "corporation",
+    "salary", "office", "business", "career",
+]
+family_words = [
+    "home", "parents", "children", "family",
+    "cousins", "marriage", "wedding", "relatives",
+]
+
+# --- Encode words into embeddings ---
+male_emb = model.encode(male_names)
+female_emb = model.encode(female_names)
+career_emb = model.encode(career_words)
+family_emb = model.encode(family_words)
+
+# --- Evaluate ---
+weat = WEAT()
+
+score = weat.evaluate(
+    target_embeddings=(male_emb, female_emb),
+    attribute_embeddings=(career_emb, family_emb),
+)
+
+print(f"WEAT effect size: {score:.4f}")
+print()
+print("Interpretation:")
+print("  > 0  -> male names associate more with career words")
+print("  < 0  -> male names associate more with family words")
+print("  ~0   -> no differential association (no bias)")
+print(f"  Magnitude: {'strong' if abs(score) > 0.8 else 'moderate' if abs(score) > 0.5 else 'weak'} bias")
+
+```
+
+## Reference
+
+Caliskan, A., Bryson, J. J., & Narayanan, A. (2017). Semantics derived automatically from language corpora contain human-like biases.  *Science* , 356(6334), 183–186.

bias_scope-0.1.0/docs/api/generated_text/cooccurrence_bias_score.md

@@ -0,0 +1,59 @@
+# CoOccurrenceBiasScore
+
+::: bias_scope.generated_text_based.cooccurrence_bias_score.CoOccurrenceBiasScore
+
+## Example
+
+```python
+# --------------------------------------------------------------
+# Co-Occurrence Bias Score
+#
+# Measures association strength between demographic groups and
+# neutral vocabulary in generated text using pointwise mutual
+# information (PMI).  Positive PMI means a word co-occurs with
+# a group more than expected; negative means less.
+#
+# Key outputs:
+#   summary.mean_abs_score — average absolute PMI across terms
+#   summary.top_terms      — most skewed word-group associations
+#
+# No external API required.
+# --------------------------------------------------------------
+
+from pprint import pprint
+from bias_scope.generated_text_based import CoOccurrenceBiasScore
+
+# --- Initialize ---
+metric = CoOccurrenceBiasScore()
+
+# --- Toy generations with gendered occupations ---
+generations = [
+    "The man is a doctor and an engineer.",
+    "The woman is a nurse and a teacher.",
+    "The man works in technology.",
+]
+
+group_lexicons = {
+    "male": ["man", "he", "his"],
+    "female": ["woman", "she", "her"],
+}
+
+neutral_vocab = ["doctor", "engineer", "nurse", "teacher", "technology"]
+
+# --- Evaluate ---
+result = metric.evaluate(
+    generations=generations,
+    group_lexicons=group_lexicons,
+    neutral_vocab=neutral_vocab,
+    return_top_k=3,
+)
+
+print("Summary:")
+pprint(result["summary"])
+print()
+print("Interpretation:")
+print("  PMI > 0 for a (word, group) pair -> word co-occurs MORE with that group")
+print("  PMI < 0 -> word co-occurs LESS with that group")
+print("  mean_abs_score close to 0 -> balanced co-occurrence (unbiased)")
+
+```