inferencebench-embeddings 0.0.2__py3-none-any.whl

inferencebench_embeddings/__init__.py

@@ -0,0 +1,15 @@
+"""InferenceBench embeddings-retrieval plugin."""
+
+from inferencebench_embeddings.plugin import (
+    EXPECTED_METRICS,
+    EmbeddingsRetrievalPlugin,
+)
+from inferencebench_embeddings.schemas import BenchmarkSpec, EngineKind, RunContext
+
+__all__ = [
+    "EXPECTED_METRICS",
+    "BenchmarkSpec",
+    "EmbeddingsRetrievalPlugin",
+    "EngineKind",
+    "RunContext",
+]

inferencebench_embeddings/benchmarks/beir-mini.yaml

@@ -0,0 +1,13 @@
+benchmark_id: embeddings.retrieval.beir-mini
+suite_version: 1.0.0
+description: BEIR-style small corpus, recall@5.
+modality: embeddings
+kind: retrieval
+dataset:
+  id: builtin-beir-mini
+  path: beir-mini-queries.jsonl
+  corpus_path: beir-mini-corpus.jsonl
+slo_template: embeddings.retrieval.standard
+warmup:
+  discard_runs: 0
+metric: recall_at_5

inferencebench_embeddings/benchmarks/long-doc.yaml

@@ -0,0 +1,13 @@
+benchmark_id: embeddings.retrieval.long-doc
+suite_version: 1.0.0
+description: Long-document corpus, nDCG@10.
+modality: embeddings
+kind: retrieval
+dataset:
+  id: builtin-long-doc
+  path: long-doc-queries.jsonl
+  corpus_path: long-doc-corpus.jsonl
+slo_template: embeddings.retrieval.standard
+warmup:
+  discard_runs: 0
+metric: ndcg_at_10

inferencebench_embeddings/benchmarks/msmarco-style.yaml

@@ -0,0 +1,13 @@
+benchmark_id: embeddings.retrieval.msmarco-style
+suite_version: 1.0.0
+description: MS MARCO-style passage ranking, MRR@10.
+modality: embeddings
+kind: retrieval
+dataset:
+  id: builtin-msmarco-style
+  path: msmarco-style-queries.jsonl
+  corpus_path: msmarco-style-corpus.jsonl
+slo_template: embeddings.retrieval.standard
+warmup:
+  discard_runs: 0
+metric: mrr_at_10

inferencebench_embeddings/benchmarks/query-expansion.yaml

@@ -0,0 +1,13 @@
+benchmark_id: embeddings.retrieval.query-expansion
+suite_version: 1.0.0
+description: Short queries against a paraphrase-rich corpus.
+modality: embeddings
+kind: retrieval
+dataset:
+  id: builtin-query-expansion
+  path: query-expansion-queries.jsonl
+  corpus_path: query-expansion-corpus.jsonl
+slo_template: embeddings.retrieval.standard
+warmup:
+  discard_runs: 0
+metric: recall_at_5

inferencebench_embeddings/datasets/beir-mini-corpus.jsonl

@@ -0,0 +1,20 @@
+{"doc_id": "doc-1", "text": "Paris is the capital and most populous city of France."}
+{"doc_id": "doc-2", "text": "Tokyo is the capital of Japan and one of the most populous metropolitan areas."}
+{"doc_id": "doc-3", "text": "The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris."}
+{"doc_id": "doc-4", "text": "Mount Fuji is the highest mountain in Japan and a symbol of the country."}
+{"doc_id": "doc-5", "text": "The Pacific Ocean is the largest and deepest ocean on Earth."}
+{"doc_id": "doc-6", "text": "William Shakespeare wrote many plays including Hamlet and Macbeth."}
+{"doc_id": "doc-7", "text": "Hamlet is one of Shakespeare's most famous tragedies, set in Denmark."}
+{"doc_id": "doc-8", "text": "Mars is the fourth planet from the Sun, often called the Red Planet."}
+{"doc_id": "doc-9", "text": "Gold is a chemical element with the symbol Au and atomic number 79."}
+{"doc_id": "doc-10", "text": "Silver is a chemical element with the symbol Ag and is widely used in jewellery."}
+{"doc_id": "doc-11", "text": "Leonardo da Vinci painted the Mona Lisa, now displayed in the Louvre."}
+{"doc_id": "doc-12", "text": "The Louvre is the world's largest art museum, located in Paris, France."}
+{"doc_id": "doc-13", "text": "Blue whales are the largest mammals on Earth and can weigh over 100 tons."}
+{"doc_id": "doc-14", "text": "Photosynthesis is the process by which plants absorb carbon dioxide and produce oxygen."}
+{"doc_id": "doc-15", "text": "The Moon landing in 1969 was a major milestone in human space exploration."}
+{"doc_id": "doc-16", "text": "Apollo 11 was the spaceflight that first landed humans on the Moon."}
+{"doc_id": "doc-17", "text": "The Atlantic Ocean separates the Americas from Europe and Africa."}
+{"doc_id": "doc-18", "text": "Albert Einstein developed the theory of relativity and won the Nobel Prize in Physics."}
+{"doc_id": "doc-19", "text": "The speed of light in vacuum is approximately 299,792 kilometres per second."}
+{"doc_id": "doc-20", "text": "DNA is a molecule composed of two strands that coil around each other to form a double helix."}

inferencebench_embeddings/datasets/beir-mini-queries.jsonl

@@ -0,0 +1,5 @@
+{"query": "What is the capital of France?", "relevant_doc_ids": ["doc-1", "doc-3", "doc-12"]}
+{"query": "Tell me about Japan's tallest mountain.", "relevant_doc_ids": ["doc-2", "doc-4"]}
+{"query": "Who wrote Hamlet?", "relevant_doc_ids": ["doc-6", "doc-7"]}
+{"query": "What is the chemical symbol for gold?", "relevant_doc_ids": ["doc-9"]}
+{"query": "When did humans first land on the Moon?", "relevant_doc_ids": ["doc-15", "doc-16"]}

inferencebench_embeddings/datasets/long-doc-corpus.jsonl

@@ -0,0 +1,10 @@
+{"doc_id": "ld-1", "text": "Transformer architectures have revolutionised natural language processing since their introduction in 2017. The original paper attention is all you need showed that self-attention layers can replace recurrent and convolutional structures while training in parallel across all positions, which dramatically reduced wall-clock training time and unlocked the scaling laws that subsequent work has explored exhaustively."}
+{"doc_id": "ld-2", "text": "Renewable energy adoption has accelerated globally over the past decade, with solar photovoltaic capacity growing more than tenfold and onshore wind seeing similar gains. The economics of grid-scale battery storage have also shifted dramatically, making it feasible to firm intermittent generation in many markets without resorting to natural gas peakers."}
+{"doc_id": "ld-3", "text": "Modern garbage collectors in managed runtimes use generational hypotheses to avoid scanning the entire heap on every cycle. Young objects die quickly, so the nursery is collected frequently with a copying algorithm, while the older generation uses a mark-and-sweep or mark-and-compact pass less often. Region-based collectors like G1 extend this with finer-grained partitioning."}
+{"doc_id": "ld-4", "text": "The history of vaccines stretches back over two centuries, beginning with Edward Jenner's observation that milkmaids exposed to cowpox seemed immune to smallpox. The twentieth century saw the development of inactivated and live-attenuated vaccines for polio, measles, and dozens of other diseases, while the twenty-first century has been defined by mRNA platforms that proved themselves during the COVID-19 pandemic."}
+{"doc_id": "ld-5", "text": "Distributed consensus algorithms like Paxos and Raft solve the problem of agreeing on a sequence of values across a set of failure-prone nodes. Raft was designed explicitly for understandability and breaks consensus into leader election, log replication, and safety, while Paxos predates Raft and is the basis for many production systems including Google's Chubby lock service."}
+{"doc_id": "ld-6", "text": "Quantum computing exploits superposition and entanglement to perform certain calculations exponentially faster than classical machines. Shor's algorithm for factoring integers and Grover's algorithm for unstructured search are the canonical examples, but practical quantum advantage on commercially relevant problems remains an open research question as engineering teams work to reduce gate error rates."}
+{"doc_id": "ld-7", "text": "Coral reefs are some of the most biodiverse ecosystems on the planet, supporting roughly a quarter of all marine species despite covering less than one percent of the ocean floor. Rising sea temperatures and ocean acidification have triggered widespread bleaching events that threaten the long-term viability of these systems, with the Great Barrier Reef among the most studied examples."}
+{"doc_id": "ld-8", "text": "Compiler optimisation involves a sequence of program transformations that preserve observable behaviour while improving some target metric such as execution time, code size, or energy usage. Modern compilers like LLVM and GCC implement hundreds of passes including inlining, constant propagation, loop unrolling, vectorisation, and aggressive dead-code elimination."}
+{"doc_id": "ld-9", "text": "The protein folding problem asks how a linear sequence of amino acids reliably collapses into a specific three-dimensional structure. AlphaFold demonstrated that deep learning models trained on the Protein Data Bank can predict structures with accuracy comparable to experimental methods for many protein families, accelerating drug discovery and basic biology research."}
+{"doc_id": "ld-10", "text": "Modern web browsers are among the most complex pieces of consumer software ever shipped. They include a multi-process architecture, a JIT-compiling JavaScript engine, a GPU-accelerated compositor, a layout engine that implements thousands of pages of CSS specification, an extension API, sandboxed rendering, and increasingly an integration point for AI assistants."}

inferencebench_embeddings/datasets/long-doc-queries.jsonl

@@ -0,0 +1,3 @@
+{"query": "How do transformer architectures change NLP training?", "relevant_doc_ids": ["ld-1"]}
+{"query": "What is the role of consensus algorithms in distributed systems?", "relevant_doc_ids": ["ld-5"]}
+{"query": "How has deep learning advanced biology and drug discovery?", "relevant_doc_ids": ["ld-9", "ld-1"]}

inferencebench_embeddings/datasets/msmarco-style-corpus.jsonl

@@ -0,0 +1,25 @@
+{"doc_id": "mm-1", "text": "Coffee is a brewed beverage prepared from roasted coffee beans. It originated in Ethiopia and is now consumed worldwide. Espresso, drip, and pour-over are common preparation methods."}
+{"doc_id": "mm-2", "text": "The Pacific Ocean is the largest ocean on Earth, covering one third of the planet's surface. It is bordered by Asia, the Americas, and Antarctica. Its average depth is roughly four kilometres."}
+{"doc_id": "mm-3", "text": "Photosynthesis is the biological process by which plants convert sunlight, water, and carbon dioxide into glucose and oxygen. Chlorophyll in the leaves absorbs light energy. The process powers most life on Earth."}
+{"doc_id": "mm-4", "text": "Football is the most popular sport in the world by viewership. The FIFA World Cup is held every four years. Major leagues operate in Europe, South America, and increasingly Asia and North America."}
+{"doc_id": "mm-5", "text": "TCP is a connection-oriented protocol that guarantees delivery and ordering through acknowledgements. UDP is a connectionless protocol that trades reliability for lower latency. Choose TCP for files, UDP for real-time streams."}
+{"doc_id": "mm-6", "text": "The Renaissance was a cultural movement that began in Italy in the fourteenth century. It produced major advances in art, science, and politics. Leonardo da Vinci and Michelangelo are emblematic figures of the period."}
+{"doc_id": "mm-7", "text": "Python is widely used in data science thanks to libraries like NumPy, pandas, scikit-learn, and PyTorch. Beginners often start with introductory courses on Kaggle or Coursera. Practice with real datasets accelerates learning."}
+{"doc_id": "mm-8", "text": "The smartphone revolutionised personal computing in the late 2000s. iOS and Android dominate the operating-system market. App stores transformed the software distribution model."}
+{"doc_id": "mm-9", "text": "Plate tectonics describes how Earth's lithosphere is divided into plates that move over the mantle. Earthquakes and volcanoes mostly occur at plate boundaries. The theory was widely accepted in the 1960s."}
+{"doc_id": "mm-10", "text": "The Mediterranean diet emphasises vegetables, fruits, whole grains, olive oil, and fish. It has been linked to lower rates of cardiovascular disease. The diet originated in countries like Greece and Italy."}
+{"doc_id": "mm-11", "text": "Vitamin D deficiency can cause fatigue, bone pain, muscle weakness, and mood changes. It is common in regions with limited sunlight exposure. Supplementation and dietary sources like fatty fish help correct it."}
+{"doc_id": "mm-12", "text": "The Industrial Revolution began in Britain in the eighteenth century. Steam power, mechanised textile production, and railways transformed economies. It also drove urbanisation and large-scale environmental change."}
+{"doc_id": "mm-13", "text": "Quantum mechanics describes physical systems at atomic scales. Superposition, entanglement, and uncertainty are core principles. The field underpins technologies from lasers to MRI scanners."}
+{"doc_id": "mm-14", "text": "Plants exchange gases through small pores called stomata, typically on the underside of leaves. During photosynthesis they take in carbon dioxide and release oxygen. The stomata also regulate water loss via transpiration."}
+{"doc_id": "mm-15", "text": "The euro is the common currency of nineteen European Union member states. It was introduced in 1999 for accounting and 2002 in physical form. The European Central Bank manages monetary policy for the eurozone."}
+{"doc_id": "mm-16", "text": "Bicycles are an efficient form of human-powered transport, widely used for commuting in cities like Amsterdam and Copenhagen. Modern designs include road, mountain, and electric variants. Cycling reduces carbon emissions and improves health."}
+{"doc_id": "mm-17", "text": "The DNA double helix was discovered in 1953 by Watson and Crick, building on work by Franklin and Wilkins. The structure consists of two strands held by base pairs. It encodes the genetic information of all known organisms."}
+{"doc_id": "mm-18", "text": "The Great Wall of China was built across many dynasties, with most surviving sections dating from the Ming period. It stretches for thousands of kilometres across northern China. The wall served as defence, signalling, and trade control."}
+{"doc_id": "mm-19", "text": "Bread is one of the oldest prepared foods, with evidence of baking going back over fourteen thousand years. Modern bread relies on wheat flour, water, salt, and yeast. Sourdough uses wild yeast and lactic-acid bacteria."}
+{"doc_id": "mm-20", "text": "The Amazon rainforest is the largest tropical forest on Earth, spanning nine countries in South America. It hosts an enormous share of global biodiversity. Deforestation threatens carbon storage and species survival."}
+{"doc_id": "mm-21", "text": "Cloud computing delivers compute, storage, and networking as on-demand services over the internet. Major providers include AWS, Azure, and Google Cloud. The model has reshaped how software is built and deployed."}
+{"doc_id": "mm-22", "text": "Sunlight exposure is the body's main source of vitamin D, with diet contributing only a small fraction. Office workers and people in high-latitude regions are at higher risk of deficiency. Routine blood tests can confirm low levels."}
+{"doc_id": "mm-23", "text": "The Eiffel Tower in Paris was completed in 1889 for the Universal Exposition. It is made of wrought iron and stands 330 metres tall. The tower is among the most-visited paid monuments in the world."}
+{"doc_id": "mm-24", "text": "Machine learning has driven advances in computer vision, language understanding, and recommendation systems. Deep neural networks scale impressively with data and compute. Training large models requires specialised hardware accelerators."}
+{"doc_id": "mm-25", "text": "Construction of the Great Wall continued under successive Chinese dynasties starting with the Qin in the third century BCE. Materials varied by region, from tamped earth to bricks. Watchtowers and garrisons were spaced along its length."}

inferencebench_embeddings/datasets/msmarco-style-queries.jsonl

@@ -0,0 +1,5 @@
+{"query": "how does photosynthesis work in plants", "relevant_doc_ids": ["mm-3", "mm-14"]}
+{"query": "best way to learn python for data science", "relevant_doc_ids": ["mm-7"]}
+{"query": "symptoms of vitamin d deficiency", "relevant_doc_ids": ["mm-11", "mm-22"]}
+{"query": "what is the difference between tcp and udp", "relevant_doc_ids": ["mm-5"]}
+{"query": "history of the great wall of china", "relevant_doc_ids": ["mm-18", "mm-25"]}

inferencebench_embeddings/datasets/query-expansion-corpus.jsonl

@@ -0,0 +1,20 @@
+{"doc_id": "qe-1", "text": "A vehicle with four wheels used for personal transport on roads."}
+{"doc_id": "qe-2", "text": "An automobile, typically powered by an internal combustion engine or electric motor."}
+{"doc_id": "qe-3", "text": "A long, articulated road vehicle used to carry many passengers between stops."}
+{"doc_id": "qe-4", "text": "A two-wheeled, pedal-driven personal transport device."}
+{"doc_id": "qe-5", "text": "A medical practitioner who diagnoses and treats illness."}
+{"doc_id": "qe-6", "text": "A physician trained to provide medical care and prescribe medication."}
+{"doc_id": "qe-7", "text": "A trained nurse who assists with patient care in hospitals or clinics."}
+{"doc_id": "qe-8", "text": "An academic who has earned a doctorate in a non-medical field."}
+{"doc_id": "qe-9", "text": "A film, typically released in cinemas, that tells a story through moving images."}
+{"doc_id": "qe-10", "text": "A motion picture, screened in theatres or streamed online."}
+{"doc_id": "qe-11", "text": "A short video clip recorded on a smartphone and shared online."}
+{"doc_id": "qe-12", "text": "A live theatrical performance staged in front of an audience."}
+{"doc_id": "qe-13", "text": "A building used as a private residence by an individual or family."}
+{"doc_id": "qe-14", "text": "A dwelling, typically detached, where people live and sleep."}
+{"doc_id": "qe-15", "text": "A small wooden structure used to store gardening tools."}
+{"doc_id": "qe-16", "text": "A high-rise apartment building containing many separate flats."}
+{"doc_id": "qe-17", "text": "The art of arranging sounds in time to produce a composition through melody, harmony, and rhythm."}
+{"doc_id": "qe-18", "text": "Songs and instrumental pieces, performed live or recorded, that people listen to for enjoyment."}
+{"doc_id": "qe-19", "text": "Ambient noise from machinery and traffic in a busy urban environment."}
+{"doc_id": "qe-20", "text": "A spoken-word podcast covering interviews with public figures."}

inferencebench_embeddings/datasets/query-expansion-queries.jsonl

@@ -0,0 +1,5 @@
+{"query": "car", "relevant_doc_ids": ["qe-1", "qe-2"]}
+{"query": "doctor", "relevant_doc_ids": ["qe-5", "qe-6"]}
+{"query": "movie", "relevant_doc_ids": ["qe-9", "qe-10"]}
+{"query": "house", "relevant_doc_ids": ["qe-13", "qe-14"]}
+{"query": "music", "relevant_doc_ids": ["qe-17", "qe-18"]}

inferencebench_embeddings/plugin.py

@@ -0,0 +1,414 @@
+"""EmbeddingsRetrievalPlugin — entry point for ``embeddings.retrieval`` benchmarks.
+
+Phase-2-quality skeleton: produces a real signed envelope by deterministically
+ranking the corpus per query via ``sha256(query + doc_id)`` sort, then scoring
+the top-k against the fixture's relevant set with recall@5 / mrr@10 / nDCG@10.
+
+Future revisions wire a real embedding model call into :meth:`_rank_corpus`;
+the rest of the pipeline (signing, aggregation, sample dump) is production-shaped.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import time
+from pathlib import Path
+
+import yaml
+
+from inferencebench.envelope import (
+    DatasetSpec as EnvDatasetSpec,
+)
+from inferencebench.envelope import (
+    EngineConfig,
+    Envelope,
+    EnvelopeBuilder,
+    ModelConfig,
+    Quantization,
+    SigningMode,
+    sign_envelope,
+)
+from inferencebench.harness import (
+    Sample,
+    collect_hardware_fingerprint,
+    collect_software_provenance,
+)
+from inferencebench.harness.metrics import EnergyReport, Percentiles, TelemetryWindow
+from inferencebench_embeddings.schemas import BenchmarkSpec, EngineKind, RunContext
+from inferencebench_embeddings.scoring import METRICS
+
+
+def _json_num(v: float) -> str:
+    """JSON-safe numeric encoder: NaN/inf become null."""
+    if isinstance(v, float) and (math.isnan(v) or math.isinf(v)):
+        return "null"
+    return repr(v)
+
+
+# Engines that require ``base_url`` (self-hosted TEI servers).
+_SELF_HOSTED_ENGINES = frozenset({EngineKind.TEI})
+
+
+def _fixtures_cache_root() -> Path:
+    """Resolve the bench-fixtures cache root for ``fixtures://`` dataset URIs."""
+    override = os.environ.get("BENCH_FIXTURES_ROOT")
+    if override:
+        return Path(override)
+    return Path.home() / ".cache" / "inferencebench" / "fixtures"
+
+
+def _compute_fixture_hash(queries: list[dict[str, object]], corpus: list[dict[str, str]]) -> str:
+    """SHA-256 over the canonical-JSON-encoded queries + corpus."""
+    canonical = json.dumps(
+        {"queries": queries, "corpus": corpus},
+        sort_keys=True,
+        separators=(",", ":"),
+        default=str,
+    )
+    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def _rank_corpus(query: str, corpus_ids: list[str]) -> list[str]:
+    """Deterministically rank corpus doc-ids for a query.
+
+    Sorts by ``sha256(query + doc_id).hexdigest()`` so the ranking is
+    reproducible across machines and Python versions but uncorrelated with
+    actual relevance — exactly what we want for a contract-validation
+    skeleton that should produce a non-degenerate metric in [0, 1].
+    """
+
+    def _key(doc_id: str) -> str:
+        h = hashlib.sha256()
+        h.update(query.encode("utf-8"))
+        h.update(b"\x00")
+        h.update(doc_id.encode("utf-8"))
+        return h.hexdigest()
+
+    return sorted(corpus_ids, key=_key)
+
+
+# Metrics this plugin is expected to emit. Consumed by ``bench coverage``.
+EXPECTED_METRICS: tuple[str, ...] = (
+    "recall_at_5_mean",
+    "recall_at_5_p50",
+    "recall_at_5_p95",
+    "mrr_at_10_mean",
+    "ndcg_at_10_mean",
+    "ok_rate",
+    "n_queries",
+    "corpus_size",
+)
+
+
+class EmbeddingsRetrievalPlugin:
+    """Plugin entry point. Registered via ``inferencebench.plugins`` entrypoint group."""
+
+    suite_id = "embeddings.retrieval"
+    version = "0.0.0"
+    description = (
+        "Embeddings retrieval benchmarks (deterministic hash ranking on bundled "
+        "corpora; real embedding-model invocation deferred)."
+    )
+
+    # ----------------------------------------------------------- benchmarks #
+    def list_benchmarks(self) -> list[BenchmarkSpec]:
+        bench_dir = self._benchmarks_dir()
+        specs: list[BenchmarkSpec] = []
+        if not bench_dir.exists():
+            return specs
+        for yml in sorted(bench_dir.glob("*.yaml")):
+            specs.append(self._load_yaml(yml))
+        return specs
+
+    def get_benchmark(self, benchmark_id: str) -> BenchmarkSpec:
+        for spec in self.list_benchmarks():
+            if spec.benchmark_id == benchmark_id:
+                return spec
+        msg = f"benchmark_id not found: {benchmark_id}"
+        raise KeyError(msg)
+
+    # ------------------------------------------------------------- validate #
+    def validate(self, spec: BenchmarkSpec, context: RunContext) -> list[str]:
+        warnings: list[str] = []
+        if not context.model_id:
+            warnings.append("model_id is empty")
+        if context.engine_kind in _SELF_HOSTED_ENGINES and not context.base_url:
+            warnings.append(
+                f"{context.engine_kind.value} needs base_url (e.g. http://localhost:8080)"
+            )
+        if not self._queries_path(spec).exists():
+            warnings.append(f"queries fixture not found: {spec.dataset.path}")
+        if not self._corpus_path(spec).exists():
+            warnings.append(f"corpus fixture not found: {spec.dataset.corpus_path}")
+        return warnings
+
+    # ------------------------------------------------------------------ run #
+    def run(self, spec: BenchmarkSpec, context: RunContext) -> Envelope:
+        """Execute the benchmark and return a SIGNED envelope."""
+        queries = self._load_queries(spec)
+        corpus = self._load_corpus(spec)
+        fixture_hash = _compute_fixture_hash(queries, corpus)
+        k, scorer = METRICS[spec.metric]
+
+        corpus_ids = [doc["doc_id"] for doc in corpus]
+
+        samples: list[Sample] = []
+        scores: list[float] = []
+        telemetry = TelemetryWindow()
+        with telemetry:
+            for idx, q in enumerate(queries):
+                query_text = str(q["query"])
+                raw_relevant = q.get("relevant_doc_ids") or []
+                # ``_load_queries`` guarantees this is a list[str], but mypy sees
+                # the dict as ``dict[str, object]`` — narrow explicitly.
+                assert isinstance(raw_relevant, list)
+                relevant = [str(x) for x in raw_relevant]
+                t_arrival = time.perf_counter() * 1000.0
+                t_start = time.perf_counter()
+                ranking = _rank_corpus(query_text, corpus_ids)
+                score = float(scorer(ranking, relevant, k))
+                total_ms = (time.perf_counter() - t_start) * 1000.0
+                scores.append(score)
+                samples.append(
+                    Sample(
+                        request_idx=idx,
+                        arrival_ms=t_arrival,
+                        start_ms=t_arrival,
+                        ttft_ms=float("nan"),
+                        total_ms=total_ms,
+                        tpot_ms=float("nan"),
+                        tokens_in=len(query_text.split()),
+                        tokens_out=k,
+                        cost_usd=0.0,
+                        finish_reason="stop",
+                        ok=True,
+                        extra={
+                            "score": score,
+                            "k": k,
+                            "n_relevant": len(relevant),
+                            "topk": ranking[:k],
+                        },
+                    )
+                )
+
+        # Best-effort diagnostic dump — never blocks the run on I/O errors.
+        self._dump_samples(context, samples)
+
+        envelope = self._build_envelope(
+            spec,
+            context,
+            samples=samples,
+            scores=scores,
+            corpus_size=len(corpus),
+            dataset_hash=fixture_hash,
+            energy=telemetry.summarise(samples),
+        )
+        signing_mode = context.extra.get("signing_mode", "dev")
+        dev_key_path = context.extra.get("dev_key_path")
+        if signing_mode == "dev":
+            if not dev_key_path:
+                msg = "dev signing requires context.extra['dev_key_path']"
+                raise ValueError(msg)
+            return sign_envelope(
+                envelope,
+                mode=SigningMode.DEV,
+                dev_key_path=Path(str(dev_key_path)),
+            )
+        return sign_envelope(envelope, mode=SigningMode.KEYLESS)
+
+    # ------------------------------------------------------------ samples #
+    def _dump_samples(self, context: RunContext, samples: list[Sample]) -> None:
+        """Write per-query samples (incl. score) to ``<output_dir>/samples-<ts>.jsonl``.
+
+        Mirrors the llm-quality plugin's diagnostic dump — failures here
+        never block the run.
+        """
+        try:
+            out_dir = Path(context.output_dir)
+            out_dir.mkdir(parents=True, exist_ok=True)
+            ts = int(time.time())
+            path = out_dir / f"samples-{ts}.jsonl"
+            with path.open("w", encoding="utf-8") as fp:
+                for s in samples:
+                    score = s.extra.get("score") if s.extra else None
+                    score_part = (
+                        ',"score":' + _json_num(float(score))
+                        if isinstance(score, (int, float))
+                        else ""
+                    )
+                    fp.write(
+                        '{"request_idx":'
+                        + str(s.request_idx)
+                        + ',"ok":'
+                        + ("true" if s.ok else "false")
+                        + ',"total_ms":'
+                        + _json_num(s.total_ms)
+                        + ',"tokens_in":'
+                        + str(s.tokens_in)
+                        + ',"tokens_out":'
+                        + str(s.tokens_out)
+                        + score_part
+                        + ',"finish_reason":"'
+                        + (s.finish_reason or "")
+                        + '"'
+                        + "}\n"
+                    )
+        except OSError:
+            pass  # diagnostics-only — never block the run
+
+    # ---------------------------------------------------------- file paths #
+    def _benchmarks_dir(self) -> Path:
+        return Path(__file__).parent / "benchmarks"
+
+    def _datasets_dir(self) -> Path:
+        return Path(__file__).parent / "datasets"
+
+    def _queries_path(self, spec: BenchmarkSpec) -> Path:
+        raw = spec.dataset.path
+        if raw.startswith("fixtures://"):
+            return _fixtures_cache_root() / f"{raw[len('fixtures://') :]}.jsonl"
+        return self._datasets_dir() / raw
+
+    def _corpus_path(self, spec: BenchmarkSpec) -> Path:
+        raw = spec.dataset.corpus_path
+        if raw.startswith("fixtures://"):
+            return _fixtures_cache_root() / f"{raw[len('fixtures://') :]}.jsonl"
+        return self._datasets_dir() / raw
+
+    def _load_yaml(self, path: Path) -> BenchmarkSpec:
+        raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
+        return BenchmarkSpec.model_validate(raw)
+
+    def _load_queries(self, spec: BenchmarkSpec) -> list[dict[str, object]]:
+        path = self._queries_path(spec)
+        if not path.exists():
+            msg = f"queries fixture not found: {path}"
+            raise FileNotFoundError(msg)
+        items: list[dict[str, object]] = []
+        with path.open("r", encoding="utf-8") as fp:
+            for line in fp:
+                line = line.strip()
+                if not line:
+                    continue
+                obj = json.loads(line)
+                if not isinstance(obj, dict):
+                    continue
+                if "query" not in obj or "relevant_doc_ids" not in obj:
+                    continue
+                items.append(
+                    {
+                        "query": str(obj["query"]),
+                        "relevant_doc_ids": [str(x) for x in obj["relevant_doc_ids"]],
+                    }
+                )
+        if not items:
+            msg = f"queries fixture is empty: {path}"
+            raise ValueError(msg)
+        return items
+
+    def _load_corpus(self, spec: BenchmarkSpec) -> list[dict[str, str]]:
+        path = self._corpus_path(spec)
+        if not path.exists():
+            msg = f"corpus fixture not found: {path}"
+            raise FileNotFoundError(msg)
+        items: list[dict[str, str]] = []
+        with path.open("r", encoding="utf-8") as fp:
+            for line in fp:
+                line = line.strip()
+                if not line:
+                    continue
+                obj = json.loads(line)
+                if not isinstance(obj, dict):
+                    continue
+                if "doc_id" not in obj or "text" not in obj:
+                    continue
+                items.append({"doc_id": str(obj["doc_id"]), "text": str(obj["text"])})
+        if not items:
+            msg = f"corpus fixture is empty: {path}"
+            raise ValueError(msg)
+        return items
+
+    # ---------------------------------------------------------- envelope #
+    def _build_envelope(
+        self,
+        spec: BenchmarkSpec,
+        context: RunContext,
+        *,
+        samples: list[Sample],
+        scores: list[float],
+        corpus_size: int,
+        dataset_hash: str,
+        energy: EnergyReport | None = None,
+    ) -> Envelope:
+        hw = collect_hardware_fingerprint()
+        sw = collect_software_provenance()
+
+        metrics: dict[str, float | int | str | None] = {}
+
+        ok_samples = [s for s in samples if s.ok]
+        n_ok = len(ok_samples)
+        metrics["n_queries"] = float(len(samples))
+        metrics["n_ok"] = float(n_ok)
+        metrics["ok_rate"] = float(n_ok) / float(len(samples)) if samples else 0.0
+        metrics["corpus_size"] = float(corpus_size)
+
+        # Headline retrieval metric — keyed by the spec's metric so downstream
+        # `bench diff` knows higher-is-better (see _HIGHER_IS_BETTER in
+        # cli/commands/diff.py).
+        if scores:
+            mean_score = sum(scores) / len(scores)
+            metric_prefix = spec.metric  # "recall_at_5" | "mrr_at_10" | "ndcg_at_10"
+            metrics[f"{metric_prefix}_mean"] = mean_score
+            if len(scores) >= 2:
+                pcts = Percentiles(scores, percentiles=(50.0, 95.0))
+                metrics[f"{metric_prefix}_p50"] = pcts.p50
+                metrics[f"{metric_prefix}_p95"] = pcts.p95
+            else:
+                metrics[f"{metric_prefix}_p50"] = mean_score
+                metrics[f"{metric_prefix}_p95"] = mean_score
+
+        total_vals = [s.total_ms for s in ok_samples if math.isfinite(s.total_ms)]
+        if total_vals:
+            metrics["total_p50_ms"] = Percentiles(total_vals).p50
+
+        # Energy / power summary from telemetry (None on plugins that haven't
+        # threaded a TelemetryWindow through yet). Mirrors llm-inference.
+        if energy is not None:
+            if energy.gpu_power_avg_w > 0:
+                metrics["power_avg_w"] = energy.gpu_power_avg_w
+                metrics["power_peak_w"] = energy.gpu_power_peak_w
+            if energy.total_energy_joules > 0:
+                metrics["energy_joules_total"] = energy.total_energy_joules
+                if energy.joules_per_token == energy.joules_per_token:  # not NaN
+                    metrics["joules_per_token"] = energy.joules_per_token
+
+        builder = EnvelopeBuilder(
+            suite_id=spec.benchmark_id,
+            suite_version=spec.suite_version,
+            model=ModelConfig(
+                id=context.model_id,
+                revision=context.model_revision,
+                provider=context.engine_kind.value,
+                endpoint_hash="0" * 64,
+            ),
+            engine=EngineConfig(
+                name=context.engine_kind.value,
+                version=context.engine_version or "unknown",
+                config_hash="0" * 64,
+            ),
+            hardware_fingerprint=hw,
+            software_provenance=sw,
+            dataset=EnvDatasetSpec(id=spec.dataset.id, hash=dataset_hash),
+            seed=0,
+            quantization=(
+                Quantization(format=context.quantization_format)
+                if context.quantization_format
+                else None
+            ),
+            metrics=metrics,
+            slo_template=spec.slo_template,
+        )
+        return builder.build()

inferencebench_embeddings/schemas.py

@@ -0,0 +1,94 @@
+"""Pydantic schemas for embeddings-retrieval benchmark specs + run context."""
+
+from __future__ import annotations
+
+from enum import StrEnum
+from pathlib import Path
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class EngineKind(StrEnum):
+    """Engines this plugin can drive.
+
+    Production-grade paths for embeddings: HuggingFace's Text Embeddings
+    Inference (TEI) for self-hosted, plus the two big provider-hosted
+    options (OpenAI text-embedding-3, Cohere embed-english).
+    """
+
+    TEI = "tei"
+    OPENAI = "openai"
+    COHERE = "cohere"
+
+
+class DatasetConfig(BaseModel):
+    """Dataset under evaluation.
+
+    Fixture is a JSONL of query records — each with the relevant doc-id set
+    and a pointer to the corpus JSONL. Corpus is a sibling JSONL containing
+    ``{"doc_id", "text"}`` rows.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    id: Annotated[str, Field(min_length=1)]
+    path: Annotated[
+        str,
+        Field(
+            min_length=1,
+            description=("Path to the queries JSONL relative to the plugin's datasets/ directory."),
+        ),
+    ]
+    corpus_path: Annotated[
+        str,
+        Field(
+            min_length=1,
+            description=("Path to the corpus JSONL relative to the plugin's datasets/ directory."),
+        ),
+    ]
+
+
+class WarmupConfig(BaseModel):
+    """Warmup parameters.
+
+    Retrieval runs are per-query and order-independent; default is zero
+    discarded runs. Surfaced for future JIT-warmup of embedding servers.
+    """
+
+    model_config = ConfigDict(extra="forbid")
+    discard_runs: Annotated[int, Field(ge=0)] = 0
+
+
+class BenchmarkSpec(BaseModel):
+    """One retrieval benchmark — fixture + metric + metadata."""
+
+    model_config = ConfigDict(extra="forbid")
+    benchmark_id: Annotated[str, Field(min_length=1)]
+    suite_version: Annotated[str, Field(pattern=r"^\d+\.\d+\.\d+(-[\w.]+)?$")]
+    description: str = ""
+    modality: Literal["embeddings"] = "embeddings"
+    kind: Literal["retrieval"] = "retrieval"
+    dataset: DatasetConfig
+    slo_template: str = "embeddings.retrieval.standard"
+    warmup: WarmupConfig = Field(default_factory=WarmupConfig)
+    metric: Literal["recall_at_5", "mrr_at_10", "ndcg_at_10"] = "recall_at_5"
+
+
+class RunContext(BaseModel):
+    """Per-invocation context (where to send requests, where to write results).
+
+    Mirrors the llm-quality plugin shape so cross-plugin tooling can reuse
+    the same context object.
+    """
+
+    model_config = ConfigDict(extra="forbid", arbitrary_types_allowed=True)
+    model_id: Annotated[str, Field(min_length=1)]
+    model_revision: Annotated[str, Field(min_length=7, max_length=40)] = "unknown00"
+    engine_kind: EngineKind
+    engine_version: str = ""
+    base_url: str = ""
+    api_key: str = ""
+    quantization_format: str = ""
+    hardware_class: str = ""
+    output_dir: Path
+    extra: dict[str, str | int | float | bool] = Field(default_factory=dict)

inferencebench_embeddings/scoring.py

@@ -0,0 +1,98 @@
+"""Deterministic IR scoring strategies for the embeddings-retrieval plugin.
+
+Three pure functions, each ``(ranking, relevant, k) -> float`` in ``[0.0, 1.0]``.
+All higher-is-better — 1.0 means the ranking placed every relevant doc at the
+top k, 0.0 means none of the relevant docs appear in the top k.
+
+No real embedding model is invoked; these are standard IR metrics on
+already-produced rank lists.
+"""
+
+from __future__ import annotations
+
+import math
+from collections.abc import Callable, Iterable, Sequence
+
+MetricFn = Callable[[Sequence[str], Iterable[str], int], float]
+
+
+def _topk(ranking: Sequence[str], k: int) -> list[str]:
+    """Slice the top-k of a ranking; tolerate ``k`` exceeding the ranking length."""
+    if k <= 0:
+        return []
+    return list(ranking[:k])
+
+
+def recall_at_k(
+    ranking: Sequence[str],
+    relevant: Iterable[str],
+    k: int,
+) -> float:
+    """Fraction of relevant docs that appear in the top-k of the ranking.
+
+    Standard recall@k: |{relevant ∩ top-k}| / |relevant|. Returns 0.0 when
+    the relevant set is empty (vacuously no docs to recall). Capped at 1.0
+    by construction.
+    """
+    relevant_set = set(relevant)
+    if not relevant_set:
+        return 0.0
+    top = set(_topk(ranking, k))
+    hits = len(top & relevant_set)
+    return hits / len(relevant_set)
+
+
+def mrr_at_k(
+    ranking: Sequence[str],
+    relevant: Iterable[str],
+    k: int,
+) -> float:
+    """Reciprocal rank of the first relevant doc within top-k, or 0.0 if none.
+
+    Strictly speaking "MRR" is the mean of reciprocal ranks across queries,
+    but per-query the value used in that mean is ``1 / rank_of_first_hit``,
+    and that is what this function returns. The caller aggregates by taking
+    the mean across queries.
+    """
+    relevant_set = set(relevant)
+    if not relevant_set:
+        return 0.0
+    for idx, doc_id in enumerate(_topk(ranking, k), start=1):
+        if doc_id in relevant_set:
+            return 1.0 / idx
+    return 0.0
+
+
+def ndcg_at_k(
+    ranking: Sequence[str],
+    relevant: Iterable[str],
+    k: int,
+) -> float:
+    """Normalised discounted cumulative gain @ k with binary relevance.
+
+    Standard binary-relevance nDCG: ``DCG@k = sum_{i=1..k} rel_i / log2(i+1)``
+    where ``rel_i`` is 1 if the i-th ranked doc is relevant else 0. The
+    ideal DCG is the same sum when the top ``min(k, |relevant|)`` slots are
+    all relevant. nDCG = DCG / IDCG. Returns 0.0 for an empty relevant set.
+    """
+    relevant_set = set(relevant)
+    if not relevant_set:
+        return 0.0
+    top = _topk(ranking, k)
+    dcg = 0.0
+    for i, doc_id in enumerate(top, start=1):
+        if doc_id in relevant_set:
+            dcg += 1.0 / math.log2(i + 1)
+    ideal_hits = min(k, len(relevant_set))
+    idcg = sum(1.0 / math.log2(i + 1) for i in range(1, ideal_hits + 1))
+    if idcg == 0.0:
+        return 0.0
+    return dcg / idcg
+
+
+# Map metric-name -> ``(k, scorer)``. Scorer signature: (ranking, relevant, k).
+METRICS: dict[str, tuple[int, MetricFn]] = {
+    "recall_at_5": (5, recall_at_k),
+    "mrr_at_10": (10, mrr_at_k),
+    "ndcg_at_10": (10, ndcg_at_k),
+}

inferencebench_embeddings-0.0.2.dist-info/METADATA

@@ -0,0 +1,42 @@
+Metadata-Version: 2.4
+Name: inferencebench-embeddings
+Version: 0.0.2
+Summary: Embeddings retrieval plugin for InferenceBench Suite (deterministic ranking skeleton; real embedding-model invocation deferred)
+Project-URL: Homepage, https://github.com/yobitelcomm/bench
+Author-email: Yobitel Communications <bench@yobitel.com>
+License: Apache-2.0
+Keywords: ai,benchmark,embeddings,ml,ndcg,recall,retrieval
+Classifier: Development Status :: 2 - Pre-Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Python: >=3.12
+Requires-Dist: inferencebench-envelope
+Requires-Dist: inferencebench-harness
+Requires-Dist: pydantic~=2.9
+Requires-Dist: pyyaml~=6.0
+Description-Content-Type: text/markdown
+
+# inferencebench-embeddings
+
+Embeddings retrieval plugin for the InferenceBench Suite.
+
+Phase-2-quality skeleton: produces signed envelopes via deterministic
+hash-based rankings, with placeholders for real embedding-model invocation
+that future revisions wire to TEI / OpenAI / Cohere.
+
+Suite ID: `embeddings.retrieval`
+
+Bundled benchmarks:
+
+- `embeddings.retrieval.beir-mini` — 5 queries × 20-doc corpus, recall@5.
+- `embeddings.retrieval.long-doc` — 3 queries with longer documents, nDCG@10.
+
+The skeleton does NOT actually embed any text. For each query it ranks the
+corpus by `sha256(query + doc_id)`, then scores the top-k against the
+fixture's relevant set. This produces a real, well-defined retrieval metric
+in [0, 1] without external dependencies — future revisions replace the
+hash rank with a real vector search.

inferencebench_embeddings-0.0.2.dist-info/RECORD

@@ -0,0 +1,21 @@
+inferencebench_embeddings/__init__.py,sha256=kzGstyjov6oEF9lOw3YzTfeIQgRMTeqs6E3sJZlGx1I,366
+inferencebench_embeddings/plugin.py,sha256=sIJzyNm5y5rHTwxVCWQZ6Ng1X82gdLBBgc5ELYNJ3po,16004
+inferencebench_embeddings/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+inferencebench_embeddings/schemas.py,sha256=k4tSBcwhZP5sBVHZyngpppfdvLVLDjcffHPkY2DfAqY,2978
+inferencebench_embeddings/scoring.py,sha256=wNxG-p7Vu3CAQ7kkM1iMINVMyvB5PeKSWfJubLz6RCw,3114
+inferencebench_embeddings/benchmarks/beir-mini.yaml,sha256=V1QVIVxiUsL_39bFbwykx6xAC8F3YbSAv5hS0Xnl0Jw,344
+inferencebench_embeddings/benchmarks/long-doc.yaml,sha256=0psmhNkaNQMTTlxZLrFTlaoXJy4HQR1Wj5q0vNjtoyc,335
+inferencebench_embeddings/benchmarks/msmarco-style.yaml,sha256=4yCD6VhFM1rVt3Lkf4GEyQu1aI-38vHOr_6hTdwMOuw,363
+inferencebench_embeddings/benchmarks/query-expansion.yaml,sha256=dRsF-iQ3wDjzpAHC2oKjujI5RD72_Ip5uema8B1bleQ,381
+inferencebench_embeddings/datasets/beir-mini-corpus.jsonl,sha256=cuW1wXndbhFt32hnqbIf_hplFOLxz-k1JyV4eH42WAk,2105
+inferencebench_embeddings/datasets/beir-mini-queries.jsonl,sha256=6nF5YAp6hxc5xXZw0JPkX1BG4Egkqy1EkUF-YM-aXFA,435
+inferencebench_embeddings/datasets/long-doc-corpus.jsonl,sha256=zFB-GoJxVdt4hBfBLQF0xrh9HZm7U98GIyh7ZlkOZI4,4144
+inferencebench_embeddings/datasets/long-doc-queries.jsonl,sha256=zammwAV-PMxPIMbAHY4KQQY_AEK68uxhKOQ_NJj0t7c,315
+inferencebench_embeddings/datasets/msmarco-style-corpus.jsonl,sha256=WkKvrs3Oniz15_dUI-BdcLx4Or1EEQlTrKR68_9yiik,6120
+inferencebench_embeddings/datasets/msmarco-style-queries.jsonl,sha256=_xnZKNxF9GT55qhpoRPC3mJ-76e7sUkXFTD2UnpfiUg,436
+inferencebench_embeddings/datasets/query-expansion-corpus.jsonl,sha256=8cZoY5WXJ7pC3PUicaCSclsN1VNmbwqobzx26OmKmT0,1994
+inferencebench_embeddings/datasets/query-expansion-queries.jsonl,sha256=KIp0BV6ZSDQxKUQuFQYv_mZjFOMcpCwbGtJZt2IF85E,289
+inferencebench_embeddings-0.0.2.dist-info/METADATA,sha256=EQJrhBzV12NAzdV42ZSVlF1vHErbX5qlWJlic6C613E,1806
+inferencebench_embeddings-0.0.2.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87
+inferencebench_embeddings-0.0.2.dist-info/entry_points.txt,sha256=DYTJRaKRl0zDqR8kSExRfxN2vi7cKznhXSpEzl5ecEI,107
+inferencebench_embeddings-0.0.2.dist-info/RECORD,,

inferencebench_embeddings-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.29.0
+Root-Is-Purelib: true
+Tag: py3-none-any

inferencebench_embeddings-0.0.2.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[inferencebench.plugins]
+embeddings.retrieval = inferencebench_embeddings.plugin:EmbeddingsRetrievalPlugin