biblicus 0.9.0__py3-none-any.whl → 0.10.0__py3-none-any.whl

biblicus/__init__.py

@@ -27,4 +27,4 @@ __all__ = [
     "RetrievalRun",
 ]
 
-__version__ = "0.9.0"
+__version__ = "0.10.0"

biblicus/_vendor/dotyaml/__init__.py

@@ -11,4 +11,3 @@ from .interpolation import interpolate_env_vars
 from .loader import ConfigLoader, load_config
 
 __all__ = ["ConfigLoader", "interpolate_env_vars", "load_config"]
-

biblicus/_vendor/dotyaml/interpolation.py

@@ -60,4 +60,3 @@ def _interpolate_string(text: str) -> str:
         raise ValueError(f"Required environment variable '{env_var}' not found")
 
     return re.sub(pattern, replace_match, text)
-

biblicus/_vendor/dotyaml/loader.py

@@ -178,4 +178,3 @@ class ConfigLoader:
         for key, value in flat_config.items():
             if override or key not in os.environ:
                 os.environ[key] = value
-

biblicus/_vendor/dotyaml/transformer.py

@@ -132,4 +132,3 @@ def convert_string_to_value(value: str) -> Any:
         return json.loads(value)
     except (json.JSONDecodeError, ValueError):
         return value
-

biblicus/analysis/__init__.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 from typing import Dict, Type
 
 from .base import CorpusAnalysisBackend
+from .profiling import ProfilingBackend
 from .topic_modeling import TopicModelingBackend
 
 
@@ -18,6 +19,7 @@ def available_analysis_backends() -> Dict[str, Type[CorpusAnalysisBackend]]:
     :rtype: dict[str, Type[CorpusAnalysisBackend]]
     """
     return {
+        ProfilingBackend.analysis_id: ProfilingBackend,
         TopicModelingBackend.analysis_id: TopicModelingBackend,
     }
 

biblicus/analysis/models.py

@@ -84,6 +84,221 @@ class AnalysisRunManifest(AnalysisSchemaModel):
     stats: Dict[str, Any] = Field(default_factory=dict)
 
 
+class ProfilingRecipeConfig(AnalysisSchemaModel):
+    """
+    Recipe configuration for profiling analysis.
+
+    :ivar schema_version: Analysis schema version.
+    :vartype schema_version: int
+    :ivar sample_size: Optional sample size for distribution metrics.
+    :vartype sample_size: int or None
+    :ivar min_text_characters: Optional minimum character count for extracted text inclusion.
+    :vartype min_text_characters: int or None
+    :ivar percentiles: Percentiles to compute for distributions.
+    :vartype percentiles: list[int]
+    :ivar top_tag_count: Maximum number of tags to include in top tag output.
+    :vartype top_tag_count: int
+    :ivar tag_filters: Optional tag filters to limit tag coverage metrics.
+    :vartype tag_filters: list[str] or None
+    """
+
+    schema_version: int = Field(default=ANALYSIS_SCHEMA_VERSION, ge=1)
+    sample_size: Optional[int] = Field(default=None, ge=1)
+    min_text_characters: Optional[int] = Field(default=None, ge=1)
+    percentiles: List[int] = Field(default_factory=lambda: [50, 90, 99])
+    top_tag_count: int = Field(default=10, ge=1)
+    tag_filters: Optional[List[str]] = None
+
+    @model_validator(mode="after")
+    def _validate_schema_version(self) -> "ProfilingRecipeConfig":
+        if self.schema_version != ANALYSIS_SCHEMA_VERSION:
+            raise ValueError(f"Unsupported analysis schema version: {self.schema_version}")
+        return self
+
+    @field_validator("percentiles", mode="after")
+    @classmethod
+    def _validate_percentiles(cls, value: List[int]) -> List[int]:
+        if not value:
+            raise ValueError("percentiles must include at least one value")
+        if any(percentile < 1 or percentile > 100 for percentile in value):
+            raise ValueError("percentiles must be between 1 and 100")
+        if value != sorted(value):
+            raise ValueError("percentiles must be sorted in ascending order")
+        return value
+
+    @field_validator("tag_filters", mode="before")
+    @classmethod
+    def _validate_tag_filters(cls, value: object) -> object:
+        if value is None:
+            return None
+        if not isinstance(value, list):
+            raise ValueError("tag_filters must be a list of strings")
+        cleaned = [str(tag).strip() for tag in value]
+        if not cleaned or any(not tag for tag in cleaned):
+            raise ValueError("tag_filters must be a list of non-empty strings")
+        return cleaned
+
+
+class ProfilingPercentileValue(AnalysisSchemaModel):
+    """
+    Percentile entry for a distribution.
+
+    :ivar percentile: Percentile value between 1 and 100.
+    :vartype percentile: int
+    :ivar value: Percentile value.
+    :vartype value: float
+    """
+
+    percentile: int = Field(ge=1, le=100)
+    value: float
+
+
+class ProfilingDistributionReport(AnalysisSchemaModel):
+    """
+    Distribution summary for numeric values.
+
+    :ivar count: Count of values included.
+    :vartype count: int
+    :ivar min_value: Minimum value observed.
+    :vartype min_value: float
+    :ivar max_value: Maximum value observed.
+    :vartype max_value: float
+    :ivar mean_value: Mean value observed.
+    :vartype mean_value: float
+    :ivar percentiles: Percentile values.
+    :vartype percentiles: list[ProfilingPercentileValue]
+    """
+
+    count: int = Field(ge=0)
+    min_value: float
+    max_value: float
+    mean_value: float
+    percentiles: List[ProfilingPercentileValue] = Field(default_factory=list)
+
+
+class ProfilingTagCount(AnalysisSchemaModel):
+    """
+    Tag count entry for profiling output.
+
+    :ivar tag: Tag name.
+    :vartype tag: str
+    :ivar count: Number of items with this tag.
+    :vartype count: int
+    """
+
+    tag: str
+    count: int = Field(ge=0)
+
+
+class ProfilingTagReport(AnalysisSchemaModel):
+    """
+    Tag coverage summary for raw items.
+
+    :ivar tagged_items: Count of items with tags.
+    :vartype tagged_items: int
+    :ivar untagged_items: Count of items without tags.
+    :vartype untagged_items: int
+    :ivar total_unique_tags: Count of unique tags.
+    :vartype total_unique_tags: int
+    :ivar top_tags: Most frequent tags.
+    :vartype top_tags: list[ProfilingTagCount]
+    :ivar tag_filters: Optional tag filters applied.
+    :vartype tag_filters: list[str] or None
+    """
+
+    tagged_items: int = Field(ge=0)
+    untagged_items: int = Field(ge=0)
+    total_unique_tags: int = Field(ge=0)
+    top_tags: List[ProfilingTagCount] = Field(default_factory=list)
+    tag_filters: Optional[List[str]] = None
+
+
+class ProfilingRawItemsReport(AnalysisSchemaModel):
+    """
+    Summary of raw corpus items.
+
+    :ivar total_items: Total number of catalog items.
+    :vartype total_items: int
+    :ivar media_type_counts: Count of items per media type.
+    :vartype media_type_counts: dict[str, int]
+    :ivar bytes_distribution: Distribution of raw item sizes in bytes.
+    :vartype bytes_distribution: ProfilingDistributionReport
+    :ivar tags: Tag coverage summary.
+    :vartype tags: ProfilingTagReport
+    """
+
+    total_items: int = Field(ge=0)
+    media_type_counts: Dict[str, int] = Field(default_factory=dict)
+    bytes_distribution: ProfilingDistributionReport
+    tags: ProfilingTagReport
+
+
+class ProfilingExtractedTextReport(AnalysisSchemaModel):
+    """
+    Summary of extracted text coverage.
+
+    :ivar source_items: Count of source items in the extraction run.
+    :vartype source_items: int
+    :ivar extracted_nonempty_items: Count of extracted items with non-empty text.
+    :vartype extracted_nonempty_items: int
+    :ivar extracted_empty_items: Count of extracted items with empty text.
+    :vartype extracted_empty_items: int
+    :ivar extracted_missing_items: Count of items with no extracted text artifact.
+    :vartype extracted_missing_items: int
+    :ivar characters_distribution: Distribution of extracted text lengths.
+    :vartype characters_distribution: ProfilingDistributionReport
+    """
+
+    source_items: int = Field(ge=0)
+    extracted_nonempty_items: int = Field(ge=0)
+    extracted_empty_items: int = Field(ge=0)
+    extracted_missing_items: int = Field(ge=0)
+    characters_distribution: ProfilingDistributionReport
+
+
+class ProfilingReport(AnalysisSchemaModel):
+    """
+    Report for profiling analysis.
+
+    :ivar raw_items: Raw corpus item summary.
+    :vartype raw_items: ProfilingRawItemsReport
+    :ivar extracted_text: Extracted text coverage summary.
+    :vartype extracted_text: ProfilingExtractedTextReport
+    :ivar warnings: Warning messages.
+    :vartype warnings: list[str]
+    :ivar errors: Error messages.
+    :vartype errors: list[str]
+    """
+
+    raw_items: ProfilingRawItemsReport
+    extracted_text: ProfilingExtractedTextReport
+    warnings: List[str] = Field(default_factory=list)
+    errors: List[str] = Field(default_factory=list)
+
+
+class ProfilingOutput(AnalysisSchemaModel):
+    """
+    Output bundle for profiling analysis.
+
+    :ivar schema_version: Analysis schema version.
+    :vartype schema_version: int
+    :ivar analysis_id: Analysis backend identifier.
+    :vartype analysis_id: str
+    :ivar generated_at: International Organization for Standardization 8601 timestamp for output creation.
+    :vartype generated_at: str
+    :ivar run: Analysis run manifest.
+    :vartype run: AnalysisRunManifest
+    :ivar report: Profiling report data.
+    :vartype report: ProfilingReport
+    """
+
+    schema_version: int = Field(default=ANALYSIS_SCHEMA_VERSION, ge=1)
+    analysis_id: str
+    generated_at: str
+    run: AnalysisRunManifest
+    report: ProfilingReport
+
+
 class TopicModelingTextSourceConfig(AnalysisSchemaModel):
     """
     Configuration for text collection within topic modeling.
@@ -124,7 +339,9 @@ class TopicModelingLlmExtractionConfig(AnalysisSchemaModel):
     """
 
     enabled: bool = Field(default=False)
-    method: TopicModelingLlmExtractionMethod = Field(default=TopicModelingLlmExtractionMethod.SINGLE)
+    method: TopicModelingLlmExtractionMethod = Field(
+        default=TopicModelingLlmExtractionMethod.SINGLE
+    )
     client: Optional[LlmClientConfig] = None
     prompt_template: Optional[str] = None
     system_prompt: Optional[str] = None
@@ -136,7 +353,9 @@ class TopicModelingLlmExtractionConfig(AnalysisSchemaModel):
             return value
         if isinstance(value, str):
             return TopicModelingLlmExtractionMethod(value)
-        raise ValueError("llm_extraction.method must be a string or TopicModelingLlmExtractionMethod")
+        raise ValueError(
+            "llm_extraction.method must be a string or TopicModelingLlmExtractionMethod"
+        )
 
     @model_validator(mode="after")
     def _validate_requirements(self) -> "TopicModelingLlmExtractionConfig":
@@ -188,7 +407,9 @@ class TopicModelingVectorizerConfig(AnalysisSchemaModel):
     def _validate_ngram_range(self) -> "TopicModelingVectorizerConfig":
         start, end = self.ngram_range
         if start < 1 or end < start:
-            raise ValueError("vectorizer.ngram_range must include two integers with start >= 1 and end >= start")
+            raise ValueError(
+                "vectorizer.ngram_range must include two integers with start >= 1 and end >= start"
+            )
         return self
 
     @field_validator("stop_words", mode="before")
@@ -201,7 +422,7 @@ class TopicModelingVectorizerConfig(AnalysisSchemaModel):
                 raise ValueError("vectorizer.stop_words must be 'english' or a list of strings")
             return value
         if isinstance(value, list):
-            if not all(isinstance(entry, str) and entry for entry in value):
+            if not value or not all(isinstance(entry, str) and entry for entry in value):
                 raise ValueError("vectorizer.stop_words must be 'english' or a list of strings")
             return value
         raise ValueError("vectorizer.stop_words must be 'english' or a list of strings")
@@ -280,7 +501,9 @@ class TopicModelingRecipeConfig(AnalysisSchemaModel):
     """
 
     schema_version: int = Field(default=ANALYSIS_SCHEMA_VERSION, ge=1)
-    text_source: TopicModelingTextSourceConfig = Field(default_factory=TopicModelingTextSourceConfig)
+    text_source: TopicModelingTextSourceConfig = Field(
+        default_factory=TopicModelingTextSourceConfig
+    )
     llm_extraction: TopicModelingLlmExtractionConfig = Field(
         default_factory=TopicModelingLlmExtractionConfig
     )

biblicus/analysis/profiling.py

@@ -0,0 +1,337 @@
+"""
+Profiling analysis backend for Biblicus.
+"""
+
+from __future__ import annotations
+
+import json
+import math
+from pathlib import Path
+from typing import Dict, Iterable, List, Sequence
+
+from pydantic import BaseModel
+
+from ..corpus import Corpus
+from ..models import CatalogItem, ExtractionRunReference
+from ..retrieval import hash_text
+from ..time import utc_now_iso
+from .base import CorpusAnalysisBackend
+from .models import (
+    AnalysisRecipeManifest,
+    AnalysisRunInput,
+    AnalysisRunManifest,
+    ProfilingDistributionReport,
+    ProfilingExtractedTextReport,
+    ProfilingOutput,
+    ProfilingPercentileValue,
+    ProfilingRawItemsReport,
+    ProfilingRecipeConfig,
+    ProfilingReport,
+    ProfilingTagCount,
+    ProfilingTagReport,
+)
+
+
+class ProfilingBackend(CorpusAnalysisBackend):
+    """
+    Profiling analysis backend for corpus composition and coverage.
+
+    :ivar analysis_id: Backend identifier.
+    :vartype analysis_id: str
+    """
+
+    analysis_id = "profiling"
+
+    def run_analysis(
+        self,
+        corpus: Corpus,
+        *,
+        recipe_name: str,
+        config: Dict[str, object],
+        extraction_run: ExtractionRunReference,
+    ) -> BaseModel:
+        """
+        Run the profiling analysis pipeline.
+
+        :param corpus: Corpus to analyze.
+        :type corpus: Corpus
+        :param recipe_name: Human-readable recipe name.
+        :type recipe_name: str
+        :param config: Analysis configuration values.
+        :type config: dict[str, object]
+        :param extraction_run: Extraction run reference for text inputs.
+        :type extraction_run: biblicus.models.ExtractionRunReference
+        :return: Profiling output model.
+        :rtype: pydantic.BaseModel
+        """
+        parsed_config = (
+            config
+            if isinstance(config, ProfilingRecipeConfig)
+            else ProfilingRecipeConfig.model_validate(config)
+        )
+        return _run_profiling(
+            corpus=corpus,
+            recipe_name=recipe_name,
+            config=parsed_config,
+            extraction_run=extraction_run,
+        )
+
+
+def _run_profiling(
+    *,
+    corpus: Corpus,
+    recipe_name: str,
+    config: ProfilingRecipeConfig,
+    extraction_run: ExtractionRunReference,
+) -> ProfilingOutput:
+    recipe = _create_recipe_manifest(name=recipe_name, config=config)
+    catalog = corpus.load_catalog()
+    run_id = _analysis_run_id(
+        recipe_id=recipe.recipe_id,
+        extraction_run=extraction_run,
+        catalog_generated_at=catalog.generated_at,
+    )
+    run_manifest = AnalysisRunManifest(
+        run_id=run_id,
+        recipe=recipe,
+        corpus_uri=catalog.corpus_uri,
+        catalog_generated_at=catalog.generated_at,
+        created_at=utc_now_iso(),
+        input=AnalysisRunInput(extraction_run=extraction_run),
+        artifact_paths=[],
+        stats={},
+    )
+    run_dir = corpus.analysis_run_dir(analysis_id=ProfilingBackend.analysis_id, run_id=run_id)
+    output_path = run_dir / "output.json"
+    run_dir.mkdir(parents=True, exist_ok=True)
+
+    ordered_items = _ordered_catalog_items(catalog.items, catalog.order)
+    raw_report = _build_raw_items_report(items=ordered_items, config=config)
+    extracted_report = _build_extracted_text_report(
+        corpus=corpus,
+        extraction_run=extraction_run,
+        config=config,
+    )
+
+    report = ProfilingReport(
+        raw_items=raw_report,
+        extracted_text=extracted_report,
+        warnings=[],
+        errors=[],
+    )
+
+    run_stats = {
+        "raw_items": raw_report.total_items,
+        "extracted_nonempty_items": extracted_report.extracted_nonempty_items,
+        "extracted_missing_items": extracted_report.extracted_missing_items,
+    }
+    run_manifest = run_manifest.model_copy(
+        update={"artifact_paths": ["output.json"], "stats": run_stats}
+    )
+    _write_analysis_run_manifest(run_dir=run_dir, manifest=run_manifest)
+
+    output = ProfilingOutput(
+        analysis_id=ProfilingBackend.analysis_id,
+        generated_at=utc_now_iso(),
+        run=run_manifest,
+        report=report,
+    )
+    _write_profiling_output(path=output_path, output=output)
+    return output
+
+
+def _create_recipe_manifest(*, name: str, config: ProfilingRecipeConfig) -> AnalysisRecipeManifest:
+    recipe_payload = json.dumps(
+        {
+            "analysis_id": ProfilingBackend.analysis_id,
+            "name": name,
+            "config": config.model_dump(),
+        },
+        sort_keys=True,
+    )
+    recipe_id = hash_text(recipe_payload)
+    return AnalysisRecipeManifest(
+        recipe_id=recipe_id,
+        analysis_id=ProfilingBackend.analysis_id,
+        name=name,
+        created_at=utc_now_iso(),
+        config=config.model_dump(),
+    )
+
+
+def _analysis_run_id(
+    *, recipe_id: str, extraction_run: ExtractionRunReference, catalog_generated_at: str
+) -> str:
+    run_seed = f"{recipe_id}:{extraction_run.as_string()}:{catalog_generated_at}"
+    return hash_text(run_seed)
+
+
+def _ordered_catalog_items(
+    items: Dict[str, CatalogItem],
+    order: Sequence[str],
+) -> List[CatalogItem]:
+    ordered: List[CatalogItem] = []
+    seen = set()
+    for item_id in order:
+        item = items.get(item_id)
+        if item is None:
+            continue
+        ordered.append(item)
+        seen.add(item_id)
+    for item_id in sorted(items):
+        if item_id in seen:
+            continue
+        ordered.append(items[item_id])
+    return ordered
+
+
+def _build_raw_items_report(
+    *, items: Sequence[CatalogItem], config: ProfilingRecipeConfig
+) -> ProfilingRawItemsReport:
+    media_type_counts: Dict[str, int] = {}
+    for item in items:
+        media_type_counts[item.media_type] = media_type_counts.get(item.media_type, 0) + 1
+
+    bytes_values = [item.bytes for item in _apply_sample(items, config.sample_size)]
+    bytes_distribution = _build_distribution(bytes_values, config.percentiles)
+    tag_report = _build_tag_report(items=items, config=config)
+
+    return ProfilingRawItemsReport(
+        total_items=len(items),
+        media_type_counts=media_type_counts,
+        bytes_distribution=bytes_distribution,
+        tags=tag_report,
+    )
+
+
+def _build_tag_report(
+    *, items: Sequence[CatalogItem], config: ProfilingRecipeConfig
+) -> ProfilingTagReport:
+    tag_filters = config.tag_filters
+    tag_filter_set = set(tag_filters or [])
+    tag_counts: Dict[str, int] = {}
+    tagged_items = 0
+
+    for item in items:
+        tags = list(item.tags)
+        if tag_filters is not None:
+            tags = [tag for tag in tags if tag in tag_filter_set]
+        if tags:
+            tagged_items += 1
+        for tag in tags:
+            tag_counts[tag] = tag_counts.get(tag, 0) + 1
+
+    untagged_items = len(items) - tagged_items
+    top_tags = sorted(tag_counts.items(), key=lambda entry: (-entry[1], entry[0]))
+    top_tags = top_tags[: config.top_tag_count]
+    return ProfilingTagReport(
+        tagged_items=tagged_items,
+        untagged_items=untagged_items,
+        total_unique_tags=len(tag_counts),
+        top_tags=[ProfilingTagCount(tag=tag, count=count) for tag, count in top_tags],
+        tag_filters=tag_filters,
+    )
+
+
+def _build_extracted_text_report(
+    *,
+    corpus: Corpus,
+    extraction_run: ExtractionRunReference,
+    config: ProfilingRecipeConfig,
+) -> ProfilingExtractedTextReport:
+    manifest = corpus.load_extraction_run_manifest(
+        extractor_id=extraction_run.extractor_id,
+        run_id=extraction_run.run_id,
+    )
+    nonempty_items = 0
+    empty_items = 0
+    missing_items = 0
+    text_lengths: List[int] = []
+    text_dir = corpus.extraction_run_dir(
+        extractor_id=extraction_run.extractor_id,
+        run_id=extraction_run.run_id,
+    )
+
+    for item_result in manifest.items:
+        if item_result.status != "extracted" or item_result.final_text_relpath is None:
+            missing_items += 1
+            continue
+        text_path = text_dir / item_result.final_text_relpath
+        text_value = text_path.read_text(encoding="utf-8")
+        stripped = text_value.strip()
+        if not stripped:
+            empty_items += 1
+            continue
+        if config.min_text_characters is not None and len(stripped) < config.min_text_characters:
+            empty_items += 1
+            continue
+        nonempty_items += 1
+        text_lengths.append(len(text_value))
+
+    sampled_lengths = _apply_sample(text_lengths, config.sample_size)
+    characters_distribution = _build_distribution(sampled_lengths, config.percentiles)
+    return ProfilingExtractedTextReport(
+        source_items=len(manifest.items),
+        extracted_nonempty_items=nonempty_items,
+        extracted_empty_items=empty_items,
+        extracted_missing_items=missing_items,
+        characters_distribution=characters_distribution,
+    )
+
+
+def _apply_sample(values: Sequence, sample_size: int | None) -> List:
+    if sample_size is None:
+        return list(values)
+    return list(values[:sample_size])
+
+
+def _build_distribution(
+    values: Sequence[int], percentiles: Iterable[int]
+) -> ProfilingDistributionReport:
+    if not values:
+        percentile_values = [
+            ProfilingPercentileValue(percentile=percentile, value=0.0) for percentile in percentiles
+        ]
+        return ProfilingDistributionReport(
+            count=0,
+            min_value=0.0,
+            max_value=0.0,
+            mean_value=0.0,
+            percentiles=percentile_values,
+        )
+    sorted_values = sorted(values)
+    count = len(sorted_values)
+    min_value = float(sorted_values[0])
+    max_value = float(sorted_values[-1])
+    mean_value = float(sum(sorted_values)) / count
+    percentile_values = [
+        ProfilingPercentileValue(
+            percentile=percentile,
+            value=float(_percentile_value(sorted_values, percentile)),
+        )
+        for percentile in percentiles
+    ]
+    return ProfilingDistributionReport(
+        count=count,
+        min_value=min_value,
+        max_value=max_value,
+        mean_value=mean_value,
+        percentiles=percentile_values,
+    )
+
+
+def _percentile_value(sorted_values: Sequence[int], percentile: int) -> int:
+    if not sorted_values:
+        return 0
+    index = max(0, math.ceil((percentile / 100) * len(sorted_values)) - 1)
+    index = min(index, len(sorted_values) - 1)
+    return int(sorted_values[index])
+
+
+def _write_analysis_run_manifest(*, run_dir: Path, manifest: AnalysisRunManifest) -> None:
+    manifest_path = run_dir / "manifest.json"
+    manifest_path.write_text(manifest.model_dump_json(indent=2) + "\n", encoding="utf-8")
+
+
+def _write_profiling_output(*, path: Path, output: ProfilingOutput) -> None:
+    path.write_text(output.model_dump_json(indent=2) + "\n", encoding="utf-8")

biblicus/analysis/topic_modeling.py

@@ -452,7 +452,7 @@ def _run_bertopic(
             except ImportError as import_error:
                 raise ValueError(
                     "Vectorizer configuration requires scikit-learn. "
-                    "Install with pip install \"biblicus[topic-modeling]\"."
+                    'Install with pip install "biblicus[topic-modeling]".'
                 ) from import_error
             bertopic_kwargs["vectorizer_model"] = CountVectorizer(
                 ngram_range=tuple(config.vectorizer.ngram_range),
@@ -504,13 +504,10 @@ def _group_documents_by_topic(
     return grouped
 
 
-def _resolve_topic_keywords(
-    *, topic_model: Any, topic_id: int
-) -> List[TopicModelingKeyword]:
+def _resolve_topic_keywords(*, topic_model: Any, topic_id: int) -> List[TopicModelingKeyword]:
     raw_keywords = topic_model.get_topic(topic_id) or []
     return [
-        TopicModelingKeyword(keyword=str(entry[0]), score=float(entry[1]))
-        for entry in raw_keywords
+        TopicModelingKeyword(keyword=str(entry[0]), score=float(entry[1])) for entry in raw_keywords
     ]
 
 

biblicus/backends/sqlite_full_text_search.py

@@ -213,8 +213,7 @@ def _create_full_text_search_schema(conn: sqlite3.Connection) -> None:
     :return: None.
     :rtype: None
     """
-    conn.execute(
-        """
+    conn.execute("""
         CREATE VIRTUAL TABLE chunks_full_text_search USING fts5(
             content,
             item_id UNINDEXED,
@@ -225,8 +224,7 @@ def _create_full_text_search_schema(conn: sqlite3.Connection) -> None:
             start_offset UNINDEXED,
             end_offset UNINDEXED
         )
-        """
-    )
+        """)
 
 
 def _build_full_text_search_index(

biblicus/cli.py

@@ -563,7 +563,9 @@ def cmd_context_pack_build(arguments: argparse.Namespace) -> int:
     """
     input_text = sys.stdin.read()
     if not input_text.strip():
-        raise ValueError("Context pack build requires a retrieval result JavaScript Object Notation on standard input")
+        raise ValueError(
+            "Context pack build requires a retrieval result JavaScript Object Notation on standard input"
+        )
     retrieval_result = RetrievalResult.model_validate_json(input_text)
     join_with = bytes(arguments.join_with, "utf-8").decode("unicode_escape")
     policy = ContextPackPolicy(join_with=join_with)
@@ -685,6 +687,58 @@ def cmd_analyze_topics(arguments: argparse.Namespace) -> int:
     return 0
 
 
+def cmd_analyze_profile(arguments: argparse.Namespace) -> int:
+    """
+    Run profiling analysis for a corpus.
+
+    :param arguments: Parsed command-line interface arguments.
+    :type arguments: argparse.Namespace
+    :return: Exit code.
+    :rtype: int
+    """
+    import yaml
+
+    corpus = (
+        Corpus.open(arguments.corpus)
+        if getattr(arguments, "corpus", None)
+        else Corpus.find(Path.cwd())
+    )
+
+    recipe_data: dict[str, object] = {}
+    if arguments.recipe is not None:
+        recipe_path = Path(arguments.recipe)
+        if not recipe_path.is_file():
+            raise FileNotFoundError(f"Recipe file not found: {recipe_path}")
+        recipe_raw = yaml.safe_load(recipe_path.read_text(encoding="utf-8")) or {}
+        if not isinstance(recipe_raw, dict):
+            raise ValueError("Profiling recipe must be a mapping/object")
+        recipe_data = recipe_raw
+
+    if arguments.extraction_run:
+        extraction_run = parse_extraction_run_reference(arguments.extraction_run)
+    else:
+        extraction_run = corpus.latest_extraction_run_reference()
+        if extraction_run is None:
+            raise ValueError("Profiling analysis requires an extraction run to supply text inputs")
+        print(
+            "Warning: using latest extraction run; pass --extraction-run for reproducibility.",
+            file=sys.stderr,
+        )
+
+    backend = get_analysis_backend("profiling")
+    try:
+        output = backend.run_analysis(
+            corpus,
+            recipe_name=arguments.recipe_name,
+            config=recipe_data,
+            extraction_run=extraction_run,
+        )
+    except ValidationError as exc:
+        raise ValueError(f"Invalid profiling recipe: {exc}") from exc
+    print(output.model_dump_json(indent=2))
+    return 0
+
+
 def build_parser() -> argparse.ArgumentParser:
     """
     Build the command-line interface argument parser.
@@ -890,14 +944,20 @@ def build_parser() -> argparse.ArgumentParser:
 
     p_crawl = sub.add_parser("crawl", help="Crawl a website prefix into the corpus.")
     _add_common_corpus_arg(p_crawl)
-    p_crawl.add_argument("--root-url", required=True, help="Root uniform resource locator to fetch.")
+    p_crawl.add_argument(
+        "--root-url", required=True, help="Root uniform resource locator to fetch."
+    )
     p_crawl.add_argument(
         "--allowed-prefix",
         required=True,
         help="Uniform resource locator prefix that limits which links are eligible for crawl.",
     )
-    p_crawl.add_argument("--max-items", type=int, default=50, help="Maximum number of items to store.")
-    p_crawl.add_argument("--tags", default=None, help="Comma-separated tags to apply to stored items.")
+    p_crawl.add_argument(
+        "--max-items", type=int, default=50, help="Maximum number of items to store."
+    )
+    p_crawl.add_argument(
+        "--tags", default=None, help="Comma-separated tags to apply to stored items."
+    )
     p_crawl.add_argument("--tag", action="append", help="Repeatable tag to apply to stored items.")
     p_crawl.set_defaults(func=cmd_crawl)
 
@@ -923,6 +983,25 @@ def build_parser() -> argparse.ArgumentParser:
     )
     p_analyze_topics.set_defaults(func=cmd_analyze_topics)
 
+    p_analyze_profile = analyze_sub.add_parser("profile", help="Run profiling analysis.")
+    _add_common_corpus_arg(p_analyze_profile)
+    p_analyze_profile.add_argument(
+        "--recipe",
+        default=None,
+        help="Optional profiling recipe YAML file.",
+    )
+    p_analyze_profile.add_argument(
+        "--recipe-name",
+        default="default",
+        help="Human-readable recipe name.",
+    )
+    p_analyze_profile.add_argument(
+        "--extraction-run",
+        default=None,
+        help="Extraction run reference in the form extractor_id:run_id.",
+    )
+    p_analyze_profile.set_defaults(func=cmd_analyze_profile)
+
     return parser
 
 

biblicus/corpus.py

@@ -622,7 +622,9 @@ class Corpus:
         data = json.loads(manifest_path.read_text(encoding="utf-8"))
         return ExtractionRunManifest.model_validate(data)
 
-    def list_extraction_runs(self, *, extractor_id: Optional[str] = None) -> List[ExtractionRunListEntry]:
+    def list_extraction_runs(
+        self, *, extractor_id: Optional[str] = None
+    ) -> List[ExtractionRunListEntry]:
         """
         List extraction runs stored under the corpus.
 
@@ -669,7 +671,9 @@ class Corpus:
                     )
                 )
 
-        entries.sort(key=lambda entry: (entry.created_at, entry.extractor_id, entry.run_id), reverse=True)
+        entries.sort(
+            key=lambda entry: (entry.created_at, entry.extractor_id, entry.run_id), reverse=True
+        )
         return entries
 
     def latest_extraction_run_reference(
@@ -1366,7 +1370,9 @@ class Corpus:
         """
         _ = filename
         item_id = str(uuid.uuid4())
-        destination_relpath = str(Path(DEFAULT_RAW_DIR) / "imports" / "crawl" / crawl_id / relative_path)
+        destination_relpath = str(
+            Path(DEFAULT_RAW_DIR) / "imports" / "crawl" / crawl_id / relative_path
+        )
         destination_path = (self.root / destination_relpath).resolve()
         destination_path.parent.mkdir(parents=True, exist_ok=True)
         destination_path.write_bytes(data)

biblicus/evidence_processing.py

@@ -99,7 +99,10 @@ class EvidenceRerankLongestText(EvidenceReranker):
         """
         return sorted(
             evidence,
-            key=lambda evidence_item: (-len((evidence_item.text or "").strip()), evidence_item.item_id),
+            key=lambda evidence_item: (
+                -len((evidence_item.text or "").strip()),
+                evidence_item.item_id,
+            ),
         )
 
 
@@ -198,4 +201,3 @@ def apply_evidence_filter(
     """
     evidence_filter = _EVIDENCE_FILTERS[filter_id]
     return evidence_filter.filter(query_text=query_text, evidence=evidence, config=config)
-

biblicus/extraction.py

@@ -345,7 +345,9 @@ def build_extraction_run(
     manifest = create_extraction_run_manifest(corpus, recipe=recipe)
     run_dir = corpus.extraction_run_dir(extractor_id=extractor_id, run_id=manifest.run_id)
     if run_dir.exists():
-        return corpus.load_extraction_run_manifest(extractor_id=extractor_id, run_id=manifest.run_id)
+        return corpus.load_extraction_run_manifest(
+            extractor_id=extractor_id, run_id=manifest.run_id
+        )
     run_dir.mkdir(parents=True, exist_ok=False)
 
     catalog = corpus.load_catalog()

biblicus/extractors/markitdown_text.py

@@ -29,6 +29,7 @@ class MarkItDownExtractorConfig(BaseModel):
 
     enable_plugins: bool = Field(default=False)
 
+
 class MarkItDownExtractor(TextExtractor):
     """
     Extractor plugin backed by the `markitdown` library.

biblicus/extractors/paddleocr_vl_text.py

@@ -152,9 +152,7 @@ class PaddleOcrVlExtractor(TextExtractor):
                 parsed_config.backend.api_provider,
                 config_override=parsed_config.backend.api_key,
             )
-            text, confidence = self._extract_via_api(
-                source_path, parsed_config, api_key
-            )
+            text, confidence = self._extract_via_api(source_path, parsed_config, api_key)
 
         return ExtractedText(
             text=text,

biblicus/user_config.py

@@ -170,9 +170,7 @@ def resolve_openai_api_key(*, config: Optional[BiblicusUserConfig] = None) -> Op
     return loaded.openai.api_key
 
 
-def resolve_huggingface_api_key(
-    *, config: Optional[BiblicusUserConfig] = None
-) -> Optional[str]:
+def resolve_huggingface_api_key(*, config: Optional[BiblicusUserConfig] = None) -> Optional[str]:
     """
     Resolve a HuggingFace API key from environment or user configuration.
 
@@ -192,9 +190,7 @@ def resolve_huggingface_api_key(
     return loaded.huggingface.api_key
 
 
-def resolve_deepgram_api_key(
-    *, config: Optional[BiblicusUserConfig] = None
-) -> Optional[str]:
+def resolve_deepgram_api_key(*, config: Optional[BiblicusUserConfig] = None) -> Optional[str]:
     """
     Resolve a Deepgram API key from environment or user configuration.
 

{biblicus-0.9.0.dist-info → biblicus-0.10.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: biblicus
-Version: 0.9.0
+Version: 0.10.0
 Summary: Command line interface and Python library for corpus ingestion, retrieval, and evaluation.
 License: MIT
 Requires-Python: >=3.9
@@ -531,12 +531,13 @@ For detailed documentation on all extractors, see the [Extractor Reference][extr
 
 ## Topic modeling analysis
 
-Biblicus can run analysis pipelines on extracted text without changing the raw corpus. Topic modeling is the first
-analysis backend. It reads an extraction run, optionally applies an LLM-driven extraction pass, applies lexical
-processing, runs BERTopic, and optionally applies an LLM fine-tuning pass to label topics. The output is structured
-JavaScript Object Notation.
+Biblicus can run analysis pipelines on extracted text without changing the raw corpus. Profiling and topic modeling
+are the first analysis backends. Profiling summarizes corpus composition and extraction coverage. Topic modeling reads
+an extraction run, optionally applies an LLM-driven extraction pass, applies lexical processing, runs BERTopic, and
+optionally applies an LLM fine-tuning pass to label topics. The output is structured JavaScript Object Notation.
 
-See `docs/ANALYSIS.md` for the analysis pipeline overview and `docs/TOPIC_MODELING.md` for topic modeling details.
+See `docs/ANALYSIS.md` for the analysis pipeline overview, `docs/PROFILING.md` for profiling, and
+`docs/TOPIC_MODELING.md` for topic modeling details.
 
 Run a topic analysis using a recipe file:
 

{biblicus-0.9.0.dist-info → biblicus-0.10.0.dist-info}/RECORD

@@ -1,14 +1,14 @@
-biblicus/__init__.py,sha256=x14R9a_6nu3qTg2F-sUOaS_ZepXNBPpa3nsEgp4PZhg,495
+biblicus/__init__.py,sha256=BejOPHIlCnT74pu9fNuLm14HsmWjGqCIwpfD9hDOqSo,496
 biblicus/__main__.py,sha256=ipfkUoTlocVnrQDM69C7TeBqQxmHVeiWMRaT3G9rtnk,117
-biblicus/cli.py,sha256=GVmZlCSZPUMBbq69yjN16f4xNw71edlFbGPHX3300oI,32643
+biblicus/cli.py,sha256=aH3plnednnYgcPnSoYQf200nboKc6N-tuc3FuLPQEcU,35132
 biblicus/constants.py,sha256=-JaHI3Dngte2drawx93cGWxFVobbgIuaVhmjUJpf4GI,333
 biblicus/context.py,sha256=qnT9CH7_ldoPcg-rxnUOtRhheOmpDAbF8uqhf8OdjC4,5832
-biblicus/corpus.py,sha256=Pq2OvXom7giwD1tuWoM3RhFnak5YFx5bCh6JTd6JYtI,55554
+biblicus/corpus.py,sha256=qSDnYJXhWlF2p_BbFLl6xtI53lIIPxwyKLLGLC432Sg,55612
 biblicus/crawl.py,sha256=n8rXBMnziBK9vtKQQCXYOpBzqsPCswj2PzVJUb370KY,6250
 biblicus/errors.py,sha256=uMajd5DvgnJ_-jq5sbeom1GV8DPUc-kojBaECFi6CsY,467
 biblicus/evaluation.py,sha256=5xWpb-8f49Osh9aHzo1ab3AXOmls3Imc5rdnEC0pN-8,8143
-biblicus/evidence_processing.py,sha256=EMv1AkV_Eufk-poBz9nRR1dZgC-QewvI-NrULBUGVGA,6074
-biblicus/extraction.py,sha256=20lRxz6Te6IcA4d-rfT4qjJtgRG_c4YvrqfXNA7EYfs,19738
+biblicus/evidence_processing.py,sha256=sJe6T1nLxvU0xs9yMH8JZZS19zHXMR-Fpr5lWi5ndUM,6120
+biblicus/extraction.py,sha256=qvrsq6zSz2Kg-cap-18HPHC9pQlqEGo7pyID2uKCyBo,19760
 biblicus/frontmatter.py,sha256=JOGjIDzbbOkebQw2RzA-3WDVMAMtJta2INjS4e7-LMg,2463
 biblicus/hook_logging.py,sha256=IMvde-JhVWrx9tNz3eDJ1CY_rr5Sj7DZ2YNomYCZbz0,5366
 biblicus/hook_manager.py,sha256=ZCAkE5wLvn4lnQz8jho_o0HGEC9KdQd9qitkAEUQRcw,6997
@@ -21,30 +21,31 @@ biblicus/retrieval.py,sha256=A1SI4WK5cX-WbtN6FJ0QQxqlEOtQhddLrL0LZIuoTC4,4180
 biblicus/sources.py,sha256=EFy8-rQNLsyzz-98mH-z8gEHMYbqigcNFKLaR92KfDE,7241
 biblicus/time.py,sha256=3BSKOSo7R10K-0Dzrbdtl3fh5_yShTYqfdlKvvdkx7M,485
 biblicus/uris.py,sha256=xXD77lqsT9NxbyzI1spX9Y5a3-U6sLYMnpeSAV7g-nM,2013
-biblicus/user_config.py,sha256=okK57CRmT0W_yrc45tMPRl_abT7-D96IOrCBZtKtumM,6507
-biblicus/_vendor/dotyaml/__init__.py,sha256=e4zbejeJRwlD4I0q3YvotMypO19lXqmT8iyU1q6SvhY,376
-biblicus/_vendor/dotyaml/interpolation.py,sha256=PfUAEEOTFobv7Ox0E6nAxht6BqhHIDe4hP32fZn5TOs,1992
-biblicus/_vendor/dotyaml/loader.py,sha256=KePkjyhKZSvQZphmlmlzTYZJBQsqL5qhtGV1y7G6wzM,5624
-biblicus/_vendor/dotyaml/transformer.py,sha256=2AKPS8DMOPuYtzmM-dlwIqVbARfbBH5jYV1m5qpR49E,3725
-biblicus/analysis/__init__.py,sha256=TrKsE2GmdZDr3OARo2poa9H0powo0bjiEEWVx0tZmEg,1192
+biblicus/user_config.py,sha256=UXUYBNUN4FR37ggZGJG1wv3K8XzsMR8pXW1T18lrivw,6495
+biblicus/_vendor/dotyaml/__init__.py,sha256=WAWdbFNFqO5cJPthxA8Kx-L76Bh07sKMosUxC_3o9qA,375
+biblicus/_vendor/dotyaml/interpolation.py,sha256=FVUkdQr_KbXjoFPvGTv6I5v0X5iZkJe5yhZtYKRbYzI,1991
+biblicus/_vendor/dotyaml/loader.py,sha256=zy_zinR5fiatmRyZSiELHv1vVz1Y2eRSboSf_x3kfi4,5623
+biblicus/_vendor/dotyaml/transformer.py,sha256=RWNrm_KAsanG409HEIWquTH9i_jz-ZFK9fM86emXeF4,3724
+biblicus/analysis/__init__.py,sha256=Z4Wb4d-EoUuGHkcfRm9ILuZ8vr9FBqRxC0u1i6Fp_0w,1288
 biblicus/analysis/base.py,sha256=gB4ilvyMpiWU1m_ydy2dIHGP96ZFIFvVUL9iVDZKPJM,1265
 biblicus/analysis/llm.py,sha256=VjkZDKauHCDfj-TP-bTbI6a9WAXEIDe8bEiwErPx9xc,3309
-biblicus/analysis/models.py,sha256=4N8abx2kSMYYfckbq_QHl5YUnups3FFx5atepYR9cu4,19705
+biblicus/analysis/models.py,sha256=LuR52w27JRzV-Mr-WAOduZrBOCTrp5uYkMc46QHTRrI,27300
+biblicus/analysis/profiling.py,sha256=z4w14LVJrTEXcQ3PBNwwb_61KuuwQgXw4-EiAaxOQ4Y,10672
 biblicus/analysis/schema.py,sha256=MCiAQJmijVk8iM8rOUYbzyaDwsMR-Oo86iZU5NCbDMM,435
-biblicus/analysis/topic_modeling.py,sha256=9jSZrlpPK44H4UMfig7YNs3pPc0pNAqu-i4OlXzHET8,19454
+biblicus/analysis/topic_modeling.py,sha256=ZGXvm2MyU6plxz2FE1RQU-3bra6QZ-t8EJj8kG1TW0M,19438
 biblicus/backends/__init__.py,sha256=wLXIumV51l6ZIKzjoKKeU7AgIxGOryG7T7ls3a_Fv98,1212
 biblicus/backends/base.py,sha256=Erfj9dXg0nkRKnEcNjHR9_0Ddb2B1NvbmRksVm_g1dU,1776
 biblicus/backends/scan.py,sha256=hdNnQWqi5IH6j95w30BZHxLJ0W9PTaOkqfWJuxCCEMI,12478
-biblicus/backends/sqlite_full_text_search.py,sha256=KgmwOiKvkA0pv7vD0V7bcOdDx_nZIOfuIN6Z4Ij7I68,16516
+biblicus/backends/sqlite_full_text_search.py,sha256=XFuIbEHYWMD9JkjgRZcgYH3kP3b4hRnJ3PwP8rSFjUU,16502
 biblicus/extractors/__init__.py,sha256=ci3oldbdQZ8meAfHccM48CqQtZsPSRg3HkPrBSZF15M,2673
 biblicus/extractors/base.py,sha256=ka-nz_1zHPr4TS9sU4JfOoY-PJh7lbHPBOEBrbQFGSc,2171
 biblicus/extractors/deepgram_stt.py,sha256=VI71i4lbE-EFHcvpNcCPRpT8z7A5IuaSrT1UaPyZ8UY,6323
 biblicus/extractors/docling_granite_text.py,sha256=aFNx-HubvaMmVJHbNqk3CR_ilSwN96-phkaENT6E2B0,6879
 biblicus/extractors/docling_smol_text.py,sha256=cSbQcT4O47MMcM6_pmQCvqgC5ferLvaxJnm3v9EQd0A,6811
-biblicus/extractors/markitdown_text.py,sha256=-7N8ebi3pYfNPnplccyy3qvsKi6uImC1xyo_dSDiD10,4546
+biblicus/extractors/markitdown_text.py,sha256=ZvN2TFh65icTTdzCe7L-ZB8zTPP2mxQ4MhOOqSc81Z0,4547
 biblicus/extractors/metadata_text.py,sha256=7FbEPp0K1mXc7FH1_c0KhPhPexF9U6eLd3TVY1vTp1s,3537
 biblicus/extractors/openai_stt.py,sha256=fggErIu6YN6tXbleNTuROhfYi7zDgMd2vD_ecXZ7eXs,7162
-biblicus/extractors/paddleocr_vl_text.py,sha256=augbxZ-kx22yHvFR1b6CUAS2I6ktXFsJx8nLWRfvdOA,11722
+biblicus/extractors/paddleocr_vl_text.py,sha256=59csxihkqK0lELpAtK2YLcfbSUvNGiuOw7CwPa_0l_c,11692
 biblicus/extractors/pass_through_text.py,sha256=DNxkCwpH2bbXjPGPEQwsx8kfqXi6rIxXNY_n3TU2-WI,2777
 biblicus/extractors/pdf_text.py,sha256=YtUphgLVxyWJXew6ZsJ8wBRh67Y5ri4ZTRlMmq3g1Bk,3255
 biblicus/extractors/pipeline.py,sha256=LY6eM3ypw50MDB2cPEQqZrjxkhVvIc6sv4UEhHdNDrE,3208
@@ -54,9 +55,9 @@ biblicus/extractors/select_override.py,sha256=gSpffFmn1ux9pGtFvHD5Uu_LO8TmmJC4L_
 biblicus/extractors/select_smart_override.py,sha256=-sLMnNoeXbCB3dO9zflQq324eHuLbd6hpveSwduXP-U,6763
 biblicus/extractors/select_text.py,sha256=w0ATmDy3tWWbOObzW87jGZuHbgXllUhotX5XyySLs-o,3395
 biblicus/extractors/unstructured_text.py,sha256=l2S_wD_htu7ZHoJQNQtP-kGlEgOeKV_w2IzAC93lePE,3564
-biblicus-0.9.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
-biblicus-0.9.0.dist-info/METADATA,sha256=7NBBKWloUkQ2mx_CuPqAQzQJWHEwM7aJT7XQHGL2VwU,27325
-biblicus-0.9.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-biblicus-0.9.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
-biblicus-0.9.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
-biblicus-0.9.0.dist-info/RECORD,,
+biblicus-0.10.0.dist-info/licenses/LICENSE,sha256=lw44GXFG_Q0fS8m5VoEvv_xtdBXK26pBcbSPUCXee_Q,1078
+biblicus-0.10.0.dist-info/METADATA,sha256=xZ7scJLdlKHRtm0EU5Ravq5ih2mS2KNfMbbLXNqZ8Ek,27455
+biblicus-0.10.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+biblicus-0.10.0.dist-info/entry_points.txt,sha256=BZmO4H8Uz00fyi1RAFryOCGfZgX7eHWkY2NE-G54U5A,47
+biblicus-0.10.0.dist-info/top_level.txt,sha256=sUD_XVZwDxZ29-FBv1MknTGh4mgDXznGuP28KJY_WKc,9
+biblicus-0.10.0.dist-info/RECORD,,