svc-processing 0.1.1__py3-none-any.whl

pipeline/cli.py

@@ -0,0 +1,92 @@
+"""Command-line interface for the SIG processing pipeline.
+
+This module is thin glue: it parses arguments, configures logging, and wires the
+two collaborators together for each input directory —
+
+* :class:`~pipeline.run_config.RunConfig` — loads the run config and builds the
+  per-directory settings (*what to run*).
+* :class:`~pipeline.runner.Pipeline` — runs the processing + resampling stages
+  (*do the work*).
+
+Invoke it as ``svc-pipeline [config]`` after installation.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+from pathlib import Path
+
+from .run_config import RunConfig
+from .runner import Pipeline
+
+
+def _configure_logging(verbose: bool) -> logging.Logger:
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(level=level, format="%(levelname)s: %(message)s", force=True)
+    return logging.getLogger("sig_pipeline")
+
+
+def _parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Run the SIG processing pipeline")
+    parser.add_argument(
+        "config",
+        nargs="?",
+        default="config.json",
+        help="Run-config JSON. Bare names resolve under config/ (default: config.json).",
+    )
+    parser.add_argument(
+        "--input-dir",
+        help="Override the config's sig_input_dir and process only this directory.",
+    )
+    parser.add_argument(
+        "--step",
+        choices=["1", "2", "all"],
+        default="all",
+        help="Which step to run: 1=process+summary CSV, 2=Python resampling only, all=run both steps.",
+    )
+    parser.add_argument(
+        "--verbose",
+        action="store_true",
+        help="Enable verbose logging before and after each processing step",
+    )
+    return parser.parse_args()
+
+
+def main() -> None:
+    args = _parse_args()
+    logger = _configure_logging(args.verbose)
+
+    # Resolve relative config, calibration, and output paths against the current
+    # working directory so the installed `svc-pipeline` command works from any
+    # directory. Absolute paths in the config are always honoured as-is.
+    base_dir = Path.cwd()
+    run_config = RunConfig.load(base_dir, str(args.config), logger)
+    run_config.ensure_no_placeholder(args.input_dir)
+
+    if args.input_dir:
+        input_dirs = [Path(str(args.input_dir)).expanduser()]
+    else:
+        input_dirs = run_config.input_directories()
+
+    # Resolve the processing block once, up front: emits a one-time parity
+    # warning for any non-default value before the per-directory work begins.
+    run_config.processing_params()
+
+    overall_results: list[tuple[Path, dict[str, Path | None]]] = []
+    for input_dir in input_dirs:
+        settings = run_config.settings_for(input_dir, verbose=args.verbose)
+        results = Pipeline(settings, logger).run(args.step)
+        overall_results.append((input_dir, results))
+
+    for directory, outputs in overall_results:
+        print(f"\nInput directory: {directory.resolve()}")
+        for label, path in outputs.items():
+            if path:
+                print(f"  {label}: {Path(path).resolve()}")
+            else:
+                print(f"  {label}: not produced")
+
+
+if __name__ == "__main__":
+    main()

pipeline/processor.py

@@ -0,0 +1,582 @@
+from __future__ import annotations
+
+import re
+import warnings
+from collections import namedtuple
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+import pandas as pd
+
+
+GroupTuple = namedtuple("GroupTuple", ["members", "name"])
+
+
+@dataclass(frozen=True)
+class GroupSpec:
+    """Tuple-like specification for grouping spectra, with optional display name."""
+    members: Sequence[int]
+    name: str | None = None
+
+    def __post_init__(self):
+        normalized_members = tuple(int(m) for m in self.members)
+        object.__setattr__(self, "members", normalized_members)
+
+    def to_namedtuple(self) -> GroupTuple:
+        return GroupTuple(self.members, self.name)
+
+    @classmethod
+    def from_csv(
+        cls,
+        csv_path: str | Path,
+        *,
+        members_column: str = "scans",
+        id_column: str = "scan_id",
+        name_column: str = "name",
+        exclude_columns: Sequence[str] = ("reference",),
+        delimiter: str = ",",
+        return_namedtuple: bool = True,
+    ) -> list["GroupSpec" | GroupTuple]:
+        df = pd.read_csv(csv_path)
+
+        column_lookup = {col.lower(): col for col in df.columns}
+
+        drop_columns = [
+            column_lookup[col.lower()]
+            for col in exclude_columns
+            if col.lower() in column_lookup
+        ]
+        if drop_columns:
+            df = df.drop(columns=drop_columns)
+            column_lookup = {col.lower(): col for col in df.columns}
+
+        resolved_members_col = column_lookup.get(members_column.lower())
+        resolved_id_col = column_lookup.get(id_column.lower())
+        resolved_name_col = column_lookup.get(name_column.lower())
+
+        if resolved_members_col is None and resolved_id_col is None:
+            raise KeyError(
+                f"Expected '{members_column}' or '{id_column}' in {csv_path}; none found."
+            )
+        if resolved_name_col is None:
+            raise KeyError(f"Column '{name_column}' not found in {csv_path}.")
+
+        specs: list["GroupSpec" | GroupTuple] = []
+        for _, row in df.iterrows():
+            if resolved_members_col is not None:
+                members_value = row[resolved_members_col]
+            else:
+                members_value = row.get(resolved_id_col)
+            if pd.isna(members_value):
+                continue
+
+            if isinstance(members_value, str):
+                tokens = [token.strip() for token in members_value.replace(";", delimiter).split(delimiter)]
+                members = tuple(int(token) for token in tokens if token)
+            elif isinstance(members_value, Iterable):
+                members = tuple(int(v) for v in members_value)
+            else:
+                members = (int(members_value),)
+
+            if not members:
+                continue
+
+            name_value = row.get(resolved_name_col)
+            group_name = None if pd.isna(name_value) else str(name_value).strip() or None
+
+            if group_name and group_name.lower() == "reference":
+                continue
+
+            spec = cls(members=members, name=group_name)
+            specs.append(spec.to_namedtuple() if return_namedtuple else spec)
+
+        return specs
+
+
+class SVCDataProcessor:
+    """Compact OO processor for SVC spectra workflows."""
+    SigEntry = namedtuple("SigEntry", ["index", "base_name", "number"])
+    SAMPLE_NAME_PATTERN = re.compile(r"^(?P<base>.+?)\.(?P<num>\d+)(?:\.sig)?$", re.IGNORECASE)
+    TRAILING_DIGITS_PATTERN = re.compile(r"^(?P<base>.*?)(?P<num>\d+)(?:\.sig)?$", re.IGNORECASE)
+
+    def _require_attributes(self, *names: str, message: str) -> None:
+        missing = [name for name in names if not hasattr(self, name)]
+        if missing:
+            raise RuntimeError(message)
+
+    # ---------- IO ----------
+    def load_csv(self, file_path: str, **kwargs):
+        """Load CSV file. Pass any additional pandas.read_csv() arguments via kwargs."""
+        self.df: pd.DataFrame = pd.read_csv(file_path, **kwargs)
+        return self
+
+    # ---------- Structure ----------
+    def split_columns(self, name_col: str | None = None):
+        self._require_attributes("df", message="Call load_csv first.")
+        self.name_col: str = name_col or self.df.columns[0]
+        num_cols = self.df.select_dtypes(include='number').columns.tolist()
+        xlike_cols = [c for c in self.df.columns if isinstance(c, str) and c.startswith("X")]
+        self.wavelength_cols: list[str] = list(dict.fromkeys(num_cols + xlike_cols))
+        return self
+
+    # ---------- Parsing ----------
+    @classmethod
+    def normalize_sample_name(cls, value: str | int | float) -> str:
+        raw = "" if value is None else str(value).strip()
+        if not raw:
+            raise ValueError("Sample name is empty or None.")
+
+        name = raw[:-4] if raw.lower().endswith(".sig") else raw
+
+        match = cls.SAMPLE_NAME_PATTERN.match(name)
+        if not match:
+            match = cls.TRAILING_DIGITS_PATTERN.match(name)
+
+        if not match:
+            raise ValueError(f"Sample name '{raw}' does not match expected pattern.")
+
+        base = match.group("base").rstrip(".")
+        num = match.group("num")
+
+        try:
+            padded_num = str(int(num)).zfill(4)
+        except ValueError as exc:
+            raise ValueError(f"Sample number '{num}' in '{raw}' is not numeric.") from exc
+
+        return f"{base}.{padded_num}"
+
+    @classmethod
+    def normalize_sample_column(
+        cls,
+        df: pd.DataFrame,
+        column: str = "name",
+        *,
+        inplace: bool = False,
+    ) -> pd.DataFrame:
+        target = df if inplace else df.copy()
+        target[column] = target[column].astype(str).apply(cls.normalize_sample_name)
+        return target
+
+    def extract_sig_entries(self):
+        self._require_attributes("df", "name_col", message="Call split_columns first.")
+        names = self.df[self.name_col].astype(str).tolist()
+        out = []
+        for i, n in enumerate(names):
+            try:
+                normalized = self.normalize_sample_name(n)
+                base_part, number_part = normalized.rsplit(".", 1)
+                number_value = int(number_part)
+                self.df.at[i, self.name_col] = normalized
+                out.append(
+                    self.SigEntry(index=i, base_name=base_part, number=number_value)
+                )
+            except ValueError:
+                warnings.warn(f"Unable to normalize sample name '{n}' at index {i}.", UserWarning)
+                out.append(
+                    self.SigEntry(index=i, base_name=n, number=None)
+                )
+        self.entries: list[self.SigEntry] = out
+        return self
+
+    # ---------- Grouping ----------
+    @staticmethod
+    def _ensure_iterable_group(g: int | Iterable[int]) -> Iterable[int]:
+        return (g,) if isinstance(g, int) else g
+
+    def group_by(self, groups: Sequence[int | Iterable[int]], *, by: Literal["number", "index"] = "number"):
+        self._require_attributes("entries", message="Call extract_sig_entries first.")
+
+        available_values = {getattr(e, by) for e in self.entries}
+        lookup = {}
+        for e in self.entries:
+            key = getattr(e, by)
+            lookup.setdefault(key, []).append(e)
+
+        grouped: list[list[self.SigEntry]] = []
+        used_values = set()
+
+        for g in groups:
+            vals = self._ensure_iterable_group(g)
+            bucket: list[self.SigEntry] = []
+            for val in vals:
+                if val not in available_values:
+                    warnings.warn(f"Cannot find {by}={val}", UserWarning)
+                else:
+                    bucket.extend(lookup[val])
+                    used_values.add(val)
+            grouped.append(bucket)
+
+        uncovered_vals = available_values - used_values
+        ungrouped_entries = [e for e in self.entries if getattr(e, by) in uncovered_vals]
+        if ungrouped_entries:
+            warnings.warn(
+                f"Entries {[getattr(e, by) for e in ungrouped_entries]} do not belong to any group",
+                UserWarning
+            )
+
+        self.grouped_entries: list[list[self.SigEntry]] = grouped
+        self.ungrouped_entries: list[self.SigEntry] = ungrouped_entries
+        return self
+
+    # ---------- Averaging ----------
+    @staticmethod
+    def _select_numeric_cols(df: pd.DataFrame, cols: Iterable[str] | None) -> list[str]:
+        return list(cols) if cols is not None else list(df.select_dtypes(include="number").columns)
+
+    @staticmethod
+    def _validate_group_indices(df: pd.DataFrame, group: Sequence["SVCDataProcessor.SigEntry"], group_id: int) -> None:
+        n = len(df)
+        bad = [e.index for e in group if e.index < 0 or e.index >= n]
+        if bad:
+            raise IndexError(f"Group {group_id} has out-of-bounds indices {bad} for DataFrame with {n} rows.")
+
+    @staticmethod
+    def _row_indices(group: Sequence["SVCDataProcessor.SigEntry"]) -> list[int]:
+        return [e.index for e in group]
+
+    @staticmethod
+    def _format_indices_display(row_idxs: Sequence[int], index_base: int) -> str:
+        if index_base not in (0, 1):
+            raise ValueError("index_base must be 0 or 1.")
+        indices = ",".join(str(i + index_base) for i in row_idxs)
+        return f"rows[{indices}]"
+
+    @staticmethod
+    def _pick_sample_name(df: pd.DataFrame, name_col: str, group: Sequence["SVCDataProcessor.SigEntry"], row_idxs: Sequence[int], strategy: str = "base_first") -> str:
+        base_names = [e.base_name for e in group if e.base_name]
+        first_row_name = str(df.iloc[row_idxs[0]][name_col])
+        if strategy == "first_cell":
+            return first_row_name
+        if strategy == "concat_base":
+            seen, uniq = set(), []
+            for b in base_names:
+                if b not in seen:
+                    seen.add(b)
+                    uniq.append(b)
+            return ";".join(uniq) if uniq else first_row_name
+        if base_names:
+            unique_bases = set(base_names)
+            if len(unique_bases) > 1:
+                warnings.warn(
+                    f"Group mixes base names {sorted(unique_bases)}; using '{base_names[0]}' (strategy='base_first').",
+                    UserWarning
+                )
+            first_number = next((e.number for e in group if e.number is not None), None)
+            if first_number is not None:
+                return f"{base_names[0]}.{str(first_number).zfill(4)}"
+            return base_names[0]
+        return first_row_name
+
+    @staticmethod
+    def _generate_sample_name_with_min(base_name: str, group: Sequence["SVCDataProcessor.SigEntry"]) -> str:
+        numbers = [e.number for e in group if e.number is not None]
+        if not numbers:
+            return base_name
+        return f"{base_name}_{min(numbers)}"
+
+    @staticmethod
+    def _aggregate_rows(
+        df: pd.DataFrame,
+        row_idxs: Sequence[int],
+        cols: Sequence[str],
+        method: Literal["mean", "median", "sum", "min", "max"],
+    ) -> pd.DataFrame:
+        subset = df.iloc[row_idxs][list(cols)]
+        if method == "mean":
+            aggregated = subset.mean(numeric_only=True, skipna=True)
+        elif method == "median":
+            aggregated = subset.median(numeric_only=True, skipna=True)
+        elif method == "sum":
+            aggregated = subset.sum(numeric_only=True, skipna=True)
+        elif method == "min":
+            aggregated = subset.min(numeric_only=True, skipna=True)
+        elif method == "max":
+            aggregated = subset.max(numeric_only=True, skipna=True)
+        else:
+            raise ValueError(f"Unsupported aggregation method '{method}'.")
+        return aggregated.to_frame().T
+
+    def average_groups(
+        self,
+        *,
+        cols: Iterable[str] | None = None,
+        index_base: int = 1,
+        name_strategy: str = "base_first",
+        base_name: str | None = None,
+        agg_method: Literal["mean", "median", "sum", "min", "max"] = "mean",
+        override_names: Sequence[str | None] | None = None,
+    ):
+        self._require_attributes(
+            "df",
+            "grouped_entries",
+            "wavelength_cols",
+            "name_col",
+            message="Run previous steps first.",
+        )
+
+        use_cols = self._select_numeric_cols(self.df, cols or self.wavelength_cols)
+        name_col = self.name_col
+
+        if override_names is not None and len(override_names) != len(self.grouped_entries):
+            raise ValueError("override_names length must match number of groups.")
+
+        out_rows = []
+        for g_id, group in enumerate(self.grouped_entries, start=1):
+            if not group:
+                warnings.warn(f"Empty group at position {g_id}; skipping.", UserWarning)
+                continue
+            self._validate_group_indices(self.df, group, g_id)
+            row_idxs = self._row_indices(group)
+
+            override_name = None
+            if override_names:
+                override_name = override_names[g_id - 1]
+
+            if override_name:
+                sample_name = override_name
+            elif base_name is not None:
+                sample_name = self._generate_sample_name_with_min(base_name, group)
+            else:
+                sample_name = self._pick_sample_name(self.df, name_col, group, row_idxs, name_strategy)
+
+            averaged = self._aggregate_rows(self.df, row_idxs, use_cols, agg_method)
+
+            group_numbers = [str(e.number) for e in group if e.number is not None]
+            grouping_str = ",".join(group_numbers) if group_numbers else ""
+
+            averaged.insert(0, "row_indices", self._format_indices_display(row_idxs, index_base))
+            averaged.insert(0, "grouping", grouping_str)
+            averaged.insert(0, "name", sample_name)
+
+            out_rows.append(averaged)
+
+        self.grouped_df: pd.DataFrame = pd.concat(out_rows, ignore_index=True) if out_rows else pd.DataFrame()
+        return self
+
+    # ---------- Ungrouped summary ----------
+    @staticmethod
+    def ungrouped_entries_to_df(ungrouped: list["SVCDataProcessor.SigEntry"]) -> pd.DataFrame:
+        if not ungrouped:
+            return pd.DataFrame(columns=["index", "number", "base_name"])
+        return pd.DataFrame([{"index": e.index, "number": e.number, "base_name": e.base_name} for e in ungrouped])
+
+    def ungrouped_as_df(self):
+        self.ungrouped_df = self.ungrouped_entries_to_df(getattr(self, "ungrouped_entries", []) or [])
+        return self
+
+    # ---------- Combine grouped + ungrouped ----------
+    def concat_grouped_and_ungrouped(
+        self,
+        *,
+        ungrouped_mode: Literal["raw", "empty"] = "raw",
+        index_base: int = 1
+    ):
+        """Stack self.grouped_df with ungrouped entries (raw spectra or empty spectral cells)."""
+        self._require_attributes("grouped_df", message="Call average_groups first.")
+        self._require_attributes(
+            "ungrouped_entries",
+            message="Call group_by first to populate ungrouped_entries.",
+        )
+        self._require_attributes(
+            "df",
+            "wavelength_cols",
+            "name_col",
+            message="Ensure load_csv/split_columns ran.",
+        )
+
+        grouped = self.grouped_df.copy()
+
+        rows = []
+        for e in (self.ungrouped_entries or []):
+            row = {col: None for col in grouped.columns}
+            row["name"] = e.base_name or str(self.df.iloc[e.index][self.name_col])
+            row["row_indices"] = self._format_indices_display([e.index], index_base)
+            row["grouping"] = str(e.number) if e.number is not None else ""
+
+            if ungrouped_mode == "raw":
+                for col in grouped.columns:
+                    if col in ("name", "row_indices", "grouping"):
+                        continue
+                    if col in self.df.columns:
+                        row[col] = self.df.iloc[e.index][col]
+            rows.append(row)
+
+        ungrouped_df_aligned = pd.DataFrame(rows, columns=grouped.columns) if rows else pd.DataFrame(columns=grouped.columns)
+        self.final_df = pd.concat([grouped, ungrouped_df_aligned], ignore_index=True)
+        return self
+
+    # ---------- Pretty prints ----------
+    def debug_print_groups(self):
+        """Print groups and ungrouped entries with clear headers and spacing."""
+        self._require_attributes("grouped_entries", message="Call group_by first.")
+
+        print("#### Group Entries ####\n")
+        print("Number of groups:", len(self.grouped_entries), "\n")
+
+        for gi, group in enumerate(self.grouped_entries, start=1):
+            print(f"##### Group {gi} #####")
+            if not group:
+                print(" (empty)\n")
+                continue
+            for e in group:
+                print(f" index={e.index}, number={e.number}, base_name={e.base_name}")
+            print()
+
+        print("##### Ungrouped Entries #####")
+        if getattr(self, "ungrouped_entries", None):
+            for e in self.ungrouped_entries:
+                print(f" index={e.index}, number={e.number}, base_name={e.base_name}")
+        else:
+            print(" None")
+        print()
+
+    # ---------- Save ----------
+    def save_csv(self, out_path: str):
+        self._require_attributes("grouped_df", message="Call average_groups first.")
+        self.grouped_df.to_csv(out_path, index=False)
+        return self
+
+
+class SigSpectraAverager:
+    """Facade around SVCDataProcessor that normalizes sample names and averages spectra."""
+
+    def __init__(self, df: pd.DataFrame, sample_col: str = "sample_name"):
+        self.sample_col = sample_col
+
+        working_df = df.copy()
+        if sample_col != "name":
+            if sample_col not in working_df.columns:
+                raise KeyError(f"Column '{sample_col}' not found in DataFrame.")
+            working_df = working_df.rename(columns={sample_col: "name"})
+
+        self.dataframe = SVCDataProcessor.normalize_sample_column(working_df, column="name", inplace=False)
+        self.processor = SVCDataProcessor()
+        self.processor.df = self.dataframe
+        self.processor.split_columns(name_col="name")
+        self.processor.extract_sig_entries()
+
+    @classmethod
+    def from_csv(cls, file_path: str, sample_col: str = "sample_name", **kwargs) -> "SigSpectraAverager":
+        df = pd.read_csv(file_path, **kwargs)
+        return cls(df, sample_col=sample_col)
+
+    @staticmethod
+    def _normalize_groups(groups: Sequence[int | Iterable[int]]) -> list[dict]:
+        normalized: list[dict] = []
+        for g in groups:
+            custom_name: str | None = None
+
+            if isinstance(g, GroupSpec):
+                members_source = g.members
+                custom_name = g.name
+            elif hasattr(g, "_fields"):
+                custom_name = getattr(g, "name", None)
+                if hasattr(g, "members"):
+                    members_source = getattr(g, "members")
+                else:
+                    members_source = [getattr(g, field) for field in g._fields if field != "name"]
+            elif hasattr(g, "members") and isinstance(getattr(g, "members"), Iterable):
+                members_source = getattr(g, "members")
+                custom_name = getattr(g, "name", None)
+            elif isinstance(g, Iterable) and not isinstance(g, (str, bytes)):
+                members_source = g
+                custom_name = getattr(g, "name", None) if hasattr(g, "name") else None
+            else:
+                members_source = (g,)
+
+            if isinstance(members_source, Iterable) and not isinstance(members_source, (str, bytes)):
+                flattened: list[int] = []
+                for value in members_source:
+                    if isinstance(value, Iterable) and not isinstance(value, (str, bytes)):
+                        flattened.extend(int(v) for v in value)
+                    else:
+                        flattened.append(int(value))
+                values = tuple(flattened) if flattened else (int(members_source),)
+            else:
+                values = (int(members_source),)
+
+            normalized.append({"members": values, "name": custom_name})
+        return normalized
+
+    def aggregate(
+        self,
+        groups: Sequence[int | Iterable[int]],
+        method: Literal["mean", "median", "sum", "min", "max"] | None = "mean",
+        *,
+        index_base: int = 1,
+        name_strategy: str = "base_first",
+        base_name: str | None = None,
+    ) -> pd.DataFrame:
+        normalized_groups = self._normalize_groups(groups)
+        member_sets = [spec["members"] for spec in normalized_groups]
+        custom_names = [spec["name"] for spec in normalized_groups]
+
+        self.processor.group_by(member_sets, by="number")
+
+        if method is None:
+            grouped_frames = []
+            for name_spec, group in zip(normalized_groups, self.processor.grouped_entries):
+                if not group:
+                    continue
+                rows = [entry.index for entry in group]
+                subset = self.processor.df.iloc[rows].copy()
+                override_name = name_spec.get("name")
+                if override_name:
+                    subset["name"] = override_name
+                grouped_frames.append(subset)
+            return pd.concat(grouped_frames, ignore_index=True) if grouped_frames else pd.DataFrame()
+
+        self.processor.average_groups(
+            cols=self.processor.wavelength_cols,
+            index_base=index_base,
+            name_strategy=name_strategy,
+            base_name=base_name,
+            agg_method=method,
+            override_names=custom_names,
+        )
+        return self.processor.grouped_df.copy()
+
+
+def find_spectra_by_name(
+    dataframes: Sequence[pd.DataFrame],
+    search_key: str,
+    *,
+    name_column: str = "name",
+    case_sensitive: bool = False,
+    exact_match: bool = False,
+) -> pd.DataFrame:
+    """Search across multiple spectra DataFrames for rows whose name column matches search_key."""
+    if not search_key:
+        raise ValueError("search_key must be a non-empty string.")
+
+    results: list[pd.DataFrame] = []
+    for idx, df in enumerate(dataframes):
+        if name_column not in df.columns:
+            raise KeyError(f"DataFrame at position {idx} lacks the '{name_column}' column.")
+
+        names = df[name_column].astype(str)
+        if exact_match:
+            mask = names.eq(search_key) if case_sensitive else names.str.lower().eq(search_key.lower())
+        else:
+            mask = names.str.contains(
+                search_key if case_sensitive else re.escape(search_key),
+                case=case_sensitive,
+                regex=not exact_match,
+                na=False,
+            )
+
+        subset = df.loc[mask]
+        if not subset.empty:
+            annotated = subset.copy()
+            annotated.insert(0, "_source_index", idx)
+            results.append(annotated)
+
+    if not results:
+        if dataframes:
+            base_cols = list(dataframes[0].columns)
+            return pd.DataFrame(columns=["_source_index"] + base_cols)
+        return pd.DataFrame()
+
+    return pd.concat(results, ignore_index=True)