flimanalyzer 0.6.0__py3-none-any.whl

flim/__init__.py

@@ -0,0 +1,2 @@
+# Importing flim does not load the tools or the GUI: tools register when flim.tools
+# is imported (the application does that), so flim.core can be used without wxPython.

flim/_version.py

@@ -0,0 +1,5 @@
+# Generated by versioningit
+__version__ = "0.6.0"
+__build_date__ = "2026-11-03 13:58:33.016407+00:00"
+__git_commit__ = "835f0b5776d57eda8214ed7da58ee09df86784ed"
+__dirty__ = False

flim/analysis/__init__.py

@@ -0,0 +1 @@
+"""Machine learning models of the autoencoder tools (``flim.analysis.ml``)."""

flim/analysis/ml/autoencoder.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# @Time  : 10/27/20 3:22 PM
+# @Author: Jiaxin_Zhang
+
+import os
+import pkgutil
+import importlib
+import inspect
+import logging
+import torch
+import torch.nn as nn
+import numpy as np
+from sklearn.base import BaseEstimator, TransformerMixin
+from abc import abstractmethod
+
+
+def get_autoencoder_classes():
+    pkdir = os.path.dirname(__file__)
+    for module_loader, name, ispkg in pkgutil.iter_modules([pkdir]):
+        importlib.import_module("." + name, __package__)
+    available_aes = {
+        create_instance(cls).get_name(): cls for cls in AbsAutoencoder.__subclasses__()
+    }
+    return available_aes
+
+
+def init_autoencoders():
+    tools = get_autoencoder_classes()
+    aes = [create_instance(tools[aname]) for aname in tools]
+    return aes
+
+
+def create_instance(clazz, **kwargs):
+    if isinstance(clazz, str):
+        modulename, _, classname = clazz.rpartition(".")
+    elif inspect.isclass(clazz):
+        modulename = clazz.__module__
+        classname = clazz.__name__
+    else:
+        logging.error(f"Error instantiating {clazz} autoencoder.")
+    logging.debug(
+        f"Autoencoder modulename={modulename}, classname={classname}, {kwargs}"
+    )
+    try:
+        module = importlib.import_module(modulename)
+        class_ = getattr(module, classname)
+        toolinstance = class_(**kwargs)
+    except Exception as err:
+        logging.error(f"Error: {err}")
+        logging.error(
+            f"Error instantiating {modulename}.{classname} autoencoder, {kwargs}"
+        )
+        toolinstance = None
+
+    return toolinstance
+
+
+class AbsAutoencoder(nn.Module, BaseEstimator, TransformerMixin):
+    def __init__(self):
+        super(AbsAutoencoder, self).__init__()
+        self.name = __name__
+
+    def get_name(self):
+        return self.name
+
+    def fit(self, X):
+        return self
+
+    def __sklearn_is_fitted__(self):
+        # trained with PyTorch before it is put into a scikit-learn pipeline
+        return True
+
+    def transform(self, X):
+        """Runs X through the model on the model's device.
+
+        Returns the encoder and decoder outputs as CPU tensors, wherever the
+        model runs (cpu, cuda or mps).
+        """
+        device = next(self.parameters()).device
+        X = torch.as_tensor(np.asarray(X), dtype=torch.float32, device=device)
+        with torch.no_grad():
+            encoder_out, decoder_out = self(X)
+        return encoder_out.cpu(), decoder_out.cpu()
+
+    @abstractmethod
+    def get_description(self):
+        return None
+
+
+class Autoencoder_One_Six(AbsAutoencoder):  # used for simulation
+    def __init__(self, nb_param=2, hidden_size=6):
+        super().__init__()
+        self.name = "AE Simulator 1-6"
+        self.fc1 = nn.Linear(nb_param, hidden_size)
+        self.fc4 = nn.Linear(hidden_size, nb_param)
+        self.activation2 = nn.ReLU()
+
+    def fit(self, loaders):
+        return self
+
+    def forward(self, x):
+        encoder_out = self.activation2(self.fc1(x))
+        decoder_out = self.fc4(encoder_out)
+        return encoder_out, decoder_out
+
+    def get_description(self):
+        return (
+            "This Autoencoder can be used to simulate new data based on an existing"
+            " dataset."
+        )
+
+
+class Autoencoder_One_Three(AbsAutoencoder):  # also for simulation but isnt used
+    def __init__(self, nb_param=2, hidden_size=3):
+        super().__init__()
+        self.name = "AE Simulator 1-3"
+        self.fc1 = nn.Linear(nb_param, hidden_size)
+        self.fc4 = nn.Linear(hidden_size, nb_param)
+        self.activation2 = nn.ReLU()
+
+    def fit(self, loaders):
+        return self
+
+    def forward(self, x):
+        encoder_out = self.activation2(self.fc1(x))
+        decoder_out = self.fc4(encoder_out)
+        return encoder_out, decoder_out
+
+    def get_description(self):
+        return (
+            "This Autoencoder can be used to simulate new data based on an existing"
+            " dataset."
+        )
+
+
+class Autoencoder_Two_Ten(AbsAutoencoder):
+    def __init__(self, nb_param=2, hidden_size_1=10, hidden_size_2=1):
+        super().__init__()
+        self.name = "AE Dimensionality Reduction 2-10"
+        self.fc1 = nn.Linear(nb_param, hidden_size_1)
+        self.fc2 = nn.Linear(hidden_size_1, hidden_size_2)
+        self.fc3 = nn.Linear(hidden_size_2, hidden_size_1)
+        self.fc4 = nn.Linear(hidden_size_1, nb_param)
+        self.activation2 = nn.ReLU()
+
+    def fit(self, loaders):
+        return self
+
+    def forward(self, x):
+        x_out = self.activation2(self.fc1(x))
+        encoder_out = self.activation2(self.fc2(x_out))
+        y_out = self.activation2(self.fc3(encoder_out))
+        decoder_out = self.fc4(y_out)
+        return encoder_out, decoder_out
+
+    def get_description(self):
+        return (
+            "This Autoencoder is used for dimensionality reduction of the feature space"
+            " of an existing dataset."
+        )
+
+
+class Autoencoder_Two_Five(AbsAutoencoder):  # also not really used
+    def __init__(self, nb_param=2, hidden_size_1=5, hidden_size_2=1):
+        super().__init__()
+        self.name = "AE Dimensionality Reduction 2-5"
+        self.fc1 = nn.Linear(nb_param, hidden_size_1)
+        self.fc2 = nn.Linear(hidden_size_1, hidden_size_2)
+        self.fc3 = nn.Linear(hidden_size_2, hidden_size_1)
+        self.fc4 = nn.Linear(hidden_size_1, nb_param)
+        self.activation2 = nn.ReLU()
+
+    def fit(self, loaders):
+        return self
+
+    def forward(self, x):
+        x_out = self.activation2(self.fc1(x))
+        encoder_out = self.activation2(self.fc2(x_out))
+        y_out = self.activation2(self.fc3(encoder_out))
+        decoder_out = self.fc4(y_out)
+        return encoder_out, decoder_out
+
+    def get_description(self):
+        return (
+            "This Autoencoder is used for dimensionality reduction of the feature space"
+            " of an existing dataset."
+        )

flim/analyzerapp.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon May  7 14:38:04 2018
+
+@author: khs3z
+"""
+
+import logging
+import argparse
+import sys
+from flim._version import __version__
+from flim.core.tools import FLIMAnalyzer, check_executor_available
+from flim.core.prefect_server import use_prefect_server
+
+
+def parse_arguments():
+    """Parses command line arguments."""
+    parser = argparse.ArgumentParser(
+        epilog="flimanalyzer --batch ARGS runs flimbatch ARGS (batch mode, no GUI)."
+    )
+    parser.add_argument(
+        "-i",
+        "--input",
+        nargs="+",
+        help="data files to open in the GUI, each in its own window",
+    )
+    parser.add_argument(
+        "-e",
+        "--executor",
+        default="LocalExecutor",
+        help=(
+            "executor to run pipeline. Options: LocalExecutor, LocalDaskExecutor,"
+            " DaskExecutor"
+        ),
+    )
+    parser.add_argument(
+        "-a",
+        "--execargs",
+        default="",
+        help=(
+            "comma separated list of arguments to pass to executor instance, e.g."
+            " num_workers=4 (LocalExecutor: number of tasks run at the same time)"
+        ),
+    )
+    parser.add_argument(
+        "--no-prefect-server",
+        action="store_true",
+        help=(
+            "don't start a local Prefect server if none is running; flows then use a"
+            " temporary server"
+        ),
+    )
+    parser.add_argument(
+        "-c", "--config", default=None, help="settings file to start with (any version)"
+    )
+    parser.add_argument(
+        "-l",
+        "--log",
+        choices=["info", "debug", "warning", "error", "critical"],
+        default="info",
+        help="logging level",
+    )
+    return parser.parse_args()
+
+
+def interactive_run(fa, files=None, settings=None):
+    try:
+        import wx  # noqa: F401
+    except ImportError:
+        sys.exit(
+            "flimanalyzer: the GUI needs wxPython, which is not installed:"
+            ' pip install "flimanalyzer[gui]". Without it, use flimbatch.'
+        )
+    from flim.gui.app import FLIMAnalyzerApp  # here: batch runs never import wx
+
+    app = FLIMAnalyzerApp(fa, config=settings, files=files)
+    app.MainLoop()
+
+
+def init_logger(loglevel):
+    for handler in logging.root.handlers[:]:
+        logging.root.removeHandler(handler)
+    numeric_level = getattr(logging, loglevel, None)
+    if not isinstance(numeric_level, int):
+        raise ValueError(f"Invalid log level: {loglevel}")
+    logging.basicConfig(
+        level=numeric_level,
+        filename="flimanalyzer.log",
+        filemode="w",
+        format="%(levelname)s \t %(asctime)-15s - %(module)s.%(funcName)s: %(message)s",
+    )
+    logging.info(f"Log level: {numeric_level}")
+
+
+def main():
+    argv = sys.argv[1:]
+    if "--batch" in argv:  # the rest is flimbatch's (flim/batch.py)
+        from flim import batch
+
+        argv.remove("--batch")
+        sys.exit(batch.main(argv))
+    # with open(os.path.join(os.path.dirname(__file__),"..","VERSION"), "r") as vf:
+    #    __version__ = vf.read().strip()
+
+    args = parse_arguments()
+    try:
+        check_executor_available(args.executor)
+    except RuntimeError as e:
+        sys.exit(f"flimanalyzer: {e}")
+    init_logger(args.log.upper())
+    version = __version__
+    logging.info(f"FLIMAnalyzer version {version}")
+
+    logging.debug(args)
+    fa = FLIMAnalyzer(executor=args.executor, execargs=args.execargs)
+    settings = None
+    if args.config:
+        from flim.core.settings import load_settings
+
+        settings = load_settings(args.config)
+    with use_prefect_server(start=not args.no_prefect_server):
+        interactive_run(fa, files=args.input, settings=settings)
+
+
+if __name__ == "__main__":
+    import platform, multiprocessing
+
+    if platform.system() == "Darwin":
+        multiprocessing.set_start_method("spawn")
+    main()

flim/batch.py

@@ -0,0 +1,235 @@
+"""Batch mode: import data files and run tools or workflows without the GUI.
+
+``flimbatch -i <files or folders> -o <folder> [-c settings.json] [--run NAME]``
+
+1. Reads the settings (any version, see flim.core.settings), else the defaults.
+2. Imports the files as the GUI's Import does: delimiter, filename parser and its
+   patterns, renamed and dropped columns, and the range filters if they are used.
+   The table is saved as ``Imported.txt`` in the output folder.
+3. Runs each ``--run`` tool or workflow on it, with its settings from the settings
+   file; the results are saved in ``<output>/<name>/``. Workflow columns that are not
+   in the data are errors unless ``--allow-guesses`` is given.
+
+The exit code is 0 if everything ran, 1 otherwise. Only the core is imported (no wx).
+"""
+
+import argparse
+import logging
+import os
+import sys
+from typing import Optional
+
+import numpy as np
+import pandas as pd
+
+from flim.core.execution import ExecutionOptions
+from flim.core.filter import RangeFilter
+from flim.core.importer import dataimporter
+from flim.core.parser import instantiate_parser
+from flim.core.paths import canonical_folder, folder_problem
+from flim.core.preprocessor import defaultpreprocessor
+from flim.core.registry import TOOLS_GROUP, discover, tools
+from flim.core.schema import TableSchema
+from flim.core.settings import AppSettings, load_settings
+
+IMPORTED = "Imported.txt"
+
+
+class BatchError(Exception):
+    """A batch step that cannot be done, with the reason for the user."""
+
+
+def import_table(files, settings: AppSettings, exclude=()) -> pd.DataFrame:
+    """Imports data files and folders with the import, preprocess and filter
+    settings."""
+    imported = settings.import_
+    parser = instantiate_parser(f"flim.core.parser.{imported.filename_parser.parser}")
+    if parser is None:
+        raise BatchError(f"Unknown filename parser {imported.filename_parser.parser}")
+    if imported.filename_parser.patterns:
+        parser.set_regexpatterns(imported.filename_parser.patterns)
+    preprocessor = defaultpreprocessor()
+    preprocessor.set_replacementheaders(settings.preprocess.rename)
+    preprocessor.set_dropcolumns(settings.preprocess.drop)
+
+    importer = dataimporter()
+    importer.set_delimiter(imported.delimiter)
+    importer.add_files(list(files), exclude=list(exclude) + imported.exclude_files)
+    if not importer.get_files():
+        raise BatchError("No data files found")
+    importer.set_parser(parser)
+    importer.set_column_combos(imported.category_combinations)
+    importer.set_preprocessor(preprocessor)
+    data, filenames, _ = importer.import_data()
+    if data is None or not filenames:
+        raise BatchError("None of the files could be read")
+    logging.info(f"Imported {len(filenames)} file(s): {data.shape[0]} rows")
+    if settings.filters.use:
+        data = filtered(data, settings.filters.range_filters)
+    return data
+
+
+def filtered(data: pd.DataFrame, range_filters) -> pd.DataFrame:
+    """The rows that pass the selected range filters (as the data windows show)."""
+    dropped = [RangeFilter(params=p).get_dropped(data) for p in range_filters]
+    dropped = [rows for rows, p in zip(dropped, range_filters) if p.get("selected")]
+    rows = np.unique(np.concatenate([np.asarray(r, dtype=int) for r in dropped] or [[]]))
+    if len(rows):
+        logging.info(f"Range filters dropped {len(rows)} rows")
+    return data.drop(data.index[rows.astype(int)]).reset_index(drop=True)
+
+
+def run_tool(
+    name: str,
+    data: pd.DataFrame,
+    settings: AppSettings,
+    output_dir: str,
+    execution: Optional[ExecutionOptions] = None,
+    allow_guesses: bool = False,
+):
+    """Runs a tool or workflow on ``data`` with its saved settings; its results are
+    saved in ``<output_dir>/<name>/``.
+
+    Returns:
+        flim.core.runner.FlowRun
+    """
+    from flim.core.runner import run_flow
+    from flim.core.workflow import Workflow, WorkflowValidationError
+
+    if name not in tools:
+        raise BatchError(f"No tool or workflow named '{name}'")
+    tool = tools.get(name)
+    required = [spec for spec in tool.inputs if not spec.optional]
+    if len(required) > 1 and not issubclass(tool, Workflow):
+        raise BatchError(f"{name} needs {len(required)} input tables; batch runs give one")
+    config = settings.tool_settings.config(tool)
+    execution = execution or ExecutionOptions()
+    autosave_dir = os.path.join(output_dir, tool.name)
+    task_runner = execution.task_runner()
+    if issubclass(tool, Workflow):
+        inputs = [data] * len(tool.inputs)  # e.g. training data and data: the same
+        try:
+            return tool().run_flow(
+                *inputs,
+                config=config,
+                task_runner=task_runner,
+                autosave_dir=autosave_dir,
+                strict=not allow_guesses,
+            )
+        except WorkflowValidationError as err:
+            raise BatchError(f"{name}:\n{err}") from None
+    many = tool.inputs[0].many  # e.g. Concatenate: a dict of tables
+    schema = TableSchema.from_frame(data)
+    schemas = [{"Imported": schema} if many else schema]
+    errors = [i for i in tool.validate(config, schemas) if i.severity == "error"]
+    if errors:
+        lines = [f"{i.field}: {i.message}" if i.field else i.message for i in errors]
+        raise BatchError(f"{name}:\n" + "\n".join(lines))
+
+    def build():
+        table = {"Imported": data} if many else data
+        for each in tool().map_configs(config):
+            tool()(table, config=each, task_tags=tool.name)
+
+    return run_flow(tool.name, build, task_runner=task_runner, autosave_dir=autosave_dir)
+
+
+def parse_arguments(argv=None):
+    parser = argparse.ArgumentParser(
+        prog="flimbatch",
+        description="Imports FLIM data files and runs tools or workflows on them.",
+    )
+    parser.add_argument(
+        "-i", "--input", nargs="+", required=True, help="data files or folders"
+    )
+    parser.add_argument(
+        "-x", "--exclude", nargs="+", default=[], help="files not to import"
+    )
+    parser.add_argument("-o", "--output", required=True, help="folder for the results")
+    parser.add_argument(
+        "-c",
+        "--settings",
+        "--config",
+        dest="settings",
+        help="settings file (saved in the GUI; any version)",
+    )
+    parser.add_argument(
+        "-r",
+        "--run",
+        action="append",
+        default=[],
+        metavar="NAME",
+        help="tool or workflow to run on the imported data (repeatable)",
+    )
+    parser.add_argument(
+        "--allow-guesses",
+        action="store_true",
+        help="workflows: use the closest columns for columns not in the data",
+    )
+    parser.add_argument(
+        "-e",
+        "--executor",
+        default="LocalExecutor",
+        help="LocalExecutor, LocalDaskExecutor or DaskExecutor",
+    )
+    parser.add_argument(
+        "-a", "--execargs", default="", help="executor arguments, e.g. num_workers=4"
+    )
+    parser.add_argument(
+        "--no-prefect-server",
+        action="store_true",
+        help="don't start a local Prefect server; flows use a temporary one",
+    )
+    parser.add_argument(
+        "-l",
+        "--log",
+        choices=["debug", "info", "warning", "error"],
+        default="info",
+        help="logging level",
+    )
+    return parser.parse_args(argv)
+
+
+def main(argv=None) -> int:
+    args = parse_arguments(argv)
+    logging.basicConfig(
+        level=args.log.upper(), format="%(levelname)s %(message)s", force=True
+    )
+    import flim.tools  # noqa: F401  (registers the tools)
+    import flim.workflow  # noqa: F401  (registers the workflows)
+    from flim.core.prefect_server import use_prefect_server
+    from flim.core.tools import check_executor_available
+
+    discover(TOOLS_GROUP)  # tools of other packages
+    try:
+        check_executor_available(args.executor)
+        settings = load_settings(args.settings) if args.settings else AppSettings.default()
+        output = canonical_folder(args.output)
+        if problem := folder_problem(output):
+            raise BatchError(f"--output: {problem}")
+        os.makedirs(output, exist_ok=True)
+        data = import_table(args.input, settings, exclude=args.exclude)
+        path = os.path.join(output, IMPORTED)
+        data.to_csv(path, sep="\t", index=False)
+        logging.info(f"Saved {path}")
+        execution = ExecutionOptions.from_legacy(args.executor, args.execargs)
+        failed = []
+        with use_prefect_server(start=not args.no_prefect_server):
+            for name in args.run:
+                run = run_tool(
+                    name, data, settings, output, execution, args.allow_guesses
+                )
+                if run.failed or run.state != "completed":
+                    failed.append(f"{name}: {', '.join(run.failed) or run.state}")
+                logging.info(f"{name}: {run.state}; results in {run.autosave_dir}")
+    except (BatchError, RuntimeError, OSError, ValueError) as err:
+        print(f"flimbatch: {err}", file=sys.stderr)
+        return 1
+    if failed:
+        print("flimbatch: failed tasks\n" + "\n".join(failed), file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

flim/configs/__init__.py

@@ -0,0 +1,77 @@
+"""Configs of the built-in tools and workflows (no wx, so usable without the GUI)."""
+
+from flim.configs.analysis import (
+    AEAugmentConfig,
+    AERunConfig,
+    AETrainConfig,
+    CategorizerConfig,
+    GMMConfig,
+    KMeansConfig,
+    KSStatsConfig,
+    PCAConfig,
+    RandomForestConfig,
+    RelativeChangeConfig,
+    SeriesAnalysisConfig,
+    SummaryConfig,
+)
+from flim.configs.data import (
+    CategoryOrderConfig,
+    ConcatConfig,
+    FilterConfig,
+    MergeConfig,
+    PivotConfig,
+    SortConfig,
+    UnpivotConfig,
+)
+from flim.configs.data import SplitConfig  # noqa: F401  (re-exported)
+from flim.configs.plot import (
+    BarPlotConfig,
+    BoxPlotConfig,
+    FLIRRPlotConfig,
+    HeatmapConfig,
+    HistogramConfig,
+    KDEConfig,
+    LinePlotConfig,
+    PairPlotConfig,
+    ScatterPlotConfig,
+    SwarmPlotConfig,
+    ViolinPlotConfig,
+)
+from flim.configs.workflow import AEFeatureConfig, AETuneConfig, BasicFLIRRConfig
+
+# config class of each plugin, by the plugin's name (the key of its saved settings)
+LEGACY_TOOL_CONFIGS = {
+    "Order Categories": CategoryOrderConfig,
+    "Concat Data": ConcatConfig,
+    "Filter": FilterConfig,
+    "Merge": MergeConfig,
+    "Pivot": PivotConfig,
+    "Sort": SortConfig,
+    "Unpivot": UnpivotConfig,
+    "Summarize": SummaryConfig,
+    "KS-Statistics": KSStatsConfig,
+    "Relative Change": RelativeChangeConfig,
+    "Series Analysis": SeriesAnalysisConfig,
+    "PCA": PCAConfig,
+    "K-Means": KMeansConfig,
+    "Gaussian Mixed Model": GMMConfig,
+    "Random Forest": RandomForestConfig,
+    "Categorize Data": CategorizerConfig,
+    "Autoencoder: Train": AETrainConfig,
+    "Autoencoder: Run": AERunConfig,
+    "Autoencoder: Augment Data": AEAugmentConfig,
+    "Bar Plot": BarPlotConfig,
+    "Box Plot": BoxPlotConfig,
+    "Swarm Plot": SwarmPlotConfig,
+    "Violin Plot": ViolinPlotConfig,
+    "Line Plot": LinePlotConfig,
+    "KDE": KDEConfig,
+    "Frequency Histogram": HistogramConfig,
+    "Heatmap": HeatmapConfig,
+    "Pair Plot": PairPlotConfig,
+    "Scatter Plot": ScatterPlotConfig,
+    "FLIRR Plot": FLIRRPlotConfig,
+    "Basic FLIRR Analysis": BasicFLIRRConfig,
+    "FLIM Data Augmentation Tuning": AETuneConfig,
+    "FLIM Feature Analysis": AEFeatureConfig,
+}