stdpipe 0.3.0__tar.gz → 0.4.1__tar.gz

{stdpipe-0.3.0/stdpipe.egg-info → stdpipe-0.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: stdpipe
-Version: 0.3.0
+Version: 0.4.1
 Summary: Simple Transient Detection Pipeline
 Author-email: Sergey Karpov <karpov.sv@gmail.com>
 Maintainer-email: Sergey Karpov <karpov.sv@gmail.com>
@@ -53,17 +53,17 @@ Requires-Dist: scipy
 Requires-Dist: astropy>=4.1
 Requires-Dist: matplotlib
 Requires-Dist: astroquery>0.4.1
-Requires-Dist: sep
+Requires-Dist: sep-x
 Requires-Dist: astroscrappy
 Requires-Dist: photutils>=2.2
-Requires-Dist: scikit-image
+Requires-Dist: scikit-image>=0.22.0
 Requires-Dist: statsmodels
 Requires-Dist: tqdm
 Requires-Dist: regions
 Requires-Dist: python-dateutil
 Requires-Dist: requests
-Requires-Dist: sip_tpv
 Requires-Dist: pyfftw
+Requires-Dist: scikit-learn
 Provides-Extra: dev
 Requires-Dist: pytest>=6.0; extra == "dev"
 Requires-Dist: pytest-cov>=2.10; extra == "dev"
@@ -82,8 +82,10 @@ Requires-Dist: sphinxcontrib-apidoc; extra == "docs"
 Requires-Dist: sphinx-prompt; extra == "docs"
 Provides-Extra: adaptive
 Requires-Dist: powerbin; extra == "adaptive"
+Provides-Extra: ml
+Requires-Dist: tensorflow<2.20,>=2.10; extra == "ml"
 Provides-Extra: all
-Requires-Dist: stdpipe[adaptive,dev,docs]; extra == "all"
+Requires-Dist: stdpipe[adaptive,dev,docs,ml]; extra == "all"
 Dynamic: license-file
 
 # STDPipe - Simple Transient Detection Pipeline
@@ -103,20 +105,28 @@ Dynamic: license-file
  - ~~pre-processing~~ - should be handled before in an instrument-specific way
      - bias/dark subtraction, flatfielding, masking
  - object detection and photometry
-     - SExtractor or SEP for detection, photutils for photometry
+     - SExtractor, SEP, or photutils (pure Python) for detection
+     - aperture, optimal extraction (Naylor 1998), and PSF fitting photometry
+     - grouped fitting for crowded fields
  - astrometric calibration
      - Astrometry.Net for blind WCS solving
-     - SCAMP or Astropy-based code for refinement
+     - quad-hash pattern matching for refinement (pure Python, no external deps, default method)
+     - SCAMP or Astropy-based code for refinement (alternative)
  - photometric calibration
      - Vizier catalogues, passband conversion (PS1 to Johnson, Gaia to Johnson, ...)
      - spatial polynomial + color term + intrinsic scatter
+ - image reprojection and stacking
+     - Lanczos interpolation with automatic oversampling and flux conservation (pure Python, default)
+     - SWarp wrapper (alternative)
  - image subtraction
      - HiPS templates
      - PanSTARRS DR1 or Legacy Survey templates
      - HOTPANTS + custom noise model
      - ZOGY
- - transient detection and photometry
+ - transient detection and classification
      - noise-weighted detection, cutout adjustment, ...
+     - feature-based real/bogus classification (no TensorFlow required)
+     - CNN-based real/bogus classification (optional, requires TensorFlow)
  - auxiliary functions
      - PSF estimation, simulated stars, FITS header utilities, plotting, ...
  - light curve creation (soon)

{stdpipe-0.3.0 → stdpipe-0.4.1}/README.md

@@ -15,20 +15,28 @@
  - ~~pre-processing~~ - should be handled before in an instrument-specific way
      - bias/dark subtraction, flatfielding, masking
  - object detection and photometry
-     - SExtractor or SEP for detection, photutils for photometry
+     - SExtractor, SEP, or photutils (pure Python) for detection
+     - aperture, optimal extraction (Naylor 1998), and PSF fitting photometry
+     - grouped fitting for crowded fields
  - astrometric calibration
      - Astrometry.Net for blind WCS solving
-     - SCAMP or Astropy-based code for refinement
+     - quad-hash pattern matching for refinement (pure Python, no external deps, default method)
+     - SCAMP or Astropy-based code for refinement (alternative)
  - photometric calibration
      - Vizier catalogues, passband conversion (PS1 to Johnson, Gaia to Johnson, ...)
      - spatial polynomial + color term + intrinsic scatter
+ - image reprojection and stacking
+     - Lanczos interpolation with automatic oversampling and flux conservation (pure Python, default)
+     - SWarp wrapper (alternative)
  - image subtraction
      - HiPS templates
      - PanSTARRS DR1 or Legacy Survey templates
      - HOTPANTS + custom noise model
      - ZOGY
- - transient detection and photometry
+ - transient detection and classification
      - noise-weighted detection, cutout adjustment, ...
+     - feature-based real/bogus classification (no TensorFlow required)
+     - CNN-based real/bogus classification (optional, requires TensorFlow)
  - auxiliary functions
      - PSF estimation, simulated stars, FITS header utilities, plotting, ...
  - light curve creation (soon)

{stdpipe-0.3.0 → stdpipe-0.4.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "stdpipe"
-version = "0.3.0"
+dynamic = ["version"]
 description = "Simple Transient Detection Pipeline"
 readme = "README.md"
 license = {file = "LICENSE.md"}
@@ -47,18 +47,17 @@ dependencies = [
     "astropy>=4.1",
     "matplotlib",
     "astroquery>0.4.1",
-    "sep",
+    "sep-x",
     "astroscrappy",
     "photutils>=2.2",
-    "scikit-image",
+    "scikit-image>=0.22.0",  # Requires >=0.22.0 for NumPy 2.0 compatibility
     "statsmodels",
     "tqdm",
     "regions",
     "python-dateutil",
     "requests",
-    # "sip_tpv @ git+https://github.com/pmvreeswijk/sip_tpv.git",
-    "sip_tpv",
     "pyfftw",
+    "scikit-learn",
 ]
 
 [project.optional-dependencies]
@@ -89,9 +88,14 @@ adaptive = [
     "powerbin",
 ]
 
+# Machine learning for real-bogus classifier
+ml = [
+    "tensorflow>=2.10,<2.20",
+]
+
 # All optional dependencies
 all = [
-    "stdpipe[dev,docs,adaptive]",
+    "stdpipe[dev,docs,adaptive,ml]",
 ]
 
 [project.urls]
@@ -109,13 +113,31 @@ Changelog = "https://github.com/karpov-sv/stdpipe/commits/master"
 zip-safe = false
 include-package-data = true
 
+# Single source of truth for the version - parsed statically from stdpipe/__init__.py
+[tool.setuptools.dynamic]
+version = {attr = "stdpipe.__version__"}
+
 [tool.setuptools.packages.find]
 where = ["."]
 include = ["stdpipe*"]
 exclude = ["tests*", "docs*", "notebooks*", "build*", "dist*"]
+# stdpipe/data/ has no __init__.py and is not meant to be importable - without
+# this it would be picked up as a namespace package, shipping the scripts in it
+namespaces = false
 
+# Listed explicitly rather than globbed, so that stray files in the working
+# directory (editor backups etc) can never end up in a release. Note that
+# data/legacysurvey.py is a one-off regeneration script, not shipped on purpose.
 [tool.setuptools.package-data]
-stdpipe = ["data/*"]
+stdpipe = [
+    "data/legacysurvey_bricks.fits.gz",
+    "data/ps1skycells.txt",
+]
+
+# data/legacysurvey.py is a one-off script for regenerating the bricks list, and
+# is picked up by the generic "all .py files" sdist rule - keep it out of the wheel
+[tool.setuptools.exclude-package-data]
+stdpipe = ["data/legacysurvey.py"]
 
 # ============================================================================
 # pytest configuration
@@ -132,6 +154,7 @@ addopts = [
     "--strict-markers",
     "--showlocals",
     "-ra",
+    "-m", "not slow",
 ]
 markers = [
     "unit: Unit tests that don't require external dependencies",
@@ -233,6 +256,9 @@ extend-exclude = [
     "dist",
 ]
 
+[tool.ruff.format]
+quote-style = "preserve"
+
 [tool.ruff.lint]
 select = [
     "E",    # pycodestyle errors

{stdpipe-0.3.0 → stdpipe-0.4.1}/stdpipe/__init__.py

@@ -6,4 +6,4 @@ detection related tasks, intended for quick and easy implementation of custom
 pipelines, as well as for interactive data analysis.
 """
 
-__version__ = "0.3.0"
+__version__ = "0.4.1"

stdpipe-0.4.1/stdpipe/artefacts.py

@@ -0,0 +1,370 @@
+# Various artefact filtering routines
+#
+# For more advanced real-bogus classification with cutout-based morphological
+# features, see the realbogus_features module:
+#
+#   from stdpipe import realbogus_features as rbf
+#   obj = rbf.classify(obj, image, method='hybrid', classifier='scoring')
+#
+# The realbogus_features module provides:
+# - Catalog-only, cutout-only, or hybrid feature extraction
+# - Multiple classifiers: scoring (no training), IsolationForest, RandomForest
+# - Generalized trend removal for all features
+# - Training utilities for custom classifiers
+
+import numpy as np
+from sklearn.ensemble import IsolationForest
+
+from . import smoothing
+from .realbogus_features import FLAG_BOGUS, FLAGS_UNRELIABLE_MASK
+
+
+def _has_column(obj, name):
+    """Check whether a table-like or dict-like catalog has a column."""
+    colnames = getattr(obj, 'colnames', None)
+    if colnames is not None:
+        return name in colnames
+    if hasattr(obj, 'keys'):
+        return name in obj.keys()
+    names = getattr(getattr(obj, 'dtype', None), 'names', None)
+    return names is not None and name in names
+
+
+def _get_catalog_features(obj):
+    """
+    Build the feature list from available catalog columns.
+
+    Prefers SExtractor-style columns (FLUX_RADIUS, FLUX_MAX/FLUX_AUTO) and
+    falls back to SEP-style ones (peak/flux) where possible, so the same
+    filter works on both `get_objects_sextractor()` and `get_objects_sep()`
+    catalogs.
+    """
+    features = []
+
+    if _has_column(obj, 'FLUX_RADIUS'):
+        features.append([np.asarray(obj['FLUX_RADIUS'], dtype=float), 'FLUX_RADIUS'])
+
+    for name in ('fwhm', 'FWHM_IMAGE'):
+        if _has_column(obj, name):
+            features.append([np.asarray(obj[name], dtype=float), 'FWHM'])
+            break
+
+    for num, den in (('FLUX_MAX', 'FLUX_AUTO'), ('peak', 'flux')):
+        if _has_column(obj, num) and _has_column(obj, den):
+            with np.errstate(divide='ignore', invalid='ignore'):
+                ratio = np.asarray(obj[num], dtype=float) / np.asarray(obj[den], dtype=float)
+            features.append([ratio, f'{num} / {den}'])
+            break
+
+    return features
+
+
+def filter_sextractor_detections(
+    obj,
+    trend_cols=['x', 'y', 'MAG_AUTO'],
+    trend_scales=None,
+    contamination='auto',
+    return_features=False,
+    return_classifier=False,
+    random_state=0,
+    verbose=True,
+    **kwargs,
+):
+    """
+    Flag detections likely to be artefacts using IsolationForest.
+
+    Builds feature vectors from FLUX_RADIUS, FWHM, and peakiness
+    (FLUX_MAX/FLUX_AUTO, or peak/flux for SEP catalogs), using whichever
+    columns are present. Optionally removes smooth spatial trends (e.g.,
+    across x/y/MAG_AUTO) via an approximate LOESS regressor before fitting
+    the outlier model.
+
+    Expected columns in `obj`
+    -------------------------
+    Required:
+    - flags
+    - at least one of the feature columns: FLUX_RADIUS; fwhm or FWHM_IMAGE;
+      FLUX_MAX + FLUX_AUTO or peak + flux
+    Trend columns (when `trend_cols` is set):
+    - columns named in `trend_cols` (default: x, y, MAG_AUTO); missing ones
+      are dropped with a log message
+
+    Parameters
+    ----------
+    obj : array-like / table
+        Detection catalog with required columns.
+    trend_cols : list[str] or None
+        Columns used to model smooth trends; set to None/[] to skip detrending.
+    trend_scales : list[float] or None
+        Per-dimension scaling for LOESS distances; must match trend_cols
+        length. If None (default), scales are auto-computed as ~3x the
+        standard deviation of each trend column.
+    contamination : float or 'auto'
+        Expected fraction of artefacts, passed to IsolationForest. With
+        'auto' (default) the threshold follows the original paper and a
+        data-dependent fraction is flagged even on clean catalogs; set an
+        explicit value (e.g. 0.02) to control the rejection rate.
+    return_features : bool
+        If True, return the feature list (arrays + labels) without fitting.
+    return_classifier : bool
+        If True, return a callable that classifies new catalogs.
+    random_state : int
+        Random seed for IsolationForest.
+    verbose : bool or callable
+        Logging control; can be a print-like function.
+    **kwargs :
+        Extra arguments passed to `ApproxLoessRegressor` (e.g., k, robust_iters).
+
+    Returns
+    -------
+    good : ndarray[bool] or callable
+        Boolean mask of “good” detections, or a classifier callable if
+        return_classifier is True.
+    """
+
+    # Simple wrapper around print for logging in verbose mode only
+    log = (verbose if callable(verbose) else print) if verbose else lambda *args, **kwargs: None
+
+    features = _get_catalog_features(obj)
+
+    if not features:
+        raise ValueError(
+            "No usable feature columns found in the catalog "
+            "(need FLUX_RADIUS, fwhm/FWHM_IMAGE, FLUX_MAX+FLUX_AUTO or peak+flux)"
+        )
+
+    if return_features:
+        return features
+
+    log(
+        "Using isolation forest outlier detection over columns ({})".format(
+            ", ".join([_[1] for _ in features])
+        )
+    )
+
+    # LOESS neighbor count; popped here so it does not leak into other kwargs
+    k = kwargs.pop('k', 20)
+
+    # Exclude blends etc from the fit, as well as broken measurements;
+    # FLAG_BOGUS itself is ignored so re-running the filter is idempotent
+    idx = (obj['flags'] & FLAGS_UNRELIABLE_MASK) == 0
+    for f in features:
+        idx &= np.isfinite(f[0]) & (f[0] > 0)
+
+    pos = None
+    if trend_cols:
+        # Drop trend columns missing from the catalog (keeping user-provided
+        # scales aligned with the surviving columns)
+        available = [_ for _ in trend_cols if _has_column(obj, _)]
+        if trend_scales is not None and len(trend_scales) != len(trend_cols):
+            raise ValueError("trend_scales length is inconsistent with trend_cols length")
+        if len(available) < len(trend_cols):
+            missing = [_ for _ in trend_cols if _ not in available]
+            log("Trend columns not found in catalog: {}".format(", ".join(missing)))
+            if trend_scales is not None:
+                trend_scales = [s for c, s in zip(trend_cols, trend_scales) if c in available]
+        trend_cols = available
+
+    if trend_cols:
+        log("Removing smooth trends in {} using approximate LOESS".format(", ".join(trend_cols)))
+
+        if trend_scales is None:
+            # Auto-compute per-dimension scales as ~3 sigma of each column
+            trend_scales = []
+            for col in trend_cols:
+                scale = 3.0 * np.nanstd(np.asarray(obj[col], dtype=float))
+                if not np.isfinite(scale) or scale <= 0:
+                    scale = 1.0
+                trend_scales.append(scale)
+            log(
+                "Auto trend scales: {}".format(
+                    ", ".join([f"{c}={s:.3g}" for c, s in zip(trend_cols, trend_scales)])
+                )
+            )
+
+        pos = np.column_stack([np.array(obj[_]) for _ in trend_cols])
+        # Rows with non-finite trend columns cannot be detrended and would
+        # get NaN features anyway; keep them out of the fits below
+        idx &= np.all(np.isfinite(pos), axis=1)
+
+    if np.sum(idx) < 3:
+        raise ValueError(
+            f"Too few clean detections ({np.sum(idx)}) to fit the outlier model"
+        )
+
+    if trend_cols:
+        trend_models = []
+        X = []
+
+        for f in features:
+            model = smoothing.ApproxLoessRegressor(k=k, scales=trend_scales, **kwargs)
+            model.fit(pos[idx], f[0][idx])
+            trend_models.append(model)
+            X.append(np.array(f[0]) - model.predict(pos))
+
+    else:
+        trend_models = None
+        X = [np.array(_[0]) for _ in features]
+
+    X = np.column_stack(X)
+    X[~np.isfinite(X)] = -100000  # Definitely outside of the good locus
+
+    clf = IsolationForest(contamination=contamination, random_state=random_state).fit(X[idx])
+
+    res = clf.predict(X)
+
+    log(f"{np.sum(res > 0)} good, {np.sum(res < 0)} outliers")
+
+    if return_classifier:
+
+        def classifier(obj):
+            features = _get_catalog_features(obj)
+
+            if trend_cols:
+                pos = np.column_stack([np.array(obj[_]) for _ in trend_cols])
+                X = []
+
+                for f, model in zip(features, trend_models):
+                    X.append(np.array(f[0]) - model.predict(pos))
+            else:
+                X = [np.array(_[0]) for _ in features]
+
+            X = np.column_stack(X)
+            X[~np.isfinite(X)] = -100000  # Definitely outside of the good locus
+
+            return clf.predict(X) > 0
+
+        return classifier
+
+    return res > 0
+
+
+def filter_detections(
+    obj,
+    image=None,
+    bg=None,
+    err=None,
+    mask=None,
+    fwhm=None,
+    method='auto',
+    classifier='isolation',
+    threshold=0.5,
+    remove_trend=True,
+    trend_cols=None,
+    trend_scales=None,
+    add_score=False,
+    flag_bogus=False,
+    verbose=True,
+    **kwargs,
+):
+    """
+    Filter detections using feature-based real-bogus classification.
+
+    This is a convenience wrapper around realbogus_features.classify() that
+    provides a simpler interface similar to filter_sextractor_detections().
+
+    For full control over feature extraction and classification, use
+    realbogus_features.classify() directly.
+
+    Parameters
+    ----------
+    obj : astropy.table.Table
+        Object catalog with 'x', 'y' columns. Modified in place when
+        `add_score` or `flag_bogus` is set.
+    image : ndarray, optional
+        Science image. If provided, cutout features will be extracted.
+    bg : ndarray or float, optional
+        Background map or scalar.
+    err : ndarray or float, optional
+        Error/noise map or scalar.
+    mask : ndarray, optional
+        Boolean mask (True = masked).
+    fwhm : float, optional
+        Image FWHM. If None, estimated from catalog.
+    method : str, optional
+        Feature extraction method:
+        - 'catalog': Catalog features only (no image needed)
+        - 'cutout': Cutout features only
+        - 'hybrid': Both catalog and cutout features
+        - 'auto': 'hybrid' if image provided, else 'catalog'
+    classifier : str, optional
+        Classifier to use:
+        - 'scoring': Rule-based scoring (no training needed)
+        - 'isolation': IsolationForest (unsupervised, default)
+    threshold : float, optional
+        Score threshold for classification. Default: 0.5.
+    remove_trend : bool, optional
+        Remove spatial trends from features. Default: True.
+    trend_cols : list of str, optional
+        Columns for trend removal. Default: ['x', 'y'].
+    trend_scales : list of float, optional
+        Scales for trend removal. Default: auto-computed.
+    add_score : bool, optional
+        Add 'rb_score' column to the input catalog. Default: False.
+    flag_bogus : bool, optional
+        Set FLAG_BOGUS (0x4000) on bogus objects in the input catalog.
+        Default: False.
+    verbose : bool, optional
+        Print progress.
+    **kwargs
+        Additional arguments passed to realbogus_features.classify().
+
+    Returns
+    -------
+    good : ndarray[bool]
+        Boolean mask of "good" (real) detections.
+
+    See Also
+    --------
+    realbogus_features.classify : Full-featured classification function.
+    filter_sextractor_detections : Original SExtractor-specific filter.
+
+    Examples
+    --------
+    >>> # Simple catalog-only filtering (like original function)
+    >>> good = filter_detections(obj, classifier='isolation')
+
+    >>> # Cutout-based filtering with scoring (no training)
+    >>> good = filter_detections(obj, image, classifier='scoring')
+
+    >>> # Hybrid with trend removal
+    >>> good = filter_detections(obj, image, method='hybrid',
+    ...                          remove_trend=True, trend_cols=['x', 'y'])
+    """
+    from . import realbogus_features as rbf
+
+    log = (verbose if callable(verbose) else print) if verbose else lambda *args, **kwargs: None
+
+    # Call the full classify function; the score is always needed to build
+    # the output mask
+    result = rbf.classify(
+        obj,
+        image=image,
+        bg=bg,
+        err=err,
+        mask=mask,
+        fwhm=fwhm,
+        method=method,
+        classifier=classifier,
+        threshold=threshold,
+        add_score=True,
+        flag_bogus=flag_bogus,
+        remove_trend=remove_trend,
+        trend_cols=trend_cols,
+        trend_scales=trend_scales,
+        verbose=verbose,
+        **kwargs,
+    )
+
+    good = np.asarray(result['rb_score'] >= threshold)
+
+    # classify() works on a copy; propagate the requested columns back to
+    # the caller's catalog so add_score/flag_bogus have a visible effect
+    if add_score:
+        obj['rb_score'] = result['rb_score']
+    if flag_bogus:
+        obj['flags'] = result['flags']
+
+    log(f"{np.sum(good)} good, {np.sum(~good)} outliers")
+
+    return good