stdpipe 0.2.4__tar.gz → 0.3.0__tar.gz

stdpipe-0.3.0/.gitignore

@@ -0,0 +1,88 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Virtual environments
+venv/
+ENV/
+env/
+.venv
+
+# IDEs
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+.dir-locals.el
+flycheck_*.py
+*#
+
+# Testing
+.pytest_cache/
+.coverage
+.coverage.*
+htmlcov/
+.tox/
+.hypothesis/
+*.cover
+.cache
+nosetests.xml
+coverage.xml
+*.log
+
+# Documentation
+docs/_build/
+docs/_static/
+docs/_templates/
+doc/_build/
+doc/api/
+.readthedocs.yaml.1
+
+# Jupyter
+.ipynb_checkpoints/
+notebooks/Untitled*.ipynb
+
+# OS
+.DS_Store
+.Rhistory
+.RData
+
+# Build and distribution
+*.tar.gz
+*.whl
+
+# Aider
+.aider*
+
+# Temporary files
+backup/
+*.tmp
+*.bak
+*~
+
+# Misc files
+*.new
+*.orig
+
+test_data/
+pyraf/

stdpipe-0.3.0/PKG-INFO

@@ -0,0 +1,161 @@
+Metadata-Version: 2.4
+Name: stdpipe
+Version: 0.3.0
+Summary: Simple Transient Detection Pipeline
+Author-email: Sergey Karpov <karpov.sv@gmail.com>
+Maintainer-email: Sergey Karpov <karpov.sv@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2022 Sergey Karpov
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+        
+Project-URL: Homepage, https://github.com/karpov-sv/stdpipe
+Project-URL: Documentation, https://stdpipe.readthedocs.io/
+Project-URL: Repository, https://github.com/karpov-sv/stdpipe
+Project-URL: Bug Tracker, https://github.com/karpov-sv/stdpipe/issues
+Project-URL: Changelog, https://github.com/karpov-sv/stdpipe/commits/master
+Keywords: astronomy,astrophysics,transients,photometry,astrometry,image-processing,pipeline
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Astronomy
+Classifier: Topic :: Scientific/Engineering :: Image Processing
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: numpy
+Requires-Dist: scipy
+Requires-Dist: astropy>=4.1
+Requires-Dist: matplotlib
+Requires-Dist: astroquery>0.4.1
+Requires-Dist: sep
+Requires-Dist: astroscrappy
+Requires-Dist: photutils>=2.2
+Requires-Dist: scikit-image
+Requires-Dist: statsmodels
+Requires-Dist: tqdm
+Requires-Dist: regions
+Requires-Dist: python-dateutil
+Requires-Dist: requests
+Requires-Dist: sip_tpv
+Requires-Dist: pyfftw
+Provides-Extra: dev
+Requires-Dist: pytest>=6.0; extra == "dev"
+Requires-Dist: pytest-cov>=2.10; extra == "dev"
+Requires-Dist: pytest-xdist>=2.0; extra == "dev"
+Requires-Dist: pytest-mock>=3.0; extra == "dev"
+Requires-Dist: pytest-timeout>=2.0; extra == "dev"
+Requires-Dist: pytest-benchmark>=3.4; extra == "dev"
+Requires-Dist: hypothesis>=6.0; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: sphinx>=4.0; extra == "docs"
+Requires-Dist: sphinx-rtd-theme; extra == "docs"
+Requires-Dist: sphinx-math-dollar; extra == "docs"
+Requires-Dist: numpydoc; extra == "docs"
+Requires-Dist: myst-parser; extra == "docs"
+Requires-Dist: sphinxcontrib-apidoc; extra == "docs"
+Requires-Dist: sphinx-prompt; extra == "docs"
+Provides-Extra: adaptive
+Requires-Dist: powerbin; extra == "adaptive"
+Provides-Extra: all
+Requires-Dist: stdpipe[adaptive,dev,docs]; extra == "all"
+Dynamic: license-file
+
+# STDPipe - Simple Transient Detection Pipeline
+*AKA: random codes noone else will ever use*
+
+*STDPipe* is a set of Python routines for astrometry, photometry and transient detection related tasks, intended for quick and easy implementation of custom pipelines, as well as for interactive data analysis.
+
+### Design principles
+ - implemented as a library of routines covering most common tasks
+ - operates on standard Python objects: NumPy arrays for images, Astropy Tables for catalogs and object lists, etc
+ - does not try to re-implement the things already implemented in other Python packages
+ - conveniently wraps external codes that do not have their own Python interfaces (*SExtractor*, *SCAMP*, *PSFEx*, *HOTPANTS*, *Astrometry.Net*, ...)
+     - wrapping is transparent: all data passed from Python, all options customizable from Python, all (or most of) outputs available back
+     - everything operates on temporary files, nothing is kept after the run unless explicitly asked for
+
+### Features
+ - ~~pre-processing~~ - should be handled before in an instrument-specific way
+     - bias/dark subtraction, flatfielding, masking
+ - object detection and photometry
+     - SExtractor or SEP for detection, photutils for photometry
+ - astrometric calibration
+     - Astrometry.Net for blind WCS solving
+     - SCAMP or Astropy-based code for refinement
+ - photometric calibration
+     - Vizier catalogues, passband conversion (PS1 to Johnson, Gaia to Johnson, ...)
+     - spatial polynomial + color term + intrinsic scatter
+ - image subtraction
+     - HiPS templates
+     - PanSTARRS DR1 or Legacy Survey templates
+     - HOTPANTS + custom noise model
+     - ZOGY
+ - transient detection and photometry
+     - noise-weighted detection, cutout adjustment, ...
+ - auxiliary functions
+     - PSF estimation, simulated stars, FITS header utilities, plotting, ...
+ - light curve creation (soon)
+     - spatial clustering, color regression, variability analysis, ...
+
+# Installation
+
+*STDpipe* is available at https://github.com/karpov-sv/stdpipe and is mirrored at https://gitlab.in2p3.fr/icare/stdpipe
+
+The package is in constant development, so to keep track of the changes the suggested way of installing it is by cloning the repository
+```
+git clone https://github.com/karpov-sv/stdpipe.git
+```
+and then installing from it in development (or "editable") mode by running the command
+```
+cd stdpipe
+python3 -m pip install -e .
+```
+This way you may update the repository or apply local patches, and it will immediately be reflected in the installed package.
+
+Apart of Python requirements that will be installed automatically, *STDPipe* also (optionally) makes use of the following external software:
+ - [SExtractor](https://github.com/astromatic/sextractor)
+ - [SCAMP](https://github.com/astromatic/scamp)
+ - [PSFEx](https://github.com/astromatic/psfex)
+ - [SWarp](https://github.com/astromatic/swarp)
+ - [HOTPANTS](https://github.com/acbecker/hotpants)
+ - [Astrometry.Net](https://github.com/dstndstn/astrometry.net)
+
+Most of them may be installed from your package manager. E.g. on Debian or Ubuntu systems it may look like that:
+```
+sudo apt install sextractor scamp psfex swarp
+```
+or, on Miniconda/Anaconda, like that:
+```
+conda install -c conda-forge astromatic-source-extractor astromatic-scamp astromatic-psfex astromatic-swarp
+```
+
+You may also check more detailed installation instructions [here](https://stdpipe.readthedocs.io/en/latest/installation.html).
+
+# Usage
+
+Please consult the [documentation for *STDPipe*](https://stdpipe.readthedocs.io/) for the basic usage patterns and description of its API. You may check the examples inside [notebooks/](notebooks/) folder, especially the [tutorial](notebooks/stdpipe_tutorial.ipynb) that demonstrates basic steps of a typical image processing.

stdpipe-0.3.0/pyproject.toml

@@ -0,0 +1,275 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "stdpipe"
+version = "0.3.0"
+description = "Simple Transient Detection Pipeline"
+readme = "README.md"
+license = {file = "LICENSE.md"}
+authors = [
+    {name = "Sergey Karpov", email = "karpov.sv@gmail.com"}
+]
+maintainers = [
+    {name = "Sergey Karpov", email = "karpov.sv@gmail.com"}
+]
+keywords = [
+    "astronomy",
+    "astrophysics",
+    "transients",
+    "photometry",
+    "astrometry",
+    "image-processing",
+    "pipeline"
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Astronomy",
+    "Topic :: Scientific/Engineering :: Image Processing",
+]
+requires-python = ">=3.8"
+
+# Core dependencies
+dependencies = [
+    "numpy",
+    "scipy",
+    "astropy>=4.1",
+    "matplotlib",
+    "astroquery>0.4.1",
+    "sep",
+    "astroscrappy",
+    "photutils>=2.2",
+    "scikit-image",
+    "statsmodels",
+    "tqdm",
+    "regions",
+    "python-dateutil",
+    "requests",
+    # "sip_tpv @ git+https://github.com/pmvreeswijk/sip_tpv.git",
+    "sip_tpv",
+    "pyfftw",
+]
+
+[project.optional-dependencies]
+# Development dependencies
+dev = [
+    "pytest>=6.0",
+    "pytest-cov>=2.10",
+    "pytest-xdist>=2.0",
+    "pytest-mock>=3.0",
+    "pytest-timeout>=2.0",
+    "pytest-benchmark>=3.4",
+    "hypothesis>=6.0",
+]
+
+# Documentation dependencies
+docs = [
+    "sphinx>=4.0",
+    "sphinx-rtd-theme",
+    "sphinx-math-dollar",
+    "numpydoc",
+    "myst-parser",
+    "sphinxcontrib-apidoc",
+    "sphinx-prompt",
+]
+
+# Adaptive binning (Voronoi tessellation)
+adaptive = [
+    "powerbin",
+]
+
+# All optional dependencies
+all = [
+    "stdpipe[dev,docs,adaptive]",
+]
+
+[project.urls]
+Homepage = "https://github.com/karpov-sv/stdpipe"
+Documentation = "https://stdpipe.readthedocs.io/"
+Repository = "https://github.com/karpov-sv/stdpipe"
+"Bug Tracker" = "https://github.com/karpov-sv/stdpipe/issues"
+Changelog = "https://github.com/karpov-sv/stdpipe/commits/master"
+
+# ============================================================================
+# Tool configurations
+# ============================================================================
+
+[tool.setuptools]
+zip-safe = false
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["stdpipe*"]
+exclude = ["tests*", "docs*", "notebooks*", "build*", "dist*"]
+
+[tool.setuptools.package-data]
+stdpipe = ["data/*"]
+
+# ============================================================================
+# pytest configuration
+# ============================================================================
+
+[tool.pytest.ini_options]
+minversion = "6.0"
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+python_classes = ["Test*"]
+python_functions = ["test_*"]
+addopts = [
+    "-v",
+    "--strict-markers",
+    "--showlocals",
+    "-ra",
+]
+markers = [
+    "unit: Unit tests that don't require external dependencies",
+    "integration: Integration tests that may require external tools",
+    "slow: Tests that take significant time to run",
+    "requires_sextractor: Tests requiring SExtractor to be installed",
+    "requires_scamp: Tests requiring SCAMP to be installed",
+    "requires_psfex: Tests requiring PSFEx to be installed",
+    "requires_hotpants: Tests requiring HOTPANTS to be installed",
+    "requires_astrometry_net: Tests requiring Astrometry.Net to be installed",
+    "requires_pyraf: Tests requiring PyRAF/IRAF to be installed",
+    "requires_network: Tests that require network access",
+]
+filterwarnings = [
+    "ignore::FutureWarning",
+    "ignore::DeprecationWarning:astropy.*",
+    "ignore::astropy.wcs.FITSFixedWarning",
+]
+
+# ============================================================================
+# coverage configuration
+# ============================================================================
+
+[tool.coverage.run]
+source = ["stdpipe"]
+omit = [
+    "*/tests/*",
+    "*/test_*.py",
+    "setup.py",
+    "*/flycheck_*.py",
+]
+parallel = true
+branch = true
+
+[tool.coverage.report]
+precision = 2
+show_missing = true
+skip_covered = false
+exclude_lines = [
+    "pragma: no cover",
+    "def __repr__",
+    "raise AssertionError",
+    "raise NotImplementedError",
+    "if __name__ == .__main__.:",
+    "if TYPE_CHECKING:",
+    "class .*\\bProtocol\\):",
+    "@(abc\\.)?abstractmethod",
+]
+
+[tool.coverage.html]
+directory = "htmlcov"
+
+# ============================================================================
+# Black code formatter
+# ============================================================================
+
+[tool.black]
+line-length = 100
+target-version = ["py38", "py39", "py310", "py311", "py312"]
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+    \.git
+  | \.venv
+  | build
+  | dist
+  | __pycache__
+  | \.pytest_cache
+  | \.eggs
+)/
+'''
+
+# ============================================================================
+# isort import sorter
+# ============================================================================
+
+[tool.isort]
+profile = "black"
+line_length = 100
+multi_line_output = 3
+include_trailing_comma = true
+force_grid_wrap = 0
+use_parentheses = true
+ensure_newline_before_comments = true
+skip_glob = ["*/flycheck_*.py"]
+
+# ============================================================================
+# ruff linter (modern alternative to flake8)
+# ============================================================================
+
+[tool.ruff]
+line-length = 100
+target-version = "py38"
+extend-exclude = [
+    "flycheck_*.py",
+    "__pycache__",
+    ".pytest_cache",
+    "build",
+    "dist",
+]
+
+[tool.ruff.lint]
+select = [
+    "E",    # pycodestyle errors
+    "W",    # pycodestyle warnings
+    "F",    # pyflakes
+    "I",    # isort
+    "N",    # pep8-naming
+    "UP",   # pyupgrade
+    "B",    # flake8-bugbear
+    "C4",   # flake8-comprehensions
+    "SIM",  # flake8-simplify
+]
+ignore = [
+    "E501",   # line too long (handled by black)
+    "B008",   # do not perform function calls in argument defaults
+    "C901",   # function is too complex
+    "N803",   # argument name should be lowercase (conflicts with astronomical conventions)
+    "N806",   # variable in function should be lowercase (conflicts with astronomical conventions)
+]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401"]  # Unused imports in __init__.py
+"tests/*" = ["F401", "F811"]  # Unused imports and redefinitions in tests
+
+# ============================================================================
+# mypy type checker
+# ============================================================================
+
+[tool.mypy]
+python_version = "3.8"
+warn_return_any = true
+warn_unused_configs = true
+disallow_untyped_defs = false  # Start lenient, can tighten later
+ignore_missing_imports = true
+exclude = [
+    "tests/",
+    "docs/",
+    "build/",
+    "dist/",
+]

stdpipe-0.3.0/setup.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+Minimal setup.py for backward compatibility.
+
+Modern configuration is in pyproject.toml.
+This file is kept for backward compatibility with older pip versions
+and tools that don't yet support PEP 517/518.
+
+For development installation:
+    pip install -e .
+
+For installation with optional dependencies:
+    pip install -e ".[dev]"
+    pip install -e ".[docs]"
+    pip install -e ".[all]"
+"""
+
+from setuptools import setup
+
+# All configuration is in pyproject.toml
+# This setup.py is just for backward compatibility
+if __name__ == "__main__":
+    setup()

{stdpipe-0.2.4 → stdpipe-0.3.0}/stdpipe/__init__.py

@@ -5,3 +5,5 @@ STDPipe - Simple Transient Detection Pipeline.
 detection related tasks, intended for quick and easy implementation of custom
 pipelines, as well as for interactive data analysis.
 """
+
+__version__ = "0.3.0"

{stdpipe-0.2.4 → stdpipe-0.3.0}/stdpipe/astrometry.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import, division, print_function, unicode_literals
 
 import os, tempfile, shutil, shlex, re, warnings
 import numpy as np
@@ -123,11 +122,11 @@ def spherical_match(ra1, dec1, ra2, dec2, sr=1 / 3600):
 
     """
 
-    # Ensure that inputs are arrays
-    ra1 = np.atleast_1d(ra1)
-    dec1 = np.atleast_1d(dec1)
-    ra2 = np.atleast_1d(ra2)
-    dec2 = np.atleast_1d(dec2)
+    # Ensure that inputs are arrays, and drop units if any
+    ra1 = np.array(np.atleast_1d(ra1))
+    dec1 = np.array(np.atleast_1d(dec1))
+    ra2 = np.array(np.atleast_1d(ra2))
+    dec2 = np.array(np.atleast_1d(dec2))
 
     idx1, idx2, dist, _ = search_around_sky(
         SkyCoord(ra1, dec1, unit='deg'), SkyCoord(ra2, dec2, unit='deg'), sr * u.deg
@@ -299,6 +298,7 @@ def blind_match_objects(
         'sort-column': 'FLUX',
         'width': width,
         'height': height,
+        'crpix-center': True,
         #
         'overwrite': True,
         'no-plots': True,
@@ -309,7 +309,7 @@ def blind_match_objects(
     if config is not None:
         opts['config'] = config
 
-    if order is not None:
+    if order is not None and order > 0:
         opts['tweak-order'] = order
     else:
         opts['no-tweak'] = True
@@ -330,52 +330,43 @@ def blind_match_objects(
 
     opts.update(extra)
 
-    # Build the command line
-    command = binname + ' ' + shlex.quote(objname) + ' ' + utils.format_long_opts(opts)
-    if not verbose:
-        command += ' > /dev/null 2>/dev/null'
-    log('Will run first iteration of Astrometry.Net like that:')
-    log(command)
-
-    res = os.system(command)
-
     wcs = None
-    if res == 0 and os.path.exists(wcsname):
-        log('Successfully run first iteration')
-        shutil.move(wcsname, tmpname)
 
-        opts['verify'] = tmpname
-        command = (
-            binname + ' ' + shlex.quote(objname) + ' ' + utils.format_long_opts(opts)
-        )
+    for iter in range(2):
+        if os.path.exists(wcsname):
+            shutil.move(wcsname, tmpname)
+            opts['verify'] = tmpname
+
+        # Build the command line
+        command = binname + ' ' + shlex.quote(objname) + ' ' + utils.format_long_opts(opts)
         if not verbose:
             command += ' > /dev/null 2>/dev/null'
-        log('Will run second iteration of Astrometry.Net like that:')
+        log('Will run iteration %d of Astrometry.Net like that:' % (iter,))
         log(command)
 
         res = os.system(command)
 
         if res == 0 and os.path.exists(wcsname):
-            log('Successfully run second iteration')
-            header = fits.getheader(wcsname)
-            wcs = WCS(header)
+            log('Successfully run iteration %d' % (iter,))
 
-            ra0, dec0, sr0 = get_frame_center(wcs=wcs, width=width, height=height)
-            pixscale = get_pixscale(wcs=wcs)
+        else:
+            log('Error %s running Astrometry.Net' % res)
 
-            log(
-                'Got WCS solution with center at %.4f %.4f radius %.2f deg and pixel scale %.2f arcsec/pix'
-                % (ra0, dec0, sr0, pixscale * 3600)
-            )
 
-            if update and wcs:
-                obj['ra'], obj['dec'] = wcs.all_pix2world(obj['x'], obj['y'], 0)
+    if res == 0 and os.path.exists(wcsname):
+        header = fits.getheader(wcsname)
+        wcs = WCS(header)
 
-        else:
-            log('Error %s running Astrometry.Net' % res)
+        ra0, dec0, sr0 = get_frame_center(wcs=wcs, width=width, height=height)
+        pixscale = get_pixscale(wcs=wcs)
 
-    else:
-        log('Error %s running Astrometry.Net' % res)
+        log(
+            'Got WCS solution with center at %.4f %.4f radius %.2f deg and pixel scale %.2f arcsec/pix'
+            % (ra0, dec0, sr0, pixscale * 3600)
+        )
+
+        if update and wcs:
+            obj['ra'], obj['dec'] = wcs.all_pix2world(obj['x'], obj['y'], 0)
 
     if _workdir is None:
         shutil.rmtree(workdir)