dde-cargo 0.1.0__py3-none-any.whl

dde_cargo/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) 2026 Tim Molteno (tim@elec.ac.nz)

dde_cargo/ddfacet-memory-plan.yml

@@ -0,0 +1,335 @@
+# Copied verbatim from the installed package in ddfacet:v0.10.0.6
+# (/opt/venv/lib/python3.13/site-packages/DDFacet/stimela/ddfacet-memory-plan.yml):
+# v0.10.0.6 ships both the cab and the module, so nothing is vendored any more.
+# The image pin follows the ddfacet pin (v0.12.1 since 2026-09-25): the planner is
+# exercised on the same line the imaging runs, so its memory model matches.
+# stimela cab: ddfacet-memory-plan
+#
+# Suggests --Parallel-NCPU, --Data-ChunkHours and (only when the band has to
+# give) --Selection-ChanStep for a DDFacet run, from the parset that run will
+# use and a memory budget.  This is the command-line form of
+# DDFacet/ToolsDir/ModEstimateMemory.py, and it ships with the package so a
+# recipe can list it directly:
+#
+#     - /opt/venv/lib/python3.13/site-packages/DDFacet/stimela/ddfacet-memory-plan.yml
+#
+# or copy the cab into the recipe's own cabs/ directory (that is what
+# meerkat_imaging does with the other DDFacet cabs).
+#
+# Wiring it into a recipe, ahead of the imaging step:
+#
+#     memory-plan:
+#         cab: ddfacet-memory-plan
+#         params:
+#             Parset: =steps.<something>.parset_out
+#             MS: =recipe.target-ms
+#             RamGB: 62
+#     ddfacet-target:
+#         params:
+#             Parallel.NCPU: =steps.memory-plan.NCPU
+#             Data.ChunkHours: =steps.memory-plan.ChunkHours
+#             Selection.ChanStep: =steps.memory-plan.ChanStep
+#
+# The estimate is +-40 % (see MANAGING_LIMITED_MEMORY.md): use it to choose
+# between "NCPU 3" and "NCPU 8", not to plan within a gigabyte of the limit.
+#
+# THREADED (the rayon line, DDFacet >= v0.12): NCPU IS NOT A MEMORY KNOB ANY MORE.
+# ModEstimateMemory's model is `main + NCPU x per-worker + chunk buffers`,
+# calibrated on the process pool, where every worker held its own copies.  On the
+# rayon line the workers are threads in one process, and a sweep of the same
+# dirty image on moist (8 cores, uv1500 fast profile, target 1, 2026-09-27)
+# measured:
+#
+#     NCPU  ChunkHours   wall   CPU   largest process
+#       2      2.0      3:35   115%      13.8 GB
+#       4      2.0      1:49   205%      13.0 GB
+#       8      2.0      1:11   422%      13.3 GB
+#       2      0.5      3:37   115%      13.7 GB
+#       8      0.5      1:11   423%      13.3 GB
+#
+# i.e. flat in NCPU, 3x faster with every core -- while the model predicts
+# 14.1 GB at NCPU 1 (right) and 2.18 GB per extra worker (wrong: 28.6 GB at 8),
+# so with an 18 GB budget the unmodified planner answered `NCPU 1`.  With
+# `Threaded: true` (the default) the memory is therefore planned at ONE worker --
+# ChunkHours and, if the band must give, ChanStep -- and NCPU is every usable
+# core (or WantNCPU).  `Threaded: false` restores the process-pool behaviour.
+#
+# RamGB and WantNCPU default to what the host actually offers when the step runs
+# (MemAvailable, capped by a cgroup memory limit if there is one; the CPU
+# affinity set), so the same recipe plans correctly on a 62 GB/12-core run box
+# and on a 30 GB/8-core desktop.  MemWatermarkGB is an output (15 % of the
+# budget): wire it to --Parallel-MemWatermarkGB instead of a fixed value, which
+# deadlocks a machine with less than that free (the pool workers pause for
+# memory the parent holds while it waits for them).
+#
+# Bootstrapping, since a step cannot be planned from the parset it is about to
+# write: plan each step from the parset of the *previous* run of that step (the
+# pipeline's pattern), and for a fresh observation -- where no earlier parset
+# exists -- pass the geometry instead of a parset (NPix, and optionally
+# NFacets/NBand/OverS/Support/Nw/ChunkHoursIn).  PlanFile is written by the
+# command, so it is declared `must_exist: false`: a File input that stimela
+# checks for existence would fail the step before the planner ran (DDFacet#17).
+
+vars:
+  ddfacet-memory-plan:
+    image:
+      registry: ghcr.io/tmolteno
+      name: ddfacet
+      # Pin this to whatever the recipe uses; a recipe with its own image var
+      # should override `image:` on the cab instead.
+      version: v0.16.1
+
+cabs:
+    ddfacet-memory-plan:
+        info: |
+            Suggest --Parallel-NCPU, --Data-ChunkHours and (only if the band has to
+            give) --Selection-ChanStep for a DDFacet run, from its parset and a
+            memory budget.  Reads the MS for its row/channel/antenna counts when
+            one is given; pass NRows/NChan/NA instead when it cannot be read.
+            Needs an image carrying DDFacet/ToolsDir/ModEstimateMemory.py.
+        image:
+            _use: vars.ddfacet-memory-plan.image
+        inputs:
+            Parset:
+                dtype: File
+                required: false
+                info: |
+                    the parset whose geometry is being planned (e.g. the parset_out
+                    of a ddfacet step).  Optional: with no parset, the geometry is
+                    taken from the inputs below, which is the bootstrap case for a
+                    fresh observation that has no earlier parset to plan from.
+            MS:
+                dtype: MS
+                required: false
+                info: measurement set to size the estimate with
+            RamGB:
+                dtype: float
+                required: false
+                info: >
+                    memory the run may use.  Default: what the host has available
+                    when the step runs -- MemAvailable, capped by the cgroup memory
+                    limit if there is one (a container or systemd scope)
+            NPix:
+                dtype: int
+                required: false
+                info: image size, when there is no Parset to read it from
+            NFacets:
+                dtype: int
+                required: false
+            NBand:
+                dtype: int
+                required: false
+            OverS:
+                dtype: int
+                required: false
+            Support:
+                dtype: int
+                required: false
+            Nw:
+                dtype: int
+                required: false
+            ChunkHoursIn:
+                dtype: float
+                required: false
+                info: the chunk size you intend, when planning without a Parset
+                default: 0.5
+            WantNCPU:
+                dtype: int
+                required: false
+                info: >
+                    worker count you would like; also the cap.  Default: the usable
+                    cores (the CPU affinity set, so a taskset/cpuset is respected)
+            Threaded:
+                dtype: bool
+                default: true
+                info: >
+                    the rayon line (DDFacet >= v0.12), where workers are threads in one
+                    process and NCPU costs no measurable memory: plan the memory at one
+                    worker and return every usable core as NCPU.  false = the process
+                    pool, where each worker costs memory (the model's original form)
+            Prefer:
+                dtype: str
+                default: fidelity
+                info: "'fidelity' keeps the full band while any worker count fits; 'speed' takes the workers first and subsamples the band if it must"
+            NDirections:
+                dtype: int
+                default: 1
+            NCorr:
+                dtype: int
+                default: 2
+            NRows:
+                dtype: int
+                required: false
+            NChan:
+                dtype: int
+                required: false
+            NA:
+                dtype: int
+                required: false
+            PlanOut:
+                dtype: File
+                default: ddfacet-memory-plan.json
+                # the command *writes* this file, so stimela must not require it
+                # to exist before the step starts (DDFacet#17)
+                must_exist: false
+                info: where to write the plan, as JSON
+        outputs:
+            NCPU:
+                dtype: int
+                info: pass to --Parallel-NCPU
+            ChunkHours:
+                dtype: float
+                info: pass to --Data-ChunkHours
+            ChanStep:
+                dtype: int
+                info: pass to --Selection-ChanStep
+            MemWatermarkGB:
+                dtype: float
+                info: >
+                    pass to --Parallel-MemWatermarkGB: 15 % of the budget, so the
+                    throttle scales with the machine instead of a fixed floor that
+                    can exceed what is free (and then deadlocks)
+            RamGBUsed:
+                dtype: float
+                info: the budget the plan was made for (RamGB, or what the host offered)
+            PeakGB:
+                dtype: float
+                info: predicted peak as per-process accounting sees it (RSS sums / stimela's Mem GB)
+            HostRAMGB:
+                dtype: float
+                info: predicted peak as `free -g` sees it -- the shared arrays count as RAM only when /dev/shm is a tmpfs
+            Flags:
+                dtype: str
+                info: the three flags, ready to paste
+            Plan:
+                dtype: File
+                must_exist: false
+                info: the JSON plan (chosen settings, every combination that fits, notes)
+        flavour:
+            kind: python-code
+        command: |
+            import json
+            import math
+            from DDFacet.ToolsDir.ModEstimateMemory import suggestParameters
+
+            # Optional inputs may arrive unset, in which case stimela does not
+            # necessarily define the name at all -- look them up defensively.
+            _g = globals()
+
+
+            def _opt(name, default=None):
+                value = _g.get(name, default)
+                return default if value is None else value
+
+            # No Parset?  Build the geometry from the inputs instead, so a fresh
+            # observation can be planned before any parset exists (DDFacet#17).
+            _geometry = {}
+            for _key in ("NPix", "NFacets", "NBand", "OverS", "Support", "Nw"):
+                _value = _opt(_key)
+                if _value is not None:
+                    _geometry[_key] = _value
+            if _opt("Parset") is None and not _geometry:
+                raise RuntimeError(
+                    "ddfacet-memory-plan: give either Parset or the geometry "
+                    "(NPix, and optionally NFacets/NBand/OverS/Support/Nw)")
+            _pardict = None
+            if _opt("Parset") is None:
+                _pardict = {
+                    "Image": {"NPix": _geometry.get("NPix", 5000)},
+                    "Facets": {"NFacets": _geometry.get("NFacets", 3), "Padding": 1.7},
+                    "Freq": {"NBand": _geometry.get("NBand", 1)},
+                    "Output": {"NGridPol": 1},
+                    "CF": {"OverS": _geometry.get("OverS", 11),
+                           "Support": _geometry.get("Support", 7),
+                           "Nw": _geometry.get("Nw", 100)},
+                    "Data": {"ChunkHours": _opt("ChunkHoursIn", 0.5)},
+                    "Parallel": {"NCPU": 1},
+                }
+                print("no Parset given: planning from the inputs (NPix %d, %d "
+                      "facet(s), %d band(s))" % (_pardict["Image"]["NPix"],
+                                                 _pardict["Facets"]["NFacets"],
+                                                 _pardict["Freq"]["NBand"]))
+
+            import os
+
+
+            def _usable_cores():
+                try:
+                    return len(os.sched_getaffinity(0))
+                except AttributeError:
+                    return os.cpu_count() or 1
+
+
+            def _available_gb():
+                """MemAvailable, capped by the cgroup (v2) limit: what this run can use."""
+                avail = None
+                with open("/proc/meminfo") as _fh:
+                    for _line in _fh:
+                        if _line.startswith("MemAvailable:"):
+                            avail = int(_line.split()[1]) * 1024
+                try:
+                    _cg = open("/proc/self/cgroup").read().strip().split("::", 1)[-1]
+                    _dir = "/sys/fs/cgroup" + _cg
+                    _max = open(_dir + "/memory.max").read().strip()
+                    if _max != "max":
+                        _free = int(_max) - int(open(_dir + "/memory.current").read())
+                        avail = _free if avail is None else min(avail, _free)
+                except (OSError, ValueError):
+                    pass
+                return avail / float(1 << 30)
+
+
+            ram_gb = _opt("RamGB")
+            ram_source = "RamGB"
+            if ram_gb is None:
+                ram_gb = _available_gb()
+                ram_source = "available on this host"
+            cores = int(_opt("WantNCPU") or _usable_cores())
+            threaded = bool(_opt("Threaded", True))
+            print("budget %.1f GB (%s); %d core(s)%s; %s model"
+                  % (ram_gb, ram_source, cores,
+                     "" if _opt("WantNCPU") else " usable",
+                     "threaded (memory planned at one worker)" if threaded
+                     else "process-pool"))
+
+            # Threaded: the memory does not move with NCPU (measured flat, header),
+            # so the feasibility search runs at one worker and NCPU is every core.
+            suggestion = suggestParameters(
+                GD=_pardict, parset=_opt("Parset"), ms=_opt("MS"), ram_gb=ram_gb,
+                ncpu=1 if threaded else cores, max_ncpu=1 if threaded else cores,
+                prefer=_opt("Prefer", "fidelity"),
+                ndirections=_opt("NDirections", 1), ncorr=_opt("NCorr", 2),
+                nrows=_opt("NRows"), nchan=_opt("NChan"), na=_opt("NA"))
+            if threaded:
+                print("the memory model's own summary, at ONE worker (its NCPU line is "
+                      "not the plan -- see 'plan:' below):")
+            print(suggestion)
+
+            NCPU = cores if threaded else int(suggestion.best["ncpu"])
+            ChunkHours = float(suggestion.best["chunk_hours"])
+            ChanStep = int(suggestion.best["chan_step"])
+            peak = float(suggestion.best["peak_gb"])
+            PeakGB = peak if math.isfinite(peak) else 0.0
+            host = float(suggestion.best.get("host_ram_gb", peak))
+            HostRAMGB = host if math.isfinite(host) else 0.0
+            MemWatermarkGB = float(max(1, round(0.15 * ram_gb)))
+            RamGBUsed = float(ram_gb)
+            Flags = " ".join(
+                (["--Selection-ChanStep %d" % ChanStep] if ChanStep != 1 else [])
+                + ["--Parallel-NCPU %d" % NCPU, "--Data-ChunkHours %g" % ChunkHours,
+                   "--Parallel-MemWatermarkGB %g" % MemWatermarkGB])
+            print("plan: %s" % Flags)
+            Plan = _opt("PlanOut", "ddfacet-memory-plan.json")
+            with open(Plan, "w") as _fh:
+                json.dump(dict(ncpu=NCPU, chunk_hours=ChunkHours, chan_step=ChanStep,
+                               peak_gb=PeakGB, host_ram_gb=HostRAMGB,
+                               mem_watermark_gb=MemWatermarkGB, ram_gb=RamGBUsed,
+                               ram_source=ram_source, threaded=threaded,
+                               flags=Flags, notes=suggestion.notes,
+                               feasible=[{k: v for k, v in f.items() if k != "estimate"}
+                                         for f in suggestion.feasible]),
+                          _fh, indent=2)
+            if not suggestion.feasible:
+                raise RuntimeError(
+                    "ddfacet-memory-plan: nothing in the search space fits %.1f GB "
+                    "-- see the plan at %s" % (ram_gb, Plan))