dasksf 0.2.0__tar.gz → 1.2.3__tar.gz

{dasksf-0.2.0 → dasksf-1.2.3}/PKG-INFO

@@ -1,18 +1,17 @@
 Metadata-Version: 2.3
 Name: dasksf
-Version: 0.2.0
+Version: 1.2.3
 Summary: Radio astronomy source finding using dask and plugging in other source finders
 License: BSD-3-Clause
 Author: vla22
 Author-email: vla22@cam.ac.uk
-Requires-Python: >=3.12,<4.0
+Requires-Python: >=3.14,<4.0
 Classifier: License :: OSI Approved :: BSD License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: dask[complete] (>=2025.2.0,<2026.0.0)
 Requires-Dist: distributed (>=2025.2.0,<2026.0.0)
-Requires-Dist: fits2tmp (>=0.3.0,<0.4.0)
+Requires-Dist: fits2tmp (>=1.0.2,<2.0.0)
+Requires-Dist: numpy (==2.1.2)
 Description-Content-Type: text/markdown
 
 # dasksf

{dasksf-0.2.0 → dasksf-1.2.3}/pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "dasksf"
-version = "0.2.0"
+version = "1.2.3"
 description = "Radio astronomy source finding using dask and plugging in other source finders"
 authors = [
     { name ="vla22", email= "vla22@cam.ac.uk"},
@@ -11,10 +11,11 @@ readme = "README.md"
 repository = "https://gitlab.developers.cam.ac.uk/phy/ra/vla22-phd/dasksf"
 
 [tool.poetry.dependencies]
-python = "^3.12"
+python = "^3.14"
 dask = {extras = ["complete"], version = "^2025.2.0"}
 distributed = "^2025.2.0"
-fits2tmp = "^0.3.0"
+numpy = "=2.1.2"
+fits2tmp = "^1.0.2"
 
 [project.scripts]
 def_cluster = "dasksf.def_cluster:main"

{dasksf-0.2.0 → dasksf-1.2.3}/src/dasksf/mypipe.py

@@ -7,6 +7,7 @@ from dask.distributed import performance_report, get_task_stream
 from dask import delayed
 from dask.diagnostics import Profiler, ResourceProfiler, CacheProfiler, visualize
 from datetime import timedelta
+import time
 import tempfile
 import shutil
 from functools import reduce
@@ -30,7 +31,6 @@ class Pipeline:
         self.tmpdir = tmpdir
         self.sofia_params = sofia_params
         self.im_shape = [int(x) for x in im_shape.split(",")]
-        print(self.im_shape)
         self.overlap = [int(x) for x in overlap.split(",")]
         self.im_as_slice = []
         for i in range(len(self.im_shape)):
@@ -47,6 +47,9 @@ class Pipeline:
         else:
             self.chunks = self.chunk_info
             self.ch_pix = "auto"
+        self.overlap_pix = 0
+        for i in range(int(self.n_dims)):
+            self.overlap_pix = self.overlap[i] + self.overlap_pix
         if "fits" in self.data:
             self.filetype = "fits"
         elif "zarr" in self.data:
@@ -67,45 +70,61 @@ class Pipeline:
         self.outputlog.addHandler(output_handler)
     
     def run_sof(self, filename):
-        subprocess.run(["sofia", self.sofia_params, "input.data="+filename, "output.directory="+self.tmpdir])
+        try:
+            a = subprocess.run(["sofia", self.sofia_params, "input.data="+filename, "output.directory="+self.tmpdir])
+            subprocess.run(["rm", "-r", filename])
+            return a.returncode
+        except Exception as e:
+            self.mainlog.error(e)
+            return
 
     def dedupe(self, file1, file2):
         if "_cat.xml" not in file1:
             file1 = file1 + "_cat.xml"
         if "_cat.xml" not in file2:
             file2 = file2 + "_cat.xml"
+        tempfile.tempdir = self.tmpdir
         f = tempfile.NamedTemporaryFile(delete=False)
-        subprocess.run(["stilts", "tskymatch2", "in1="+file1, "in2="+file2, "join=1or2", "out="+f.name + "_cat.xml", "error=1"])
-        return f.name
-        
+        try: 
+            subprocess.run(["stilts", "tskymatch2", "in1="+file1, "in2="+file2, "join=1or2", "out="+f.name + "_cat.xml", "error=1"])
+            subprocess.run(["rm", file1])
+            subprocess.run(["rm", file2])
+            return f.name + "_cat.xml"
+        except Exception as e:
+            self.mainlog.error(e, "input_files=", file1, file2)
+            with open(file1, 'rb') as tf, open(self.outputdir + file1, 'wb') as destfil:
+                shutil.copyfileobj(tf, destfil)
+            with open(file2, 'rb') as tf, open(self.outputdir + file2, 'wb') as destfil:
+                shutil.copyfileobj(tf, destfil)
+    
     def run_sofia_pipeline(self):
-        print(self.im_as_slice)
         client = self.client
         start_time = time.monotonic()
-        #print(type(self.data), type(self.tmpdir), self.chunks)
-        with performance_report(filename=self.outputdir + "perf_report.html"), Profiler() as prof, ResourceProfiler(dt=0.05) as rprof, CacheProfiler() as cprof, get_task_stream(plot="save", filename=(self.outputdir + "task_stream.html")) as ts:
-            #filelist = f2t.fits2tmp.fakefits2tmpfiles(self.filetype, self.data, self.tmpdir, tuple(self.im_as_slice), self.header_loc, self.chunks, self.overlap)
-            filelist = f2t.fits2tmp.fakefits2tmpfiles(self.filetype, self.data, self.tmpdir, self.im_shape, self.header_loc, self.chunks, self.overlap)
-            print(filelist)
+        # performance_report(filename=self.outputdir + "perf_report.html"), 
+        with Profiler() as prof, ResourceProfiler(dt=0.05) as rprof, CacheProfiler() as cprof, get_task_stream(plot="save", filename=(self.outputdir + "task_stream.html")) as ts:
+            filelist = []
+            f2t.fits2tmp.fakefits2tmpfiles(self.filetype, self.data, self.tmpdir, self.im_shape, self.header_loc, self.chunks, self.overlap, fpaths=filelist)
             scatter_files = client.scatter(filelist)
             load_time = time.monotonic() 
             outlist = []
             for i in filelist:
-                a = self.run_sof(i)
-                outlist.append(i[i.index('/'):])
+                try:
+                    a = self.run_sof(i)
+                    if a == 0:
+                        outlist.append(i[i.index('/'):])
+                except Exception as e:
+                    self.mainlog.error(e)
             calc_done = time.monotonic()
             output = client.gather(outlist)
-            print(outlist)
-            fin_cat = reduce(self.dedupe, output)
+            fincat = reduce(self.dedupe, output)
             end_time = time.monotonic()
-            print(fin_cat)
-            with open(fin_cat + "_cat.xml", 'rb') as tf, open(self.outputdir + "/combined_catalogue.xml", 'wb') as destfil:
+            with open(fincat, 'rb') as tf, open(self.outputdir + "/combined_catalogue"+ str(time.time_ns()) +".xml", 'wb') as destfil:
                 shutil.copyfileobj(tf, destfil)
             self.outputlog.info(
-                ", %s, %s,%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s", 
+                ", %s, %s,%s, %s, %s, %s, %s, %s, %s,%s, %s, %s, %s, %s, %s", 
                 #%s, %s, %s",
                 str(self.platform),
-                ##str(self.dask_version),
+                #str(self.dask_version),
                 str(self.workflow),
                 str(self.nodes),
                 str(self.n_workers),
@@ -114,6 +133,7 @@ class Pipeline:
                 self.n_dims,
                 self.im_pix,
                 self.ch_pix,
+                self.overlap_pix,
                 self.filetype,
                 str(timedelta(seconds = calc_done - start_time)),
                 str(timedelta(seconds = load_time - start_time)),