graphviper 0.0.1__tar.gz

graphviper-0.0.1/LICENSE.txt

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, Associated Universities, Inc.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

graphviper-0.0.1/MANIFEST.in

@@ -0,0 +1,4 @@
+include README.md
+include MANIFEST.in
+include LICENSE
+include CONTRIBUTOR_LICENSING_AGREEMENT

graphviper-0.0.1/PKG-INFO

@@ -0,0 +1,78 @@
+Metadata-Version: 2.1
+Name: graphviper
+Version: 0.0.1
+Summary: Astro Visibility and Image Parallel Execution Reduction
+Author-email: Jan-Willem Steeb <jsteeb@nrao.edu>
+License: BSD 3-Clause License
+        
+        Copyright (c) 2023, Associated Universities, Inc.
+        
+        Redistribution and use in source and binary forms, with or without
+        modification, are permitted provided that the following conditions are met:
+        
+        1. Redistributions of source code must retain the above copyright notice, this
+           list of conditions and the following disclaimer.
+        
+        2. Redistributions in binary form must reproduce the above copyright notice,
+           this list of conditions and the following disclaimer in the documentation
+           and/or other materials provided with the distribution.
+        
+        3. Neither the name of the copyright holder nor the names of its
+           contributors may be used to endorse or promote products derived from
+           this software without specific prior written permission.
+        
+        THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+        AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+        IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+        DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+        FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+        DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+        SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+        CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+        OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+        OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+        
+Requires-Python: <3.12,>=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: astropy
+Requires-Dist: dask
+Requires-Dist: distributed
+Requires-Dist: gdown
+Requires-Dist: matplotlib
+Requires-Dist: numba>=0.57.0
+Requires-Dist: numpy<=1.25.0
+Requires-Dist: prettytable
+Requires-Dist: pytest
+Requires-Dist: pytest-cov
+Requires-Dist: pytest-html
+Requires-Dist: scipy
+Requires-Dist: xarray
+Requires-Dist: zarr
+Requires-Dist: bokeh
+Requires-Dist: jupyterlab
+Requires-Dist: pyarrow
+Requires-Dist: graphviz
+Requires-Dist: xradio
+Requires-Dist: python_casacore>=3.5.2; sys_platform != "darwin"
+Provides-Extra: docs
+Requires-Dist: ipykernel; extra == "docs"
+Requires-Dist: ipympl; extra == "docs"
+Requires-Dist: ipython; extra == "docs"
+Requires-Dist: jupyter-client; extra == "docs"
+Requires-Dist: nbsphinx; extra == "docs"
+Requires-Dist: recommonmark; extra == "docs"
+Requires-Dist: scanpydoc; extra == "docs"
+Requires-Dist: sphinx-autoapi; extra == "docs"
+Requires-Dist: sphinx-autosummary-accessors; extra == "docs"
+Requires-Dist: sphinx_rtd_theme; extra == "docs"
+Requires-Dist: twine; extra == "docs"
+Requires-Dist: pandoc; extra == "docs"
+
+# GraphVIPER
+
+[![Python 3.8 3.9 3.10 3.11](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11-blue)](https://www.python.org/downloads/release/python-380/)
+
+### 
+
+

graphviper-0.0.1/README.md

@@ -0,0 +1,7 @@
+# GraphVIPER
+
+[![Python 3.8 3.9 3.10 3.11](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11-blue)](https://www.python.org/downloads/release/python-380/)
+
+### 
+
+

graphviper-0.0.1/pyproject.toml

@@ -0,0 +1,50 @@
+[project]
+name = "graphviper"
+version = "0.0.1"
+description = "Astro Visibility and Image Parallel Execution Reduction"
+authors = [
+    {name = "Jan-Willem Steeb", email="jsteeb@nrao.edu"},
+]
+license = {file = "LICENSE.txt"}
+readme = "README.md"
+requires-python = ">= 3.8, < 3.12"
+
+dependencies = [
+'astropy',
+'dask',
+'distributed',
+'gdown',
+'matplotlib',
+'numba>=0.57.0',
+'numpy<=1.25.0',
+'prettytable',
+'pytest',
+'pytest-cov',
+'pytest-html',
+'scipy',
+'xarray',
+'zarr',
+'bokeh',
+'jupyterlab',
+'pyarrow',
+'graphviz',
+'xradio',
+'python_casacore>=3.5.2; sys_platform != "darwin" '
+]
+
+
+[project.optional-dependencies]
+docs = [
+    'ipykernel',
+    'ipympl',
+    'ipython',
+    'jupyter-client',
+    'nbsphinx',
+    'recommonmark',
+    'scanpydoc',
+    'sphinx-autoapi',
+    'sphinx-autosummary-accessors',
+    'sphinx_rtd_theme',
+    'twine',
+    'pandoc'
+]

graphviper-0.0.1/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

graphviper-0.0.1/src/graphviper/dask/__init__.py

@@ -0,0 +1 @@
+from .client import local_client, slurm_cluster_client

graphviper-0.0.1/src/graphviper/dask/_scheduler.py

@@ -0,0 +1,208 @@
+"""
+MIT License
+
+Copyright (c) 2022 Jonathan Simon Kenyon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+from collections import defaultdict
+from distributed import SchedulerPlugin
+from dask.core import reverse_dict
+from dask.base import tokenize
+from dask.order import graph_metrics, ndependencies
+import click
+from distributed.diagnostics.plugin import SchedulerPlugin
+import numpy as np
+
+
+def unravel_deps(hlg_deps, name, unravelled_deps=None):
+    """Recursively construct a set of all dependencies for a specific task."""
+
+    if unravelled_deps is None:
+        unravelled_deps = set()
+
+    for dep in hlg_deps[name]:
+        unravelled_deps |= {dep}
+        unravel_deps(hlg_deps, dep, unravelled_deps)
+
+    return unravelled_deps
+
+
+def get_node_depths(dependencies, root_nodes, metrics):
+    node_depths = {}
+
+    for k in dependencies.keys():
+        # Get dependencies per node.
+        deps = unravel_deps(dependencies, k)
+        # Associate nodes with root nodes.
+        roots = root_nodes & deps
+        offset = metrics[k][-1]
+        node_depths[k] = max(metrics[r][-1] - offset for r in roots) if roots else 0
+
+    return node_depths
+
+
+class schedular(SchedulerPlugin):
+    def __init__(self, autorestrictor, local_cache):
+        self.autorestrictor = autorestrictor
+        self.local_cache = local_cache
+        super().__init__()
+
+    def add_worker(self, scheduler, worker):
+        if self.local_cache:
+            # Set the resource label to the ip of the node that the worker is on, so that tasks that require a specific node can be assigned to the correct worker.
+            ip = worker[worker.rfind("/") + 1 : worker.rfind(":")]
+            scheduler.add_resources(worker=worker, resources={ip: 1})
+
+    def update_graph(self, scheduler, dsk=None, keys=None, restrictions=None, **kw):
+        if self.autorestrictor:
+            print("Using autorestrictor")
+            """Processes dependencies to assign tasks to specific workers."""
+            workers = list(scheduler.workers.keys())
+            n_worker = len(workers)
+
+            tasks = scheduler.tasks
+            dependencies = kw["dependencies"]
+
+            # print('In update_graph :', scheduler, ',*,', dsk, ',*,', keys , ',*,', restrictions , ',*,', kw)
+            if dependencies:
+                dependents = reverse_dict(dependencies)
+
+                # print('reversed dict:', dependents)
+
+                _, total_dependencies = ndependencies(dependencies, dependents)
+                # TODO: Avoid calling graph metrics.
+                metrics = graph_metrics(dependencies, dependents, total_dependencies)
+
+                # Terminal nodes have no dependents, root nodes have no dependencies.
+                # Horizontal partition nodes are initialized as the terminal nodes.
+                part_nodes = {k for (k, v) in dependents.items() if not v}
+                root_nodes = {k for (k, v) in dependencies.items() if not v}
+
+                # Figure out the depth of every task. Depth is defined as maximum
+                # distance from a root node. TODO: Optimize get_node_depths.
+
+                node_depths = get_node_depths(dependencies, root_nodes, metrics)
+                # try:
+                max_depth = max(node_depths.values())
+                # except:
+                #        print('&&&&& dependencies, root_nodes, metrics',node_depths,',*,',dependencies, root_nodes, metrics)
+
+                # If we have fewer partition nodes than workers, we cannot utilise all
+                # the workers and are likely dealing with a reduction. We work our way
+                # back through the graph, starting at the deepest terminal nodes, and
+                # try to find a depth at which there was enough work to utilise all
+                # workers.
+                while (len(part_nodes) < n_worker) & (max_depth > 0):
+                    _part_nodes = part_nodes.copy()
+                    for pn in _part_nodes:
+                        if node_depths[pn] == max_depth:
+                            part_nodes ^= set((pn,))
+                            part_nodes |= dependencies[pn]
+                    max_depth -= 1
+                    if max_depth <= 0:
+                        return  # In this case, there in nothing we can do - fall back.
+
+                part_roots = {}
+                part_dependencies = {}
+                part_dependents = {}
+
+                for pn in part_nodes:
+                    # Get dependencies per partition node.
+                    part_dependencies[pn] = unravel_deps(dependencies, pn)
+                    # Get dependents per partition node.
+                    part_dependents[pn] = unravel_deps(dependents, pn)
+                    # Associate partition nodes with root nodes.
+                    part_roots[pn] = root_nodes & part_dependencies[pn]
+
+                # Create a unique token for each set of partition roots. TODO: This is
+                # very strict. What about nodes with very similar roots? Tokenization
+                # may be overkill too.
+                root_tokens = {tokenize(*sorted(v)): v for v in part_roots.values()}
+
+                hash_map = defaultdict(set)
+                group_offset = 0
+
+                # Associate partition roots with a specific group if they are not a
+                # subset of another, larger root set.
+                for k, v in root_tokens.items():
+                    if any(v < vv for vv in root_tokens.values()):  # Strict subset.
+                        continue
+                    else:
+                        hash_map[k] |= set([group_offset])
+                        group_offset += 1
+
+                # If roots were a subset, they should share the group of their
+                # superset/s.
+                for k, v in root_tokens.items():
+                    if not v:  # Special case - no dependencies. Handled below.
+                        continue
+                    shared_roots = {
+                        kk: None for kk, vv in root_tokens.items() if v < vv
+                    }
+                    if shared_roots:
+                        hash_map[k] = set().union(
+                            *[hash_map[kk] for kk in shared_roots.keys()]
+                        )
+
+                task_groups = defaultdict(set)
+
+                for pn in part_nodes:
+                    pdp = part_dependencies[pn]
+                    pdn = part_dependents[pn]
+
+                    if pdp:
+                        groups = hash_map[tokenize(*sorted(part_roots[pn]))]
+                    else:  # Special case - no dependencies.
+                        groups = {group_offset}
+                        group_offset += 1
+
+                    for g in groups:
+                        task_groups[g] |= pdp | pdn | {pn}
+
+                worker_loads = {wkr: 0 for wkr in workers}
+
+                for task_group in task_groups.values():
+                    assignee = min(worker_loads, key=worker_loads.get)
+                    worker_loads[assignee] += len(task_group)
+
+                    for task_name in task_group:
+                        try:
+                            task = tasks[task_name]
+                        except KeyError:  # Keys may not have an assosciated task.
+                            continue
+
+                        # print('^^^^^^',dir(task))
+                        # if task._worker_restrictions is None:
+                        #    task._worker_restrictions = set()
+                        # task._worker_restrictions |= {assignee}
+                        # task._loose_restrictions = False
+
+                        if task.worker_restrictions is None:
+                            task.worker_restrictions = set()
+                        task.worker_restrictions |= {assignee}
+                        task.loose_restrictions = False
+
+
+@click.command()
+@click.option("--autorestrictor", default=False)
+@click.option("--local_cache", default=False)
+def dask_setup(scheduler, autorestrictor, local_cache):
+    plugin = schedular(autorestrictor, local_cache)
+    scheduler.add_plugin(plugin)

graphviper-0.0.1/src/graphviper/dask/_worker.py

@@ -0,0 +1,71 @@
+import click
+
+from graphviper._logger import _setup_worker_logger
+
+
+class _worker:
+    def __init__(self, local_cache, log_parms):
+        # print("init local cache")
+        self.local_cache = local_cache
+
+        print("log_parms", log_parms)
+        # /.lustre/aoc/projects/ngvla/viper/ngvla_sim/viper_
+        self.log_to_term = log_parms["log_to_term"]
+        self.log_to_file = log_parms["log_to_file"]
+        self.log_file = log_parms["log_file"]
+        self.log_level = log_parms["log_level"]
+
+    def get_logger(self):
+        return self.logger
+
+    def setup(self, worker):
+        """
+        Run when the plugin is attached to a worker. This happens when the plugin is registered
+        and attached to existing workers, or when a worker is created after the plugin has been
+        registered.
+        """
+
+        self.logger = _setup_worker_logger(
+            self.log_to_term,
+            self.log_to_file,
+            self.log_file,
+            self.log_level,
+            str(worker.id),
+        )
+        self.logger.debug(
+            "Logger created on worker " + str(worker.id) + ",*," + str(worker.address)
+        )
+        # Documentation https://distributed.dask.org/en/stable/worker.html#distributed.worker.Worker
+        self.worker = worker
+
+        if self.local_cache:
+            ip = worker.address[
+                worker.address.rfind("/") + 1 : worker.address.rfind(":")
+            ]
+            self.logger.debug(str(worker.id) + ",*," + ip)
+            worker.state.available_resources = {
+                **worker.state.available_resources,
+                **{ip: 1},
+            }
+            # print(worker.state.available_resources)
+
+
+# https://github.com/dask/distributed/issues/4169
+@click.command()
+@click.option("--local_cache", default=False)
+# @click.option("--log_parms", default={'log_to_term':True,'log_to_file':False,'log_file':'viper_', 'log_level':'DEBUG'})
+@click.option("--log_to_term", default=True)
+@click.option("--log_to_file", default=False)
+@click.option("--log_file", default="viper_")
+@click.option("--log_level", default="INFO")
+async def dask_setup(
+    worker, local_cache, log_to_term, log_to_file, log_file, log_level
+):
+    log_parms = {
+        "log_to_term": log_to_term,
+        "log_to_file": log_to_file,
+        "log_file": log_file,
+        "log_level": log_level,
+    }
+    plugin = _worker(local_cache, log_parms)
+    await worker.client.register_worker_plugin(plugin, name="viper_worker")