anuga 1.3.5__zip

Users/steve/Miniconda/envs/anuga/Lib/site-packages/anuga/parallel/parallel_advection.py

@@ -0,0 +1,205 @@
+import sys
+
+
+"""Class Parallel_domain -
+2D triangular domains for finite-volume computations of
+the advection equation, with extra structures to allow
+communication between other Parallel_domains and itself
+
+This module contains a specialisation of class Domain from module advection.py
+
+Ole Nielsen, Stephen Roberts, Duncan Gray, Christopher Zoppou
+Geoscience Australia, 2004-2005
+"""
+
+from anuga.advection import *
+
+from anuga import Domain
+
+import numpy as num
+
+import pypar
+
+
+class Parallel_domain(Domain):
+
+    def __init__(self,
+                 coordinates,
+                 vertices,
+                 boundary = None,
+                 full_send_dict = None,
+                 ghost_recv_dict = None,
+                 velocity = None):
+
+        Domain.__init__(self,
+                        coordinates,
+                        vertices,
+                        boundary,
+                        velocity = velocity,
+                        full_send_dict=full_send_dict,
+                        ghost_recv_dict=ghost_recv_dict,
+                        processor=pypar.rank(),
+                        numproc=pypar.size()
+                        )
+
+        N = self.number_of_elements
+
+
+        self.communication_time = 0.0
+        self.communication_reduce_time = 0.0
+
+
+        print 'processor',self.processor
+        print 'numproc',self.numproc
+
+    def check_integrity(self):
+        Domain.check_integrity(self)
+
+        msg = 'Will need to check global and local numbering'
+        assert self.conserved_quantities[0] == 'stage', msg
+
+    def update_timestep(self, yieldstep, finaltime):
+
+        #LINDA:
+        # moved the calculation so that it is done after timestep
+        # has been broadcast
+        
+#        # Calculate local timestep
+#        Domain.update_timestep(self, yieldstep, finaltime)
+
+        import time
+        t0 = time.time()
+
+        # For some reason it looks like pypar only reduces numeric arrays
+        # hence we need to create some dummy arrays for communication
+        ltimestep = num.ones( 1, num.float )
+        ltimestep[0] = self.flux_timestep
+        gtimestep = num.zeros( 1, num.float ) # Buffer for results
+
+        #ltimestep = self.flux_timeste
+
+        #print self.processor, ltimestep, gtimestep
+        
+        gtimestep = pypar.reduce(ltimestep, pypar.MIN, 0, buffer=gtimestep)
+
+        #print self.processor, ltimestep, gtimestep
+        
+        pypar.broadcast(gtimestep,0)
+
+        #print self.processor, ltimestep, gtimestep
+
+        self.flux_timestep = gtimestep[0]
+        
+        self.communication_reduce_time += time.time()-t0
+
+        # LINDA:
+        # Now update time stats
+        
+        # Calculate local timestep
+        Domain.update_timestep(self, yieldstep, finaltime)
+
+    def update_ghosts(self):
+
+        # We must send the information from the full cells and
+        # receive the information for the ghost cells
+        # We have a dictionary of lists with ghosts expecting updates from
+        # the separate processors
+
+        #from Numeric import take,put
+        import numpy as num
+        import time
+        t0 = time.time()
+
+        stage_cv = self.quantities['stage'].centroid_values
+
+        # update of non-local ghost cells
+        for iproc in range(self.numproc):
+            if iproc == self.processor:
+                #Send data from iproc processor to other processors
+                for send_proc in self.full_send_dict:
+                    if send_proc != iproc:
+
+                        Idf  = self.full_send_dict[send_proc][0]
+                        Xout = self.full_send_dict[send_proc][2]
+
+                        N = len(Idf)
+
+                        #for i in range(N):
+                        #    Xout[i,0] = stage_cv[Idf[i]]
+                        Xout[:,0] = num.take(stage_cv, Idf)
+
+                        pypar.send(Xout,send_proc)
+
+
+            else:
+                #Receive data from the iproc processor
+                if  self.ghost_recv_dict.has_key(iproc):
+
+                    # LINDA:
+                    # now store ghost as local id, global id, value
+                    Idg = self.ghost_recv_dict[iproc][0]
+                    X = self.ghost_recv_dict[iproc][2]
+
+                    X = pypar.receive(iproc,X)
+                    N = len(Idg)
+
+                    num.put(stage_cv, Idg, X[:,0])
+                    #for i in range(N):
+                    #    stage_cv[Idg[i]] = X[i,0]
+
+
+        #local update of ghost cells
+        iproc = self.processor
+        if self.full_send_dict.has_key(iproc):
+
+            # LINDA:
+            # now store full as local id, global id, value
+            Idf  = self.full_send_dict[iproc][0]
+
+            # LINDA:
+            # now store ghost as local id, global id, value
+            Idg = self.ghost_recv_dict[iproc][0]
+
+            N = len(Idg)
+
+            #for i in range(N):
+            #    #print i,Idg[i],Idf[i]
+            #    stage_cv[Idg[i]] = stage_cv[Idf[i]]
+
+            num.put(stage_cv, Idg, num.take(stage_cv, Idf))
+
+
+        self.communication_time += time.time()-t0
+
+
+    ## def write_time(self):
+    ##     if self.min_timestep == self.max_timestep:
+    ##         print 'Processor %d, Time = %.4f, delta t = %.8f, steps=%d (%d)'\
+    ##               %(self.processor, self.time, self.min_timestep, self.number_of_steps,
+    ##                 self.number_of_first_order_steps)
+    ##     elif self.min_timestep > self.max_timestep:
+    ##         print 'Processor %d, Time = %.4f, steps=%d (%d)'\
+    ##               %(self.processor, self.time, self.number_of_steps,
+    ##                 self.number_of_first_order_steps)
+    ##     else:
+    ##         print 'Processor %d, Time = %.4f, delta t in [%.8f, %.8f], steps=%d (%d)'\
+    ##               %(self.processor, self.time, self.min_timestep,
+    ##                 self.max_timestep, self.number_of_steps,
+    ##                 self.number_of_first_order_steps)
+
+
+
+    ## def evolve(self, yieldstep = None, finaltime = None):
+    ##     """Specialisation of basic evolve method from parent class
+    ##     """
+
+    ##     #Initialise real time viz if requested
+    ##     if self.time == 0.0:
+    ##         pass
+
+    ##     #Call basic machinery from parent class
+    ##     for t in Domain.evolve(self, yieldstep, finaltime):
+
+    ##         #Pass control on to outer loop for more specific actions
+    ##         yield(t)
+

Users/steve/Miniconda/envs/anuga/Lib/site-packages/anuga/parallel/parallel_api.py

@@ -0,0 +1,467 @@
+"""Trying to lump parallel stuff into simpler interface
+
+
+"""
+
+import numpy as num
+
+# The abstract Python-MPI interface
+from anuga.utilities.parallel_abstraction import size, rank, get_processor_name
+from anuga.utilities.parallel_abstraction import finalize, send, receive
+from anuga.utilities.parallel_abstraction import pypar_available, barrier
+
+
+
+# ANUGA parallel engine (only load if pypar can)
+if pypar_available:
+    from anuga.parallel.sequential_distribute import sequential_distribute_dump
+    from anuga.parallel.sequential_distribute import sequential_distribute_load
+    
+    from anuga.parallel.distribute_mesh  import send_submesh
+    from anuga.parallel.distribute_mesh  import rec_submesh
+    from anuga.parallel.distribute_mesh  import extract_submesh
+
+    # Mesh partitioning using Metis
+    from anuga.parallel.distribute_mesh import build_submesh
+    from anuga.parallel.distribute_mesh import pmesh_divide_metis_with_map
+
+    from anuga.parallel.parallel_shallow_water import Parallel_domain
+    
+    
+
+from anuga.abstract_2d_finite_volumes.neighbour_mesh import Mesh
+
+#------------------------------------------------------------------------------
+# Read in processor information
+#------------------------------------------------------------------------------
+
+numprocs = size()
+myid = rank()
+processor_name = get_processor_name()
+#print 'I am processor %d of %d on node %s' %(myid, numprocs, processor_name)
+
+
+
+def distribute(domain, verbose=False, debug=False, parameters = None):
+    """ Distribute the domain to all processes
+
+    parameters allows user to change size of ghost layer
+    """
+
+    if not pypar_available or numprocs == 1 : return domain # Bypass
+
+        
+    if myid == 0:
+        from sequential_distribute import Sequential_distribute
+        partition = Sequential_distribute(domain, verbose, debug, parameters)
+
+        partition.distribute(numprocs)
+
+        kwargs, points, vertices, boundary, quantities, boundary_map, \
+                domain_name, domain_dir, domain_store, domain_store_centroids, \
+                domain_minimum_storable_height, domain_minimum_allowed_height, \
+                domain_flow_algorithm, georef = partition.extract_submesh(0)
+        
+        for p in range(1, numprocs):
+
+            tostore = partition.extract_submesh(p)
+
+            send(tostore,p)
+
+    else:
+
+        kwargs, points, vertices, boundary, quantities, boundary_map, \
+                       domain_name, domain_dir, domain_store, domain_store_centroids, \
+                       domain_minimum_storable_height, domain_minimum_allowed_height, \
+                       domain_flow_algorithm, georef = receive(0)
+
+    #---------------------------------------------------------------------------
+    # Now Create parallel domain
+    #---------------------------------------------------------------------------
+    parallel_domain = Parallel_domain(points, vertices, boundary, **kwargs)
+
+
+    #------------------------------------------------------------------------
+    # Copy in quantity data
+    #------------------------------------------------------------------------
+    for q in quantities:
+        try:
+            parallel_domain.set_quantity(q, quantities[q])
+        except KeyError:
+            #print 'Try to create quantity %s'% q
+            from anuga import Quantity
+            Q = Quantity(parallel_domain, name=q, register=True)
+            parallel_domain.set_quantity(q, quantities[q])
+
+    #------------------------------------------------------------------------
+    # Transfer boundary conditions to each subdomain
+    #------------------------------------------------------------------------
+    boundary_map['ghost'] = None  # Add binding to ghost boundary
+    parallel_domain.set_boundary(boundary_map)
+
+
+    #------------------------------------------------------------------------
+    # Transfer other attributes to each subdomain
+    #------------------------------------------------------------------------
+    parallel_domain.set_name(domain_name)
+    parallel_domain.set_datadir(domain_dir)
+    parallel_domain.set_store(domain_store)
+    parallel_domain.set_store_centroids(domain_store_centroids)
+    parallel_domain.set_minimum_storable_height(domain_minimum_storable_height)
+    parallel_domain.set_minimum_allowed_height(domain_minimum_allowed_height)
+    parallel_domain.set_flow_algorithm(domain_flow_algorithm)
+    parallel_domain.geo_reference = georef
+
+
+    return parallel_domain
+
+
+        
+        
+
+
+
+def old_distribute(domain, verbose=False, debug=False, parameters = None):
+    """ Distribute the domain to all processes
+
+    parameters values  
+    """
+
+
+    if debug:
+        verbose = True
+        
+    barrier()
+
+    # FIXME: Dummy assignment (until boundaries are refactored to
+    # be independent of domains until they are applied)
+    if myid == 0:
+        bdmap = {}
+        for tag in domain.get_boundary_tags():
+            bdmap[tag] = None
+    
+    
+        domain.set_boundary(bdmap)
+
+
+    if not pypar_available or numprocs == 1 : return domain # Bypass
+
+    # For some obscure reason this communication must happen prior to
+    # the more complex mesh distribution - Oh Well!
+    if myid == 0:
+        domain_name = domain.get_name()
+        domain_dir = domain.get_datadir()
+        domain_store = domain.get_store()
+        domain_store_centroids = domain.get_store_centroids()
+        domain_smooth = domain.smooth
+        domain_reduction = domain.reduction
+        domain_minimum_storable_height = domain.minimum_storable_height
+        domain_flow_algorithm = domain.get_flow_algorithm()
+        domain_minimum_allowed_height = domain.get_minimum_allowed_height()
+        georef = domain.geo_reference
+        number_of_global_triangles = domain.number_of_triangles
+        number_of_global_nodes = domain.number_of_nodes
+        
+        # FIXME - what other attributes need to be transferred?
+
+        for p in xrange(1, numprocs):
+            # FIXME SR: Creates cPickle dump
+            send((domain_name, domain_dir, domain_store, \
+                  domain_store_centroids, domain_smooth, domain_reduction, \
+                  domain_minimum_storable_height, domain_flow_algorithm, \
+                  domain_minimum_allowed_height, georef, \
+                  number_of_global_triangles, number_of_global_nodes), p)
+    else:
+        if verbose: print 'P%d: Receiving domain attributes' %(myid)
+
+        domain_name, domain_dir, domain_store, \
+                  domain_store_centroids, domain_smooth, domain_reduction, \
+                  domain_minimum_storable_height, domain_flow_algorithm, \
+                  domain_minimum_allowed_height, georef, \
+                  number_of_global_triangles, \
+                  number_of_global_nodes = receive(0)
+
+
+
+    # Distribute boundary conditions
+    # FIXME: This cannot handle e.g. Time_boundaries due to
+    # difficulties pickling functions
+    if myid == 0:
+        boundary_map = domain.boundary_map
+        for p in xrange(1, numprocs):
+            # FIXME SR: Creates cPickle dump
+            send(boundary_map, p)
+    else:
+        if verbose: print 'P%d: Receiving boundary map' %(myid)        
+
+        boundary_map = receive(0)
+        
+
+    send_s2p = False
+
+    if myid == 0:
+        # Partition and distribute mesh.
+        # Structures returned is in the
+        # correct form for the ANUGA data structure
+
+
+        points, vertices, boundary, quantities,\
+                ghost_recv_dict, full_send_dict,\
+                number_of_full_nodes, number_of_full_triangles,\
+                s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
+                ghost_layer_width =\
+                distribute_mesh(domain, verbose=verbose, debug=debug, parameters=parameters)
+            
+        # Extract l2g maps
+        #tri_l2g  = extract_l2g_map(tri_map)
+        #node_l2g = extract_l2g_map(node_map)
+        #tri_l2g = p2s_map[tri_l2g]
+        
+        if debug:
+            print 'P%d' %myid
+            print 'tri_map ',tri_map
+            print 'node_map',node_map
+            print 'tri_l2g', tri_l2g
+            print 'node_l2g', node_l2g
+            print 's2p_map', s2p_map
+            print 'p2s_map', p2s_map
+
+
+        def protocol(x):
+            vanilla=False
+            import pypar
+            control_info, x = pypar.create_control_info(x, vanilla, return_object=True)
+            print 'protocol', control_info[0]
+            
+        # Send serial to parallel (s2p) and parallel to serial (p2s) triangle mapping to proc 1 .. numprocs
+
+
+        if send_s2p :
+            n = len(s2p_map)
+            s2p_map_keys_flat = num.reshape(num.array(s2p_map.keys(),num.int), (n,1) )
+            s2p_map_values_flat = num.array(s2p_map.values(),num.int)
+            s2p_map_flat = num.concatenate( (s2p_map_keys_flat, s2p_map_values_flat), axis=1 )
+
+            n = len(p2s_map)
+            p2s_map_keys_flat = num.reshape(num.array(p2s_map.keys(),num.int), (n,2) )
+            p2s_map_values_flat = num.reshape(num.array(p2s_map.values(),num.int) , (n,1))
+            p2s_map_flat = num.concatenate( (p2s_map_keys_flat, p2s_map_values_flat), axis=1 )
+
+            for p in range(1, numprocs):
+
+                # FIXME SR: Creates cPickle dump
+                send(s2p_map_flat, p)
+                # FIXME SR: Creates cPickle dump
+                #print p2s_map
+                send(p2s_map_flat, p)
+        else:
+            if verbose: print 'Not sending s2p_map and p2s_map'
+            s2p_map = None
+            p2s_map = None
+
+        if verbose: print 'Communication done'
+        
+    else:
+        # Read in the mesh partition that belongs to this
+        # processor
+        if verbose: print 'P%d: Receiving submeshes' %(myid)                
+        points, vertices, boundary, quantities,\
+                ghost_recv_dict, full_send_dict,\
+                number_of_full_nodes, number_of_full_triangles, \
+                tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
+                rec_submesh(0, verbose)
+
+
+
+        # Extract l2g maps
+        #tri_l2g  = extract_l2g_map(tri_map)
+        #node_l2g = extract_l2g_map(node_map)
+        #tri_l2g = p2s_map[tri_l2g]
+        
+        # Receive serial to parallel (s2p) and parallel to serial (p2s) triangle mapping
+        if send_s2p :
+            s2p_map_flat = receive(0)
+            s2p_map = dict.fromkeys(s2p_map_flat[:,0], s2p_map_flat[:,1:2])
+
+            p2s_map_flat = receive(0)
+            p2s_map_keys = [tuple(x) for x in p2s_map_flat[:,0:1]]
+
+            p2s_map = dict.fromkeys(p2s_map_keys, p2s_map_flat[:,2])
+        else:
+            s2p_map = None
+            p2s_map = None
+            
+    #------------------------------------------------------------------------
+    # Build the domain for this processor using partion structures
+    #------------------------------------------------------------------------
+
+    if verbose: print 'myid = %g, no_full_nodes = %g, no_full_triangles = %g' % (myid, number_of_full_nodes, number_of_full_triangles)
+
+    
+    domain = Parallel_domain(points, vertices, boundary,
+                             full_send_dict=full_send_dict,
+                             ghost_recv_dict=ghost_recv_dict,
+                             number_of_full_nodes=number_of_full_nodes,
+                             number_of_full_triangles=number_of_full_triangles,
+                             geo_reference=georef,
+                             number_of_global_triangles = number_of_global_triangles,
+                             number_of_global_nodes = number_of_global_nodes,
+                             s2p_map = s2p_map,
+                             p2s_map = p2s_map, ## jj added this
+                             tri_l2g = tri_l2g, ## SR added this
+                             node_l2g = node_l2g,
+                             ghost_layer_width = ghost_layer_width)
+
+    #------------------------------------------------------------------------
+    # Transfer initial conditions to each subdomain
+    #------------------------------------------------------------------------
+    for q in quantities:
+        domain.set_quantity(q, quantities[q]) 
+
+
+    #------------------------------------------------------------------------
+    # Transfer boundary conditions to each subdomain
+    #------------------------------------------------------------------------
+    boundary_map['ghost'] = None  # Add binding to ghost boundary
+    domain.set_boundary(boundary_map)
+
+
+    #------------------------------------------------------------------------
+    # Transfer other attributes to each subdomain
+    #------------------------------------------------------------------------
+    domain.set_name(domain_name)
+    domain.set_datadir(domain_dir)
+    domain.set_store(domain_store)
+    domain.set_store_centroids(domain_store_centroids)
+    domain.set_store_vertices_smoothly(domain_smooth,domain_reduction)
+    domain.set_minimum_storable_height(domain_minimum_storable_height)
+    domain.set_minimum_allowed_height(domain_minimum_allowed_height)
+    domain.set_flow_algorithm(domain_flow_algorithm)
+    domain.geo_reference = georef   
+
+    #------------------------------------------------------------------------
+    # Return parallel domain to all nodes
+    #------------------------------------------------------------------------
+    return domain    
+
+
+
+
+
+
+def distribute_mesh(domain, verbose=False, debug=False, parameters=None):
+    """ Distribute andsend the mesh info to all the processors.
+    Should only be run from processor 0 and will send info to the other
+    processors.
+    There should be a corresponding  rec_submesh called from all the other
+    processors
+    """
+
+    if debug:
+        verbose = True
+
+    numprocs = size()
+
+    
+    # Subdivide the mesh
+    if verbose: print 'Subdivide mesh'
+    new_nodes, new_triangles, new_boundary, triangles_per_proc, quantities, \
+           s2p_map, p2s_map = \
+           pmesh_divide_metis_with_map(domain, numprocs)
+
+    #PETE: s2p_map (maps serial domain triangles to parallel domain triangles)
+    #      sp2_map (maps parallel domain triangles to domain triangles)
+
+
+
+    # Build the mesh that should be assigned to each processor,
+    # this includes ghost nodes and the communication pattern
+    if verbose: print 'Build submeshes'    
+    submesh = build_submesh(new_nodes, new_triangles, new_boundary, quantities, triangles_per_proc, parameters)
+
+    if verbose:
+        for p in range(numprocs):
+            N = len(submesh['ghost_nodes'][p])                
+            M = len(submesh['ghost_triangles'][p])
+            print 'There are %d ghost nodes and %d ghost triangles on proc %d'\
+                  %(N, M, p)
+
+    #if debug:
+    #    from pprint import pprint
+    #    pprint(submesh)
+
+
+    # Send the mesh partition to the appropriate processor
+    if verbose: print 'Distribute submeshes'        
+    for p in range(1, numprocs):
+        send_submesh(submesh, triangles_per_proc, p2s_map, p, verbose)
+
+    # Build the local mesh for processor 0
+    points, vertices, boundary, quantities, \
+            ghost_recv_dict, full_send_dict, \
+            tri_map, node_map, tri_l2g, node_l2g, ghost_layer_width =\
+              extract_submesh(submesh, triangles_per_proc, p2s_map, 0)
+
+
+              
+    # Keep track of the number full nodes and triangles.
+    # This is useful later if one needs access to a ghost-free domain
+    # Here, we do it for process 0. The others are done in rec_submesh.
+    number_of_full_nodes = len(submesh['full_nodes'][0])
+    number_of_full_triangles = len(submesh['full_triangles'][0])
+        
+    #print
+    #for p in range(numprocs):
+    #    print 'Process %d:' %(p)
+    #
+    #    print 'full_triangles:'
+    #    print submesh['full_triangles'][p]
+    #
+    #    print 'full_nodes:'
+    #    print submesh['full_nodes'][p]
+    #
+    #    print 'ghost_triangles:'
+    #    print submesh['ghost_triangles'][p]#
+    #
+    #    print 'ghost_nodes:'
+    #   print submesh['ghost_nodes'][p]                                
+    #    print
+    #
+    #print 'Receive dict'
+    #print ghost_recv_dict
+    #
+    #print 'Send dict'
+    #print full_send_dict        
+
+
+    # Return structures necessary for building the parallel domain
+    return points, vertices, boundary, quantities,\
+           ghost_recv_dict, full_send_dict,\
+           number_of_full_nodes, number_of_full_triangles, \
+           s2p_map, p2s_map, tri_map, node_map, tri_l2g, node_l2g, \
+           ghost_layer_width
+    
+
+
+## def extract_l2g_map(map):
+##     # Extract l2g data  from corresponding map
+##     # Maps
+
+##     import numpy as num
+    
+##     # FIXME: this is where we loss the original order of 
+##     # sequential domain
+##     b = num.arange(len(map))
+
+##     l_ids = num.extract(map>-1,map)
+##     g_ids = num.extract(map>-1,b)
+
+## #    print len(g_ids)
+## #    print len(l_ids)
+## #    print l_ids
+
+##     l2g = num.zeros_like(g_ids)
+##     l2g[l_ids] = g_ids
+
+##     return l2g
+