anuga 1.3.5__zip

Users/steve/Miniconda/envs/anuga/Lib/site-packages/anuga/file_conversion/file_conversion.py

@@ -0,0 +1,367 @@
+""" Conversion routines.
+    ANUGA needs to deal with many different file formats, and this
+    module provides routines for easily converting between them.
+    
+    These routines are necessarily high level, sitting above the various
+    ANUGA modules. They take a file as input, and output a file.
+"""
+
+#non ANUGA imports
+from anuga.file.netcdf import NetCDFFile
+import numpy as num
+import os.path
+import os
+
+#ANUGA imports
+from anuga.coordinate_transforms.geo_reference import Geo_reference, \
+     write_NetCDF_georeference, ensure_geo_reference
+from anuga.abstract_2d_finite_volumes.pmesh2domain import \
+     pmesh_to_domain_instance
+from anuga.utilities.numerical_tools import ensure_numeric
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_mode_a, \
+                            netcdf_float
+
+from anuga.anuga_exceptions import *
+
+
+#shallow water imports
+from anuga.file.sww import Read_sww, Write_sww
+from anuga.shallow_water.shallow_water_domain import Domain
+from anuga.shallow_water.shallow_water_domain import Domain
+
+
+def sww2obj(filename, size):
+    """ Convert netcdf based data output to obj
+
+        Convert SWW data to OBJ data.
+        basefilename Stem of filename, needs size and extension added.
+        size The number of lines to write.
+    """
+
+    if filename[-4:] != '.sww':
+        raise IOError('Output file %s should be of type .sww.' % sww_file)
+
+    basefilename = filename[:-4]
+
+    # Get NetCDF
+    nc_fname = create_filename('.', basefilename, 'sww', size)
+    log.critical('Reading from %s' % nc_fname)
+    fid = NetCDFFile(nc_fname, netcdf_mode_r)  #Open existing file for read
+
+    # Get the variables
+    x = fid.variables['x']
+    y = fid.variables['y']
+    z = fid.variables['elevation']
+    time = fid.variables['time']
+    stage = fid.variables['stage']
+
+    M = size  #Number of lines
+    xx = num.zeros((M,3), num.float)
+    yy = num.zeros((M,3), num.float)
+    zz = num.zeros((M,3), num.float)
+
+    for i in range(M):
+        for j in range(3):
+            xx[i,j] = x[i+j*M]
+            yy[i,j] = y[i+j*M]
+            zz[i,j] = z[i+j*M]
+
+    # Write obj for bathymetry
+    FN = create_filename('.', basefilename, 'obj', size)
+    write_obj(FN,xx,yy,zz)
+
+    # Now read all the data with variable information, combine with
+    # x,y info and store as obj
+    for k in range(len(time)):
+        t = time[k]
+        log.critical('Processing timestep %f' % t)
+
+        for i in range(M):
+            for j in range(3):
+                zz[i,j] = stage[k,i+j*M]
+
+        #Write obj for variable data
+        #FN = create_filename(basefilename, 'obj', size, time=t)
+        FN = create_filename('.', basefilename[:5], 'obj', size, time=t)
+        write_obj(FN, xx, yy, zz)
+
+def timefile2netcdf(file_text, file_out = None, quantity_names=None, \
+                                time_as_seconds=False):
+    """Template for converting typical text files with time series to
+    NetCDF tms file.
+
+    The file format is assumed to be either two fields separated by a comma:
+
+        time [DD/MM/YY hh:mm:ss], value0 value1 value2 ...
+
+    E.g
+
+      31/08/04 00:00:00, 1.328223 0 0
+      31/08/04 00:15:00, 1.292912 0 0
+
+    or time (seconds), value0 value1 value2 ...
+
+      0.0, 1.328223 0 0
+      0.1, 1.292912 0 0
+
+    will provide a time dependent function f(t) with three attributes
+
+    filename is assumed to be the rootname with extensions .txt/.tms and .sww
+    """
+
+    import time, calendar
+    from anuga.config import time_format
+    from anuga.utilities.numerical_tools import ensure_numeric
+
+    if file_text[-4:] != '.txt':
+        raise IOError('Input file %s should be of type .txt.' % file_text)
+
+    if file_out == None:
+        file_out = file_text[:-4] + '.tms'
+
+    fid = open(file_text)
+    line = fid.readline()
+    fid.close()
+
+    fields = line.split(',')
+    msg = "File %s must have the format 'datetime, value0 value1 value2 ...'" \
+          % file_text
+    assert len(fields) == 2, msg
+
+    if not time_as_seconds:
+        try:
+            starttime = calendar.timegm(time.strptime(fields[0], time_format))
+        except ValueError:
+            msg = 'First field in file %s must be' % file_text
+            msg += ' date-time with format %s.\n' % time_format
+            msg += 'I got %s instead.' % fields[0]
+            raise DataTimeError, msg
+    else:
+        try:
+            starttime = float(fields[0])
+        except Error:
+            msg = "Bad time format"
+            raise DataTimeError, msg
+
+    # Split values
+    values = []
+    for value in fields[1].split():
+        values.append(float(value))
+
+    q = ensure_numeric(values)
+
+    msg = 'ERROR: File must contain at least one independent value'
+    assert len(q.shape) == 1, msg
+
+    # Read times proper
+    from anuga.config import time_format
+    import time, calendar
+
+    fid = open(file_text)
+    lines = fid.readlines()
+    fid.close()
+
+    N = len(lines)
+    d = len(q)
+
+    T = num.zeros(N, num.float)       # Time
+    Q = num.zeros((N, d), num.float)  # Values
+
+    for i, line in enumerate(lines):
+        fields = line.split(',')
+        if not time_as_seconds:
+            realtime = calendar.timegm(time.strptime(fields[0], time_format))
+        else:
+            realtime = float(fields[0])
+        T[i] = realtime - starttime
+
+        for j, value in enumerate(fields[1].split()):
+            Q[i, j] = float(value)
+
+    msg = 'File %s must list time as a monotonuosly ' % file_text
+    msg += 'increasing sequence'
+    assert num.alltrue(T[1:] - T[:-1] > 0), msg
+
+    #Create NetCDF file
+    fid = NetCDFFile(file_out, netcdf_mode_w)
+
+    fid.institution = 'Geoscience Australia'
+    fid.description = 'Time series'
+
+    #Reference point
+    #Start time in seconds since the epoch (midnight 1/1/1970)
+    #FIXME: Use Georef
+    fid.starttime = starttime
+
+    # dimension definitions
+    #fid.createDimension('number_of_volumes', self.number_of_volumes)
+    #fid.createDimension('number_of_vertices', 3)
+
+    fid.createDimension('number_of_timesteps', len(T))
+
+    fid.createVariable('time', netcdf_float, ('number_of_timesteps',))
+
+    fid.variables['time'][:] = T
+
+    for i in range(Q.shape[1]):
+        try:
+            name = quantity_names[i]
+        except:
+            name = 'Attribute%d' % i
+
+        fid.createVariable(name, netcdf_float, ('number_of_timesteps',))
+        fid.variables[name][:] = Q[:,i]
+
+    fid.close()
+    
+    
+
+def tsh2sww(filename, verbose=False):
+    """
+    to check if a tsh/msh file 'looks' good.
+    """
+
+    if filename[-4:] != '.tsh' and filename[-4:] != '.msh':
+        raise IOError('Input file %s should be .tsh or .msh.' % name_out)
+
+    if verbose == True: log.critical('Creating domain from %s' % filename)
+
+    domain = pmesh_to_domain_instance(filename, Domain)
+
+    if verbose == True: log.critical("Number of triangles = %s" % len(domain))
+
+    domain.smooth = True
+    domain.format = 'sww'   #Native netcdf visualisation format
+    file_path, filename = path.split(filename)
+    filename, ext = path.splitext(filename)
+    domain.set_name(filename)
+    domain.reduction = mean
+
+    if verbose == True: log.critical("file_path = %s" % file_path)
+
+    if file_path == "":
+        file_path = "."
+    domain.set_datadir(file_path)
+
+    if verbose == True:
+        log.critical("Output written to %s%s%s.%s"
+                     % (domain.get_datadir(), sep, domain.get_name(),
+                        domain.format))
+
+    sww = SWW_file(domain)
+    sww.store_connectivity()
+    sww.store_timestep()
+
+
+
+def get_min_max_indices(latitudes_ref, longitudes_ref,
+                         minlat=None, maxlat=None,
+                         minlon=None, maxlon=None):
+    """
+    Return max, min indexes (for slicing) of the lat and long arrays to cover
+    the area specified with min/max lat/long.
+
+    Think of the latitudes and longitudes describing a 2d surface.
+    The area returned is, if possible, just big enough to cover the
+    inputed max/min area. (This will not be possible if the max/min area
+    has a section outside of the latitudes/longitudes area.)
+
+    asset  longitudes are sorted,
+    long - from low to high (west to east, eg 148 - 151)
+    assert latitudes are sorted, ascending or decending
+    """
+
+    latitudes = latitudes_ref[:]
+    longitudes = longitudes_ref[:]
+
+    latitudes = ensure_numeric(latitudes)
+    longitudes = ensure_numeric(longitudes)
+
+    assert num.allclose(num.sort(longitudes), longitudes)
+
+    #print latitudes[0],longitudes[0]
+    #print len(latitudes),len(longitudes)
+    #print latitudes[len(latitudes)-1],longitudes[len(longitudes)-1]
+
+    lat_ascending = True
+    if not num.allclose(num.sort(latitudes), latitudes):
+        lat_ascending = False
+        # reverse order of lat, so it's in ascending order
+        latitudes = latitudes[::-1]
+        assert num.allclose(num.sort(latitudes), latitudes)
+
+    largest_lat_index = len(latitudes)-1
+
+    #Cut out a smaller extent.
+    if minlat == None:
+        lat_min_index = 0
+    else:
+        lat_min_index = num.searchsorted(latitudes, minlat)-1
+        if lat_min_index < 0:
+            lat_min_index = 0
+
+    if maxlat == None:
+        lat_max_index = largest_lat_index #len(latitudes)
+    else:
+        lat_max_index = num.searchsorted(latitudes, maxlat)
+        if lat_max_index > largest_lat_index:
+            lat_max_index = largest_lat_index
+
+    if minlon == None:
+        lon_min_index = 0
+    else:
+        lon_min_index = num.searchsorted(longitudes, minlon)-1
+        if lon_min_index < 0:
+            lon_min_index = 0
+
+    if maxlon == None:
+        lon_max_index = len(longitudes)
+    else:
+        lon_max_index = num.searchsorted(longitudes, maxlon)
+
+    # Reversing the indexes, if the lat array is decending
+    if lat_ascending is False:
+        lat_min_index, lat_max_index = largest_lat_index - lat_max_index, \
+                                       largest_lat_index - lat_min_index
+    lat_max_index = lat_max_index + 1 # taking into account how slicing works
+    lon_max_index = lon_max_index + 1 # taking into account how slicing works
+
+    return lat_min_index, lat_max_index, lon_min_index, lon_max_index
+
+
+def write_obj(filename, x, y, z):
+    """Store x,y,z vectors into filename (obj format).
+
+       Vectors are assumed to have dimension (M,3) where
+       M corresponds to the number elements.
+       triangles are assumed to be disconnected
+
+       The three numbers in each vector correspond to three vertices,
+
+       e.g. the x coordinate of vertex 1 of element i is in x[i,1]
+    """
+
+    import os.path
+
+    root, ext = os.path.splitext(filename)
+    if ext == '.obj':
+        FN = filename
+    else:
+        FN = filename + '.obj'
+
+    outfile = open(FN, 'wb')
+    outfile.write("# Triangulation as an obj file\n")
+
+    M, N = x.shape
+    assert N == 3  #Assuming three vertices per element
+
+    for i in range(M):
+        for j in range(N):
+            outfile.write("v %f %f %f\n" % (x[i,j], y[i,j], z[i,j]))
+
+    for i in range(M):
+        base = i * N
+        outfile.write("f %d %d %d\n" % (base+1, base+2, base+3))
+
+    outfile.close()
+

Users/steve/Miniconda/envs/anuga/Lib/site-packages/anuga/file_conversion/grd2array.py

@@ -0,0 +1,122 @@
+""" Load a DEM file, decimate it, and resave it.
+"""
+
+import numpy as num
+import os
+
+from anuga.config import netcdf_mode_r, netcdf_mode_w, netcdf_float
+
+def grd2array(filename, verbose=False):
+        """Read Digital Elevation model from the following ASCII format (.asc or .grd)
+    
+        Example:
+        ncols         3121
+        nrows         1800
+        xllcorner     722000
+        yllcorner     5893000
+        cellsize      25
+        NODATA_value  -9999
+        138.3698 137.4194 136.5062 135.5558 ..........
+    
+    
+        An accompanying file with same basename but extension .prj must exist
+        and is used to fix the UTM zone, datum, false northings and eastings.
+    
+        The prj format is assumed to be as
+    
+        Projection    UTM
+        Zone          56
+        Datum         WGS84
+        Zunits        NO
+        Units         METERS
+        Spheroid      WGS84
+        Xshift        0.0000000000
+        Yshift        10000000.0000000000
+        Parameters
+        """
+
+
+
+        msg = 'Filename must be a text string'
+        assert isinstance(filename, basestring), msg
+        
+        msg = 'Extension should be .grd or asc'
+        assert os.path.splitext(filename)[1] in ['.grd', '.asc'], msg
+     
+
+            
+    
+        root = filename[:-4]
+    
+    
+        #Read DEM data
+        datafile = open(filename)
+    
+        if verbose: log.critical('Reading data from %s' % (filename))
+    
+        lines = datafile.readlines()
+        datafile.close()
+    
+        if verbose: log.critical('Got %d lines' % len(lines))
+    
+
+        ncols = int(lines[0].split()[1].strip())
+        nrows = int(lines[1].split()[1].strip())
+
+    
+        # Do cellsize (line 4) before line 2 and 3
+        cellsize = float(lines[4].split()[1].strip())
+    
+        # Checks suggested by Joaquim Luis
+        # Our internal representation of xllcorner
+        # and yllcorner is non-standard.
+        xref = lines[2].split()
+        if xref[0].strip() == 'xllcorner':
+            xllcorner = float(xref[1].strip()) # + 0.5*cellsize # Correct offset
+        elif xref[0].strip() == 'xllcenter':
+            xllcorner = float(xref[1].strip())
+        else:
+            msg = 'Unknown keyword: %s' % xref[0].strip()
+            raise Exception, msg
+    
+        yref = lines[3].split()
+        if yref[0].strip() == 'yllcorner':
+            yllcorner = float(yref[1].strip()) # + 0.5*cellsize # Correct offset
+        elif yref[0].strip() == 'yllcenter':
+            yllcorner = float(yref[1].strip())
+        else:
+            msg = 'Unknown keyword: %s' % yref[0].strip()
+            raise Exception, msg
+    
+        NODATA_value = int(float(lines[5].split()[1].strip()))
+    
+        assert len(lines) == nrows + 6
+    
+  
+        #Store data
+        import numpy
+    
+        datafile = open(filename)
+        Z = numpy.loadtxt(datafile, skiprows=6)
+        datafile.close()
+        
+        #print Z.shape
+        #print Z
+        
+        # For raster data we need to a flip and transpose
+        Z = numpy.flipud(Z)
+
+        # Transpose z to have y coordinates along the first axis and x coordinates
+        # along the second axis
+        Z = Z.transpose()
+    
+        x = num.linspace(xllcorner, xllcorner+cellsize*(ncols-1), ncols)
+        y = num.linspace(yllcorner, yllcorner+cellsize*(nrows-1), nrows)
+        
+        # get rid of NODATA
+        import numpy
+        Z = numpy.where(Z == NODATA_value , numpy.nan, Z)
+        
+        return x,y, Z
+        
+

Users/steve/Miniconda/envs/anuga/Lib/site-packages/anuga/file_conversion/sdf2pts.py

@@ -0,0 +1,181 @@
+
+from anuga.coordinate_transforms.geo_reference import Geo_reference
+from anuga.geospatial_data.geospatial_data import Geospatial_data
+
+def _read_hecras_cross_sections(lines):
+    """Return block of surface lines for each cross section
+    Starts with SURFACE LINE,
+    Ends with END CROSS-SECTION
+    """
+
+    points = []
+
+    reading_surface = False
+    for i, line in enumerate(lines):
+        if len(line.strip()) == 0:    #Ignore blanks
+            continue
+
+        if lines[i].strip().startswith('SURFACE LINE'):
+            reading_surface = True
+            continue
+
+        if lines[i].strip().startswith('END') and reading_surface:
+            yield points
+            reading_surface = False
+            points = []
+
+        if reading_surface:
+            fields = line.strip().split(',')
+            easting = float(fields[0])
+            northing = float(fields[1])
+            elevation = float(fields[2])
+            points.append([easting, northing, elevation])
+
+
+
+def sdf2pts(name_in,
+                              name_out=None,
+                              verbose=False):
+    """Read HEC-RAS Elevation datal from the following ASCII format (.sdf)
+
+    basename_in Sterm of input filename.
+    basename_out Sterm of output filename.
+    verbose True if this function is to be verbose.
+
+    Example:
+
+# RAS export file created on Mon 15Aug2005 11:42
+# by HEC-RAS Version 3.1.1
+
+BEGIN HEADER:
+  UNITS: METRIC
+  DTM TYPE: TIN
+  DTM: v:\1\cit\perth_topo\river_tin
+  STREAM LAYER: c:\local\hecras\21_02_03\up_canning_cent3d.shp
+  CROSS-SECTION LAYER: c:\local\hecras\21_02_03\up_can_xs3d.shp
+  MAP PROJECTION: UTM
+  PROJECTION ZONE: 50
+  DATUM: AGD66
+  VERTICAL DATUM:
+  NUMBER OF REACHES:  19
+  NUMBER OF CROSS-SECTIONS:  14206
+END HEADER:
+
+Only the SURFACE LINE data of the following form will be utilised
+  CROSS-SECTION:
+    STREAM ID:Southern-Wungong
+    REACH ID:Southern-Wungong
+    STATION:19040.*
+    CUT LINE:
+      405548.671603161 , 6438142.7594925
+      405734.536092045 , 6438326.10404912
+      405745.130459356 , 6438331.48627354
+      405813.89633823 , 6438368.6272789
+    SURFACE LINE:
+     405548.67,   6438142.76,   35.37
+     405552.24,   6438146.28,   35.41
+     405554.78,   6438148.78,   35.44
+     405555.80,   6438149.79,   35.44
+     405559.37,   6438153.31,   35.45
+     405560.88,   6438154.81,   35.44
+     405562.93,   6438156.83,   35.42
+     405566.50,   6438160.35,   35.38
+     405566.99,   6438160.83,   35.37
+     ...
+   END CROSS-SECTION
+
+    Convert to NetCDF pts format which is
+
+    points:  (Nx2) float array
+    elevation: N float array
+    """
+
+    import os
+    from anuga.file.netcdf import NetCDFFile
+
+    if name_in[-4:] != '.sdf':
+        raise IOError('Input file %s should be of type .sdf.' % name_in)
+
+    if name_out is None:
+        name_out = name_in[:-4] + '.pts'
+    elif name_out[-4:] != '.pts':
+        raise IOError('Input file %s should be of type .pts.' % name_out)
+
+    # Get ASCII file
+    infile = open(name_in, 'r')
+
+    if verbose: log.critical('Reading DEM from %s' % (root + '.sdf'))
+
+    lines = infile.readlines()
+    infile.close()
+
+    if verbose: log.critical('Converting to pts format')
+
+    # Scan through the header, picking up stuff we need.
+    i = 0
+    while lines[i].strip() == '' or lines[i].strip().startswith('#'):
+        i += 1
+
+    assert lines[i].strip().upper() == 'BEGIN HEADER:'
+    i += 1
+
+    assert lines[i].strip().upper().startswith('UNITS:')
+    units = lines[i].strip().split()[1]
+    i += 1
+
+    assert lines[i].strip().upper().startswith('DTM TYPE:')
+    i += 1
+
+    assert lines[i].strip().upper().startswith('DTM:')
+    i += 1
+
+    assert lines[i].strip().upper().startswith('STREAM')
+    i += 1
+
+    assert lines[i].strip().upper().startswith('CROSS')
+    i += 1
+
+    assert lines[i].strip().upper().startswith('MAP PROJECTION:')
+    projection = lines[i].strip().split(':')[1]
+    i += 1
+
+    assert lines[i].strip().upper().startswith('PROJECTION ZONE:')
+    zone = int(lines[i].strip().split(':')[1])
+    i += 1
+
+    assert lines[i].strip().upper().startswith('DATUM:')
+    datum = lines[i].strip().split(':')[1]
+    i += 1
+
+    assert lines[i].strip().upper().startswith('VERTICAL DATUM:')
+    i += 1
+
+    assert lines[i].strip().upper().startswith('NUMBER OF REACHES:')
+    i += 1
+
+    assert lines[i].strip().upper().startswith('NUMBER OF CROSS-SECTIONS:')
+    number_of_cross_sections = int(lines[i].strip().split(':')[1])
+    i += 1
+
+    # Now read all points
+    points = []
+    elevation = []
+    for j, entries in enumerate(_read_hecras_cross_sections(lines[i:])):
+        for k, entry in enumerate(entries):
+            points.append(entry[:2])
+            elevation.append(entry[2])
+
+    msg = 'Actual #number_of_cross_sections == %d, Reported as %d'\
+          %(j+1, number_of_cross_sections)
+    assert j+1 == number_of_cross_sections, msg
+
+    # Get output file, write PTS data
+    if name_out == None:
+        ptsname = name_in[:-4] + '.pts'
+    else:
+        ptsname = name_out
+
+    geo_ref = Geo_reference(zone, 0, 0, datum, projection, units)
+    geo = Geospatial_data(points, {"elevation":elevation},
+                          verbose=verbose, geo_reference=geo_ref)
+    geo.export_points_file(ptsname)

Users/steve/Miniconda/envs/anuga/Lib/site-packages/anuga/file_conversion/setup.py

@@ -0,0 +1,29 @@
+from __future__ import division, print_function
+
+import os
+import sys
+
+from os.path import join
+
+def configuration(parent_package='',top_path=None):
+    
+    from numpy.distutils.misc_util import Configuration
+    from numpy.distutils.system_info import get_info
+    
+    config = Configuration('file_conversion', parent_package, top_path)
+
+    config.add_data_dir('tests')
+    config.add_data_dir(join('tests','data'))
+
+    util_dir = os.path.abspath(join(os.path.dirname(__file__),'..','utilities'))
+    
+    config.add_extension('calc_grid_values_ext',
+                         sources=['calc_grid_values_ext.c'],
+                         include_dirs=[util_dir])
+    
+
+    return config
+    
+if __name__ == '__main__':
+    from numpy.distutils.core import setup
+    setup(configuration=configuration)