metradar 0.1.0__py3-none-any.whl

metradar/retrieve_micaps_server.py

@@ -0,0 +1,2061 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (c) 2019 NMC Developers.
+# Distributed under the terms of the GPL V3 License.
+
+"""
+This is the retrieve module which get data from MICAPS cassandra service
+with Python API.
+Checking url, like:
+http://10.32.8.164:8080/DataService?requestType=getLatestDataName&directory=ECMWF_HR/TMP/850&fileName=&filter=*.024
+"""
+
+import warnings
+import re
+import http.client
+import urllib.parse
+import pickle
+import bz2
+from io import BytesIO
+from datetime import datetime, timedelta
+import numpy as np
+import xarray as xr
+import pandas as pd
+from tqdm import tqdm
+import DataBlock_pb2
+import config as CONFIG
+import os
+# add by zwj 20210311
+import struct
+def _structure_size(structure):
+    """ Find the size of a structure in bytes. """
+    return struct.calcsize('<' + ''.join([i[1] for i in structure]))
+
+def _unpack_from_buf(buf, pos, structure):
+    """ Unpack a structure from a buffer. """
+    size = _structure_size(structure)
+    return _unpack_structure(buf[pos:pos + size], structure)
+
+def _unpack_structure(string, structure):
+    """ Unpack a structure from a string """
+    fmt = '<' + ''.join([i[1] for i in structure])  # little-endian
+    lst = struct.unpack(fmt, string)
+    return dict(zip([i[0] for i in structure], lst))
+
+def get_http_result(host, port, url):
+    """
+    Get the http contents.
+    """
+
+    http_client = None
+    try:
+        http_client = http.client.HTTPConnection(host, port, timeout=120)
+        http_client.request('GET', url)
+        response = http_client.getresponse()
+        return response.status, response.read()
+    except Exception as e:
+        print(e)
+        return 0,
+    finally:
+        if http_client:
+            http_client.close()
+
+gds_element_id_dict ={
+    "经度": 1,  
+    "纬度": 2,  
+    "测站高度": 3,  
+    "测站级别": 4,  
+    "测站类型": 5,  
+    "气压传感器海拔高度": 6,  
+    "温湿传感器离地面高度": 7,  
+    "温湿传感器距水面高度": 8,  
+    "风速传感器距地面高度": 9,  
+    "风传感器距甲板平台高度": 10,  
+    "风速传感器距水面高度": 11,  
+    "移动平台移动方向": 12,  
+    "移动平台移动速度": 13,  
+    "海盐传感器距海面深度": 14,  
+    "浪高传感器距海面高度": 15,  
+    "浮标方位": 16,  
+    "总水深": 17,  
+    "海面_水面以下深度": 18,  
+    "船面距海面高度": 19,  
+    "方位或方位角": 20,  
+    "字符型站名": 21,  
+    "风向": 201,  
+    "风速": 203,  
+    "平均风向_1分钟": 205,  
+    "平均风速_1分钟": 207,  
+    "平均风向_2分钟": 209,  
+    "平均风速_2分钟": 211,  
+    "平均风向_10分钟": 213,  
+    "平均风速_10分钟": 215,  
+    "最大风速的风向": 217,  
+    "最大风速": 219,  
+    "瞬时风向": 221,  
+    "瞬时风速": 223,  
+    "极大风速的风向": 225,  
+    "极大风速": 227,  
+    "过去6小时极大瞬时风速的风向": 229,  
+    "过去6小时极大瞬时风速": 231,  
+    "过去12小时极大瞬时风速的风向": 233,  
+    "过去12小时极大瞬时风速": 235,  
+    "风力": 237,  
+    "海平面气压": 401,  
+    "变压_3小时": 403,  
+    "变压_24小时": 405,  
+    "本站气压": 407,  
+    "最高气压": 409,  
+    "最低气压": 411,  
+    "气压": 413,  
+    "日平均气压": 415,  
+    "日平均海平面气压": 417,  
+    "高度_探空": 419,  
+    "位势高度_探空": 421,  
+    "温度": 601,  
+    "最高气温": 603,  
+    "最低气温": 605,  
+    "变温_24小时": 607,  
+    "过去24小时最高气温": 609,  
+    "过去24小时最低气温": 611,  
+    "日平均气温": 613,  
+    "露点温度": 801,  
+    "温度露点差": 803,  
+    "相对湿度": 805,  
+    "最小相对湿度": 807,  
+    "日平均相对湿度": 809,  
+    "水汽压": 811,  
+    "日平均水汽压": 813,  
+    "降水量": 1001,  
+    "降水_1小时": 1003,  
+    "降水_3小时": 1005,  
+    "降水_6小时": 1007,  
+    "降水_12小时": 1009,  
+    "降水_24小时": 1011,  
+    "日总降水": 1013,  
+    "降水量_20_08时": 1015,  
+    "降水量_08_20时": 1017,  
+    "降水量_20_20时": 1019,  
+    "降水量_08_08时": 1021,  
+    "蒸发": 1023,  
+    "蒸发_大型": 1025,  
+    "可降水分_预报降水量": 1027,  
+    "平均水平能见度_1分钟": 1201,  
+    "平均水平能见度_10分钟": 1203,  
+    "最小水平能见度": 1205,  
+    "水平能见度_人工": 1207,  
+    "总云量": 1401,  
+    "低云量": 1403,  
+    "云底高度": 1405,  
+    "低云状": 1407,  
+    "中云状": 1409,  
+    "高云状": 1411,  
+    "日平均总云量": 1413,  
+    "日平均低云量": 1415,  
+    "云量_低云或中云": 1417,  
+    "云类型 ": 1419,  
+    "现在天气": 1601,  
+    "过去天气1": 1603,  
+    "过去天气2": 1605,  
+    "龙卷类型": 1801,  
+    "龙卷所在方位": 1803,  
+    "最大冰雹直径": 1805,  
+    "雷暴": 1807,  
+    "电流强度_闪电定位": 1809,  
+    "地面温度": 2001,  
+    "最高地面温度": 2003,  
+    "最低地面温度": 2005,  
+    "过去12小时最低地面温度": 2007,  
+    "地温_5cm": 2009,  
+    "地温_10cm": 2011,  
+    "地温_15cm": 2013,  
+    "地温_20cm": 2015,  
+    "地温_40cm": 2017,  
+    "地温_80cm": 2019,  
+    "地温_160cm": 2021,  
+    "地温_320cm": 2023,  
+    "草面_雪面温度": 2025,  
+    "草面_雪面最高温度": 2027,  
+    "草面_雪面最低温度": 2029,  
+    "日平均地面温度": 2031,  
+    "日平均5cm地温": 2033,  
+    "日平均10cm地温": 2035,  
+    "日平均15cm地温": 2037,  
+    "日平均20cm地温": 2039,  
+    "日平均40cm地温": 2041,  
+    "日平均80cm地温": 2043,  
+    "日平均160cm地温": 2045,  
+    "日平均320cm地温": 2047,  
+    "日平均草面_雪面温度": 2049,  
+    "地面状态": 2201,  
+    "积雪深度": 2203,  
+    "雪压": 2205,  
+    "电线积冰直径": 2207,  
+    "电线积冰_现象": 2209,  
+    "电线积冰_南北方向直径": 2211,  
+    "电线积冰_南北方向厚度": 2213,  
+    "电线积冰_南北方向重量": 2215,  
+    "电线积冰_东西方向直径": 2217,  
+    "电线积冰_东西方向厚度": 2219,  
+    "电线积冰_东西方向重量": 2221,  
+    "船上结冰原因": 2223,  
+    "船上结冰厚度": 2225,  
+    "船上结冰速度": 2227,  
+    "海冰密集度": 2229,  
+    "冰情发展": 2231,  
+    "冰总量和类型": 2233,  
+    "冰缘方位": 2235,  
+    "冰情": 2237,  
+    "最高气压出现时间": 10001,  
+    "最低气压出现时间": 10003,  
+    "最高气温出现时间": 10005,  
+    "最低气温出现时间": 10007,  
+    "最小相对湿度出现时间": 10009,  
+    "最大风速出现时间": 10011,  
+    "极大风速出现时间": 10013,  
+    "最高地面温度出现时间": 10015,  
+    "最低地面温度出现时间": 10017,  
+    "草面_雪面最低温度出现时间": 10019,  
+    "草面_雪面最高温度出现时间": 10021,  
+    "最小水平能见度出现时间": 10023,  
+    "天气出现时间": 10025,  
+    "海表最高温度出现时间": 10027,  
+    "海表最低温度出现时间": 10029,  
+    "最大波高出现时间": 10031,  
+    "风速表类型": 2401,  
+    "湿球温度测量方法": 2403,  
+    "海面温度测量方法": 2405,  
+    "洋流测量方法": 2407,  
+    "气压倾向特征": 2409,  
+    "海面温度": 2601,  
+    "湿球温度": 2603,  
+    "海面盐度": 2605,  
+    "海表最高温度": 2607,  
+    "海表最低温度": 2609,  
+    "海水温度": 2611,  
+    "海水盐度": 2613,  
+    "海面海流方向": 2801,  
+    "海面海流速度": 2803,  
+    "洋流方向和速度的平均周期": 2805,  
+    "表层海洋面流速": 2807,  
+    "表层海洋面波向": 2809,  
+    "海流方向": 2811,  
+    "海流速度": 2813,  
+    "波浪方向": 3001,  
+    "波浪周期": 3003,  
+    "波浪高度": 3005,  
+    "风浪方向": 3007,  
+    "风浪周期": 3009,  
+    "风浪高度": 3011,  
+    "第一涌浪方向": 3013,  
+    "第一涌浪周期": 3015,  
+    "第一涌浪高度": 3017,  
+    "第二涌浪方向": 3019,  
+    "第二涌浪周期": 3021,  
+    "第二涌浪高度": 3023,  
+    "有效波高": 3025,  
+    "有效波高的周期": 3027,  
+    "平均波高": 3029,  
+    "平均波周期": 3031,  
+    "最大波高": 3033,  
+    "最大波高的周期": 3035,  
+    "人工测量浪高": 3037,  
+    "仪器测量浪高": 3039,  
+    "浪级代码": 3041
+}
+
+class GDSDataService:
+    def __init__(self):
+        # set MICAPS GDS服务器地址
+        self.berror = False
+        if CONFIG is None:
+            self.berror=True
+
+        cc = CONFIG._get_config_from_rcfile()
+        
+        self.gdsIp = cc['MICAPS']['GDS_IP']
+        self.gdsPort = cc['MICAPS']['GDS_PORT']
+        # print(self.gdsIp)
+        # print(self.gdsPort)
+    def getLatestDataName(self, directory, filter):
+        return get_http_result(
+            self.gdsIp, self.gdsPort, "/DataService" +
+            self.get_concate_url("getLatestDataName", directory, "", filter))
+
+    def getData(self, directory, fileName):
+        return get_http_result(
+            self.gdsIp, self.gdsPort, "/DataService" +
+            self.get_concate_url("getData", directory, fileName, ""))
+
+    def getFileList(self,directory):
+        return get_http_result(
+            self.gdsIp, self.gdsPort, "/DataService" + 
+            self.get_concate_url("getFileList", directory, "",""))
+
+    # 将请求参数拼接到url
+    def get_concate_url(self, requestType, directory, fileName, filter):
+        url = ""
+        url += "?requestType=" + requestType
+        url += "&directory=" + directory
+        url += "&fileName=" + fileName
+        url += "&filter=" + filter
+        return urllib.parse.quote(url, safe=':/?=&')
+
+
+def get_file_list(path, latest=None):
+    """return file list of cassandra data servere path
+    
+    Args:
+        path (string): cassandra data servere path.
+        latest (integer): get the latest n files.
+    
+    Returns:
+        list: list of filenames.
+    """
+
+    # connect to data service
+    service = GDSDataService()
+    if service.berror == True:
+        return None
+    # 获得指定目录下的所有文件
+    status, response = service.getFileList(path)
+    MappingResult = DataBlock_pb2.MapResult()
+    file_list = []
+    if status == 200:
+        if MappingResult is not None:
+            # Protobuf的解析
+            MappingResult.ParseFromString(response)
+            results = MappingResult.resultMap
+            # 遍历指定目录
+            for name_size_pair in results.items():
+                if (name_size_pair[1] != 'D'):
+                    file_list.append(name_size_pair[0])
+
+    # sort the file list
+    if latest is not None:
+        file_list.sort(reverse=True)
+        file_list = file_list[0:min(len(file_list), latest)]
+
+    return file_list
+
+
+def get_latest_initTime(directory, suffix="*.006"):
+    """
+    Get the latest initial time string.
+    
+    Args:
+        directory (string): the data directory on the service.
+        suffix (string, optional):  the filename filter pattern.
+    Examples:
+    >>> initTime = get_latest_initTime("ECMWF_HR/TMP/850")
+    """
+
+    # connect to data service
+    service = GDSDataService()
+    if service.berror == True:
+        return None
+    # get lastest data filename
+    try:
+        status, response = service.getLatestDataName(directory, suffix)
+    except ValueError:
+        print('Can not retrieve data from ' + directory)
+        return None
+    StringResult = DataBlock_pb2.StringResult()
+    if status == 200:
+        StringResult.ParseFromString(response)
+        if StringResult is not None:
+            filename = StringResult.name
+            if filename == '':
+                return None
+            else:
+                return filename.split('.')[0]
+        else:
+            return None
+
+    # extract initial time
+    # return filename.split(".")[0]
+    return None
+
+
+def get_model_grid(directory, filename=None, suffix="*.024",
+                   varname='data', varattrs={'units':''}, scale_off=None,
+                   levattrs={'long_name':'pressure_level', 'units':'hPa',
+                             '_CoordinateAxisType':'Pressure'}, cache=True):
+    """
+    Retrieve numeric model grid forecast from MICAPS cassandra service.
+    Support ensemble member forecast.
+    :param directory: the data directory on the service
+    :param filename: the data filename, if none, will be the latest file.
+    :param suffix: the filename filter pattern which will be used to
+                   find the specified file.
+    :param varname: set variable name.
+    :param varattrs: set variable attributes, dictionary type.
+    :param scale_off: [scale, offset], return values = values*scale + offset.
+    :param levattrs: set level coordinate attributes, diectionary type.
+    :param cache: cache retrieved data to local directory, default is True.
+    :return: data, xarray type
+    :Examples:
+    >>> data = get_model_grid("ECMWF_HR/TMP/850")
+    >>> data_ens = get_model_grid("ECMWF_ENSEMBLE/RAW/HGT/500", filename='18021708.024')
+    >>> data_ens = get_model_grid('ECMWF_ENSEMBLE/RAW/TMP_2M', '19083008.024')
+    """
+
+    # get data file name
+    if filename is None:
+        try:
+            # connect to data service
+            service = GDSDataService()
+            if service.berror == True:
+                return None
+            status, response = service.getLatestDataName(directory, suffix)
+        except ValueError:
+            print('Can not retrieve data from ' + directory)
+            return None
+        StringResult = DataBlock_pb2.StringResult()
+        if status == 200:
+            StringResult.ParseFromString(response)
+            if StringResult is not None:
+                filename = StringResult.name
+                if filename == '':
+                    return None
+            else:
+                return None
+
+    # retrieve data from cached file
+    if cache:
+        cache_file = CONFIG.get_cache_file(directory, filename, name="MICAPS_DATA")
+        if cache_file.is_file():
+            with open(cache_file, 'rb') as f:
+                data = pickle.load(f)
+                return data
+    
+    # get data contents
+    try:
+        service = GDSDataService()
+        if service.berror == True:
+            return None
+        status, response = service.getData(directory, filename)
+    except ValueError:
+        print('Can not retrieve data' + filename + ' from ' + directory)
+        return None
+    ByteArrayResult = DataBlock_pb2.ByteArrayResult()
+    if status == 200:
+        ByteArrayResult.ParseFromString(response)
+        if ByteArrayResult.errorCode == 1:
+            return None
+        if ByteArrayResult is not None:
+            byteArray = ByteArrayResult.byteArray
+            if byteArray == '':
+                print('There is no data ' + filename + ' in ' + directory)
+                return None
+
+            # define head information structure (278 bytes)
+            head_dtype = [('discriminator', 'S4'), ('type', 'i2'),
+                          ('modelName', 'S20'), ('element', 'S50'),
+                          ('description', 'S30'), ('level', 'f4'),
+                          ('year', 'i4'), ('month', 'i4'), ('day', 'i4'),
+                          ('hour', 'i4'), ('timezone', 'i4'),
+                          ('period', 'i4'), ('startLongitude', 'f4'),
+                          ('endLongitude', 'f4'), ('longitudeGridSpace', 'f4'),
+                          ('longitudeGridNumber', 'i4'),
+                          ('startLatitude', 'f4'), ('endLatitude', 'f4'),
+                          ('latitudeGridSpace', 'f4'),
+                          ('latitudeGridNumber', 'i4'),
+                          ('isolineStartValue', 'f4'),
+                          ('isolineEndValue', 'f4'),
+                          ('isolineSpace', 'f4'),
+                          ('perturbationNumber', 'i2'),
+                          ('ensembleTotalNumber', 'i2'),
+                          ('minute', 'i2'), ('second', 'i2'),
+                          ('Extent', 'S92')]
+
+            # read head information
+            head_info = np.frombuffer(byteArray[0:278], dtype=head_dtype)
+
+            # get required grid information
+            data_type = head_info['type'][0]
+            nlon = head_info['longitudeGridNumber'][0]
+            nlat = head_info['latitudeGridNumber'][0]
+            nmem = head_info['ensembleTotalNumber'][0]
+
+            # define data structure
+            if data_type == 4:
+                data_dtype = [('data', 'f4', (nlat, nlon))]
+                data_len = nlat * nlon * 4
+            elif data_type == 11:
+                data_dtype = [('data', 'f4', (2, nlat, nlon))]
+                data_len = 2 * nlat * nlon * 4
+            else:
+                raise Exception("Data type is not supported")
+
+            # read data
+            if nmem == 0:
+                data = np.frombuffer(byteArray[278:], dtype=data_dtype)
+                data = np.squeeze(data['data'])
+            else:
+                if data_type == 4:
+                    data = np.full((nmem, nlat, nlon), np.nan)
+                else:
+                    data = np.full((2, nmem, nlat, nlon), np.nan)
+                ind = 0
+                for imem in range(nmem):
+                    head_info_mem = np.frombuffer(
+                        byteArray[ind:(ind+278)], dtype=head_dtype)
+                    ind += 278
+                    data_mem = np.frombuffer(
+                        byteArray[ind:(ind+data_len)], dtype=data_dtype)
+                    ind += data_len
+                    number = head_info_mem['perturbationNumber'][0]
+                    if data_type == 4:
+                        data[number, :, :] = np.squeeze(data_mem['data'])
+                    else:
+                        data[:, number, :, :] = np.squeeze(data_mem['data'])
+
+            # scale and offset the data, if necessary.
+            if scale_off is not None:
+                data = data * scale_off[0] + scale_off[1]
+
+            # construct longitude and latitude coordinates
+            slon = head_info['startLongitude'][0]
+            dlon = head_info['longitudeGridSpace'][0]
+            slat = head_info['startLatitude'][0]
+            dlat = head_info['latitudeGridSpace'][0]
+            lon = np.arange(nlon) * dlon + slon
+            lat = np.arange(nlat) * dlat + slat
+            level = np.array([head_info['level'][0]])
+
+            # construct initial time and forecast hour
+            init_time = datetime(head_info['year'][0], head_info['month'][0],
+                                 head_info['day'][0], head_info['hour'][0])
+            fhour = np.array([head_info['period'][0]], dtype=np.float)
+            time = init_time + timedelta(hours=fhour[0])
+            init_time = np.array([init_time], dtype='datetime64[ms]')
+            time = np.array([time], dtype='datetime64[ms]')
+
+            # construct ensemble number
+            if nmem != 0:
+                number = np.arange(nmem)
+
+            # define coordinates
+            time_coord = ('time', time)
+            lon_coord = ('lon', lon, {
+                'long_name':'longitude', 'units':'degrees_east', '_CoordinateAxisType':'Lon'})
+            lat_coord = ('lat', lat, {
+                'long_name':'latitude', 'units':'degrees_north', '_CoordinateAxisType':'Lat'})
+            if level[0] != 0:
+                level_coord = ('level', level, levattrs)
+            if nmem != 0:
+                number_coord = ('number', number, {'_CoordinateAxisType':'Ensemble'})
+
+            # create to xarray
+            if data_type == 4:
+                if nmem == 0:
+                    if level[0] == 0:
+                        data = data[np.newaxis, ...]
+                        data = xr.Dataset({
+                            varname:(['time', 'lat', 'lon'], data, varattrs)},
+                            coords={
+                                'time':time_coord, 'lat':lat_coord, 'lon':lon_coord})
+                    else:
+                        data = data[np.newaxis, np.newaxis, ...]
+                        data = xr.Dataset({
+                            varname:(['time', 'level', 'lat', 'lon'], data, varattrs)},
+                            coords={
+                                'time':time_coord, 'level':level_coord, 
+                                'lat':lat_coord, 'lon':lon_coord})
+                else:
+                    if level[0] == 0:
+                        data = data[np.newaxis, ...]
+                        data = xr.Dataset({
+                            varname:(['time', 'number', 'lat', 'lon'], data, varattrs)},
+                            coords={
+                                'time':time_coord, 'number':number_coord, 
+                                'lat':lat_coord, 'lon':lon_coord})
+                    else:
+                        data = data[np.newaxis, :, np.newaxis, ...]
+                        data = xr.Dataset({
+                            varname:(['time', 'number', 'level', 'lat', 'lon'], data, varattrs)},
+                            coords={
+                                'time':time_coord, 'number':number_coord, 'level':level_coord, 
+                                'lat':lat_coord, 'lon':lon_coord})
+            elif data_type == 11:
+                speedattrs = {'long_name':'wind speed', 'units':'m/s'}
+                angleattrs = {'long_name':'wind angle', 'units':'degree'}
+                if nmem == 0:
+                    speed = np.squeeze(data[0, :, :])
+                    angle = np.squeeze(data[1, :, :])
+                    if level[0] == 0:
+                        speed = speed[np.newaxis, ...]
+                        angle = angle[np.newaxis, ...]
+                        data = xr.Dataset({
+                            'speed': (['time', 'lat', 'lon'], speed, speedattrs),
+                            'angle': (['time', 'lat', 'lon'], angle, angleattrs)},
+                            coords={'lon': lon_coord, 'lat': lat_coord, 'time': time_coord})
+                    else:
+                        speed = speed[np.newaxis, np.newaxis, ...]
+                        angle = angle[np.newaxis, np.newaxis, ...]
+                        data = xr.Dataset({
+                            'speed': (['time', 'level', 'lat', 'lon'], speed, speedattrs),
+                            'angle': (['time', 'level', 'lat', 'lon'], angle, angleattrs)},
+                            coords={'lon': lon_coord, 'lat': lat_coord, 
+                                    'level': level_coord, 'time': time_coord})
+                else:
+                    speed = np.squeeze(data[0, :, :, :])
+                    angle = np.squeeze(data[1, :, :, :])
+                    if level[0] == 0:
+                        speed = speed[np.newaxis, ...]
+                        angle = angle[np.newaxis, ...]
+                        data = xr.Dataset({
+                            'speed': (
+                                ['time', 'number', 'lat', 'lon'], speed, speedattrs),
+                            'angle': (
+                                ['time', 'number', 'lat', 'lon'], angle, angleattrs)},
+                            coords={
+                                'lon': lon_coord, 'lat': lat_coord,
+                                'number': number_coord, 'time': time_coord})
+                    else:
+                        speed = speed[np.newaxis, :, np.newaxis, ...]
+                        angle = angle[np.newaxis, :, np.newaxis, ...]
+                        data = xr.Dataset({
+                            'speed': (
+                                ['time', 'number', 'level', 'lat', 'lon'],
+                                speed, speedattrs),
+                            'angle': (
+                                ['time', 'number', 'level', 'lat', 'lon'],
+                                angle, angleattrs)},
+                            coords={
+                                'lon': lon_coord, 'lat': lat_coord, 'level': level_coord,
+                                'number': number_coord, 'time': time_coord})
+            
+            # add time coordinates
+            data.coords['forecast_reference_time'] = init_time[0]
+            data.coords['forecast_period'] = ('time', fhour, {
+                'long_name':'forecast_period', 'units':'hour'})
+
+            # add attributes
+            data.attrs['Conventions'] = "CF-1.6"
+            data.attrs['Origin'] = 'MICAPS Cassandra Server'
+
+            # sort latitude coordinates
+            data = data.loc[{'lat':sorted(data.coords['lat'].values)}]
+
+            # cache data
+            if cache:
+                with open(cache_file, 'wb') as f:
+                    pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+            # return data
+            return data
+
+        else:
+            return None
+    else:
+        return None
+
+
+def get_model_grids(directory, filenames, allExists=True, pbar=False, **kargs):
+    """
+    Retrieve multiple time grids from MICAPS cassandra service.
+    
+    Args:
+        directory (string): the data directory on the service.
+        filenames (list): the list of filenames.
+        allExists (boolean): all files should exist, or return None.
+        pbar (boolean): Show progress bar, default to False.
+        **kargs: key arguments passed to get_model_grid function.
+    """
+
+    dataset = []
+    if pbar:
+        tqdm_filenames = tqdm(filenames, desc=directory + ": ")
+    else:
+        tqdm_filenames = filenames
+    for filename in tqdm_filenames:
+        data = get_model_grid(directory, filename=filename, **kargs)
+        if data:
+            dataset.append(data)
+        else:
+            if allExists:
+                warnings.warn("{} doese not exists.".format(directory+'/'+filename))
+                return None
+    
+    return xr.concat(dataset, dim='time')
+
+
+def get_model_points(directory, filenames, points, **kargs):
+    """
+    Retrieve point time series from MICAPS cassandra service.
+    Return xarray, (time, points)
+    
+    Args:
+        directory (string): the data directory on the service.
+        filenames (list): the list of filenames.
+        points (dict): dictionary, {'lon':[...], 'lat':[...]}.
+        **kargs: key arguments passed to get_model_grids function.
+    Examples:
+    >>> directory = "NWFD_SCMOC/TMP/2M_ABOVE_GROUND"
+    >>> fhours = np.arange(3, 75, 3)
+    >>> filenames = ["19083008."+str(fhour).zfill(3) for fhour in fhours]
+    >>> points = {'lon':[116.3833, 110.0], 'lat':[39.9, 32]}
+    >>> data = get_model_points(dataDir, filenames, points)
+    """
+
+    data = get_model_grids(directory, filenames, **kargs)
+    if data:
+        return data.interp(lon=('points', points['lon']), lat=('points', points['lat']))
+    else:
+        return None
+
+
+def get_model_3D_grid(directory, filename, levels, allExists=True, pbar=False, **kargs):
+    """
+    Retrieve 3D [level, lat, lon] grids from  MICAPS cassandra service.
+    
+    Args:
+        directory (string): the data directory on the service, which includes all levels.
+        filename (string): the data file name.
+        levels (list): the high levels.
+        allExists (boolean): all levels should be exist, if not, return None.
+        pbar (boolean): show progress bar.
+        **kargs: key arguments passed to get_model_grid function.
+    Examples:
+    >>> directory = "ECMWF_HR/TMP"
+    >>> levels = [1000, 950, 925, 900, 850, 800, 700, 600, 500, 400, 300, 250, 200, 100]
+    >>> filename = "19083008.024"
+    >>> data = get_model_3D_grid(directory, filename, levels)
+    """
+
+    dataset = []
+    if pbar:
+        tqdm_levels = tqdm(levels, desc=directory+": ")
+    else:
+        tqdm_levels = levels
+    for level in tqdm_levels:
+        if directory[-1] == '/':
+            dataDir = directory + str(int(level)).strip()
+        else:
+            dataDir = directory + '/' + str(int(level)).strip()
+        data = get_model_grid(dataDir, filename=filename, **kargs)
+        if data:
+                dataset.append(data)
+        else:
+            if allExists:
+                warnings.warn("{} doese not exists.".format(dataDir+'/'+filename))
+                return None
+
+    return xr.concat(dataset, dim='level')
+
+
+def get_model_3D_grids(directory, filenames, levels, allExists=True, pbar=True, **kargs):
+    """
+     Retrieve 3D [time, level, lat, lon] grids from  MICAPS cassandra service.
+    
+    Args:
+        directory (string): the data directory on the service, which includes all levels.
+        filenames (list): the list of data filenames, should be the same initial time.
+        levels (list): the high levels.
+        allExists (bool, optional): all files should exist, or return None.. Defaults to True.
+        pbar (boolean): Show progress bar, default to True.
+        **kargs: key arguments passed to get_model_grid function.
+    Examples:
+    >>> directory = "ECMWF_HR/TMP"
+    >>> levels = [1000, 950, 925, 900, 850, 800, 700, 600, 500, 400, 300, 250, 200, 100]
+    >>> fhours = np.arange(0, 75, 3)
+    >>> filenames = ["19083008."+str(fhour).zfill(3) for fhour in fhours]
+    >>> data =  get_model_3D_grids(directory, filenames, levels)
+    """
+
+    dataset = []
+    if pbar:
+        tqdm_filenames = tqdm(filenames, desc=directory+": ")
+    else:
+        tqdm_filenames = filenames
+    for filename in tqdm_filenames:
+        dataset_temp = []
+        for level in levels:
+            if directory[-1] == '/':
+                dataDir = directory + str(int(level)).strip()
+            else:
+                dataDir = directory + '/' + str(int(level)).strip()
+            data = get_model_grid(dataDir, filename=filename, **kargs)
+            if data:
+                    dataset_temp.append(data)
+            else:
+                if allExists:
+                    warnings.warn("{} doese not exists.".format(dataDir+'/'+filename))
+                    return None
+        dataset.append(xr.concat(dataset_temp, dim='level'))
+    
+    return xr.concat(dataset, dim='time')
+
+
+def get_model_profiles(directory, filenames, levels, points, **kargs):
+    """
+    Retrieve time series of vertical profile from 3D [time, level, lat, lon] grids from  MICAPS cassandra service.
+    
+    Args:
+        directory (string): the data directory on the service, which includes all levels.
+        filenames (list): the list of data filenames or one file.
+        levels (list): the high levels.
+        points (dict): dictionary, {'lon':[...], 'lat':[...]}.
+        **kargs: key arguments passed to get_model_3D_grids function.
+    Examples:
+      directory = "ECMWF_HR/TMP"
+      levels = [1000, 950, 925, 900, 850, 800, 700, 600, 500, 400, 300, 250, 200, 100]
+      filenames = ["20021320.024"]
+      points = {'lon':[116.3833, 110.0], 'lat':[39.9, 32]}
+      data = get_model_profiles(directory, filenames, levels, points)
+    """
+
+    data = get_model_3D_grids(directory, filenames, levels, **kargs)
+    if data:
+        return data.interp(lon=('points', points['lon']), lat=('points', points['lat']))
+    else:
+        return None
+
+
+def get_station_data(directory, filename=None, suffix="*.000", dropna=True, cache=True):
+    """
+    Retrieve station data from MICAPS cassandra service.
+    :param directory: the data directory on the service
+    :param filename: the data filename, if none, will be the latest file.
+    :param suffix: the filename filter pattern which will
+                   be used to find the specified file.
+    :param dropna: the column which values is all na will be dropped.
+    :param cache: cache retrieved data to local directory, default is True.
+    :return: pandas DataFrame.
+    :example:
+    >>> data = get_station_data("SURFACE/PLOT_10MIN")
+    >>> data = get_station_data("SURFACE/TMP_MAX_24H_NATIONAL", filename="20190705150000.000")
+    """
+
+    # get data file name
+    if filename is None:
+        try:
+            # connect to data service
+            service = GDSDataService()
+            if service.berror == True:
+                return None
+            status, response = service.getLatestDataName(directory, suffix)
+        except ValueError:
+            print('Can not retrieve data from ' + directory)
+            return None
+        StringResult = DataBlock_pb2.StringResult()
+        if status == 200:
+            StringResult.ParseFromString(response)
+            if StringResult is not None:
+                filename = StringResult.name
+                if filename == '':
+                    return None
+            else:
+                return None
+
+    # retrieve data from cached file
+    if cache:
+        cache_file = CONFIG.get_cache_file(directory, filename, name="MICAPS_DATA")
+        if cache_file.is_file():
+            with open(cache_file, 'rb') as f:
+                data = pickle.load(f)
+                return data
+
+    # get data contents
+    try:
+        service = GDSDataService()
+        if service.berror == True:
+            return None
+        status, response = service.getData(directory, filename)
+    except ValueError:
+        print('Can not retrieve data' + filename + ' from ' + directory)
+        return None
+    ByteArrayResult = DataBlock_pb2.ByteArrayResult()
+    if status == 200:
+        ByteArrayResult.ParseFromString(response)
+        if ByteArrayResult is not None:
+            byteArray = ByteArrayResult.byteArray
+
+            # define head structure
+            head_dtype = [('discriminator', 'S4'), ('type', 'i2'),
+                          ('description', 'S100'),
+                          ('level', 'f4'), ('levelDescription', 'S50'),
+                          ('year', 'i4'), ('month', 'i4'), ('day', 'i4'),
+                          ('hour', 'i4'), ('minute', 'i4'), ('second', 'i4'),
+                          ('Timezone', 'i4'), ('extent', 'S100')]
+
+            # read head information
+            head_info = np.frombuffer(byteArray[0:288], dtype=head_dtype)
+            ind = 288
+            # zwj add 20210830
+            if head_info.size==0:
+                return None
+            # read the number of stations
+            station_number = np.frombuffer(
+                byteArray[ind:(ind+4)], dtype='i4')[0]
+            ind += 4
+
+            # read the number of elements
+            element_number = np.frombuffer(
+                byteArray[ind:(ind+2)], dtype='i2')[0]
+            ind += 2
+
+            # construct record structure
+            element_type_map = {
+                1: 'b1', 2: 'i2', 3: 'i4', 4: 'i8', 5: 'f4', 6: 'f8', 7: 'S'}
+            element_map = {}
+            for i in range(element_number):
+                element_id = str(
+                    np.frombuffer(byteArray[ind:(ind+2)], dtype='i2')[0])
+                ind += 2
+                element_type = np.frombuffer(
+                    byteArray[ind:(ind+2)], dtype='i2')[0]
+                ind += 2
+                element_map[element_id] = element_type_map[element_type]
+
+            # loop every station to retrieve record
+            record_head_dtype = [
+                ('ID', 'i4'), ('lon', 'f4'), ('lat', 'f4'), ('numb', 'i2')]
+            records = []
+            # zwj add 20210830
+            if station_number==0:
+                return None
+
+            for i in range(station_number):
+                record_head = np.frombuffer(
+                    byteArray[ind:(ind+14)], dtype=record_head_dtype)
+                ind += 14
+                record = {
+                    'ID': record_head['ID'][0], 'lon': record_head['lon'][0],
+                    'lat': record_head['lat'][0]}
+                for j in range(record_head['numb'][0]):    # the record element number is not same, missing value is not included.
+                    element_id = str(
+                        np.frombuffer(byteArray[ind:(ind + 2)], dtype='i2')[0])
+                    ind += 2
+                    element_type = element_map[element_id]
+                    if element_type == 'S':                # if the element type is string, we need get the length of string
+                        str_len = np.frombuffer(byteArray[ind:(ind + 2)], dtype='i2')[0]
+                        ind += 2
+                        element_type = element_type + str(str_len)
+                    element_len = int(element_type[1:])
+                    record[element_id] = np.frombuffer(
+                        byteArray[ind:(ind + element_len)],
+                        dtype=element_type)[0]
+                    ind += element_len
+                records += [record]
+
+            # convert to pandas data frame
+            records = pd.DataFrame(records)
+            records.set_index('ID')
+
+            # get time
+            time = datetime(
+                head_info['year'][0], head_info['month'][0],
+                head_info['day'][0], head_info['hour'][0],
+                head_info['minute'][0], head_info['second'][0])
+            records['time'] = time
+
+            # change column name for common observation
+            cur_dict=dict([(str(gds_element_id_dict[key]),key) for  key in gds_element_id_dict.keys()])
+
+            records.rename(columns=cur_dict,inplace=True)
+            # records.rename(columns={'3': 'Alt', '4': 'Grade', '5': 'Type', '21': 'Name',
+            #     '201': 'Wind_angle', '203': 'Wind_speed', '205': 'Wind_angle_1m_avg', '207': 'Wind_speed_1m_avg',
+            #     '209': 'Wind_angle_2m_avg', '211': 'Wind_speed_2m_avg', '213': 'Wind_angle_10m_avg', '215': 'Wind_speed_10m_avg',
+            #     '217': 'Wind_angle_max', '219': 'Wind_speed_max', '221': 'Wind_angle_instant', '223': 'Wind_speed_instant',
+            #     '225': 'Gust_angle', '227': 'Gust_speed', '229': 'Gust_angle_6h', '231': 'Gust_speed_6h',
+            #     '233': 'Gust_angle_12h', '235': 'Gust_speed_12h', '237': 'Wind_power', 
+            #     '401': 'Sea_level_pressure', '403': 'Pressure_3h_trend', '405': 'Pressure_24h_trend',
+            #     '407': 'Station_pressure', '409': 'Pressure_max', '411': 'Pressure_min', '413': 'Pressure',
+            #     '415': 'Pressure_day_avg', '417': 'SLP_day_avg', '419': 'Hight', '421': 'Geopotential_hight',
+            #     '601': 'Temp', '603': 'Temp_max', '605': 'Temp_min', '607': 'Temp_24h_trend', 
+            #     '609': 'Temp_24h_max', '611':'Temp_24h_min', '613': 'Temp_dav_avg',
+            #     '801': 'Dewpoint', '803': 'Dewpoint_depression', '805': 'Relative_humidity',
+            #     '807': 'Relative_humidity_min', '809': 'Relative_humidity_day_avg', 
+            #     '811': 'Water_vapor_pressure', '813': 'Water_vapor_pressure_day_avg',
+            #     '1001': 'Rain', '1003': 'Rain_1h', '1005': 'Rain_3h', '1007': 'Rain_6h', '1009': 'Rain_12h', '1013': 'Rain_day',
+            #     '1015': 'Rain_20-08', '1017': 'Rain_08-20', '1019': 'Rain_20-20', '1021': 'Rain_08-08',
+            #     '1023': 'Evaporation', '1025': 'Evaporation_large', '1027': 'Precipitable_water',
+            #     '1201': 'Vis_1min', '1203': 'Vis_10min', '1205': 'Vis_min', '1207': 'Vis_manual',
+            #     '1401': 'Total_cloud_cover', '1403': 'Low_cloud_cover', '1405': 'Cloud_base_hight',
+            #     '1407': 'Low_cloud', '1409': 'Middle_cloud', '1411': 'High_cloud',
+            #     '1413': 'TCC_day_avg', '1415': 'LCC_day_avg', '1417': 'Cloud_cover', '1419': 'Cloud_type',
+            #     '1601': 'Weather_current', '1603': 'Weather_past_1', '1605': 'Weather_past_2',
+            #     '2001': 'Surface_temp', '2003': 'Surface_temp_max', '2005': 'Surface_temp_min'},
+            #     inplace=True)
+
+            # drop all NaN columns
+            if dropna:
+                records = records.dropna(axis=1, how='all')
+
+            # cache records
+            if cache:
+                with open(cache_file, 'wb') as f:
+                    pickle.dump(records, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+            # return
+            print(filename)
+            return records
+        else:
+            return None
+    else:
+        return None
+
+
+def get_station_dataset(directory, filenames, allExists=True, pbar=False, **kargs):
+    """
+    Retrieve multiple station observation from MICAPS cassandra service.
+    
+    Args:
+        directory (string): the data directory on the service.
+        filenames (list): the list of filenames.
+        allExists (boolean): all files should exist, or return None.
+        pbar (boolean): Show progress bar, default to False.
+        **kargs: key arguments passed to get_fy_awx function.
+    """
+
+    dataset = []
+    if pbar:
+        tqdm_filenames = tqdm(filenames, desc=directory + ": ")
+    else:
+        tqdm_filenames = filenames
+    for filename in filenames:
+        data = get_station_data(directory, filename=filename, **kargs)
+        if data:
+            dataset.append(data)
+        else:
+            if allExists:
+                warnings.warn("{} doese not exists.".format(directory+'/'+filename))
+                return None
+    
+    return pd.concat(dataset)
+
+# zwj 20210311
+def get_fy_awx_zwj(directory, filename=None, suffix="*.AWX", units='', cache=True):
+    """
+    Retrieve FY satellite cloud awx format file.
+    The awx file format is refered to “气象卫星分发产品及其格式规范AWX2.1”
+    http://satellite.nsmc.org.cn/PortalSite/StaticContent/DocumentDownload.aspx?TypeID=10
+    :param directory: the data directory on the service
+    :param filename: the data filename, if none, will be the latest file.
+    :param suffix: the filename filter pattern which will be used to
+                   find the specified file.
+    :param units: data units, default is ''.
+    :param cache: cache retrieved data to local directory, default is True.
+    :return: satellite information and data.
+    :Examples:
+    >>> directory = "SATELLITE/FY2E/L1/IR1/EQUAL"
+    >>> data = get_fy_awx(directory)
+    """
+
+    service = GDSDataService()
+    if service.berror == True:
+        return None
+    status, response = service.getData(directory, filename)
+    
+    ByteArrayResult = DataBlock_pb2.ByteArrayResult()
+    if status == 200:
+        ByteArrayResult.ParseFromString(response)
+        if ByteArrayResult is not None:
+            byteArray = ByteArrayResult.byteArray
+            if byteArray == b'':
+                print('There is no data ' + filename + ' in ' + directory)
+                return None
+
+            return byteArray
+
+def get_stadata_from_mdfs(filename=None):
+    
+    if not os.path.exists(filename):
+        print(filename + ' :file not exist!')
+        return None
+
+    fin = open(filename,'rb')
+    byteArray = fin.read()
+    # byteArray = ByteArrayResult.byteArray
+
+    # define head structure
+    head_dtype = [('discriminator', 'S4'), ('type', 'i2'),
+                    ('description', 'S100'),
+                    ('level', 'f4'), ('levelDescription', 'S50'),
+                    ('year', 'i4'), ('month', 'i4'), ('day', 'i4'),
+                    ('hour', 'i4'), ('minute', 'i4'), ('second', 'i4'),
+                    ('Timezone', 'i4'), ('extent', 'S100')]
+
+    # read head information
+    head_info = np.frombuffer(byteArray[0:288], dtype=head_dtype)
+    ind = 288
+    # zwj add 20210830
+    if head_info.size==0:
+        return None
+    # read the number of stations
+    station_number = np.frombuffer(
+        byteArray[ind:(ind+4)], dtype='i4')[0]
+    ind += 4
+
+    # read the number of elements
+    element_number = np.frombuffer(
+        byteArray[ind:(ind+2)], dtype='i2')[0]
+    ind += 2
+
+    # construct record structure
+    element_type_map = {
+        1: 'b1', 2: 'i2', 3: 'i4', 4: 'i8', 5: 'f4', 6: 'f8', 7: 'S'}
+    element_map = {}
+    for i in range(element_number):
+        element_id = str(
+            np.frombuffer(byteArray[ind:(ind+2)], dtype='i2')[0])
+        ind += 2
+        element_type = np.frombuffer(
+            byteArray[ind:(ind+2)], dtype='i2')[0]
+        ind += 2
+        element_map[element_id] = element_type_map[element_type]
+
+    # loop every station to retrieve record
+    record_head_dtype = [
+        ('ID', 'i4'), ('lon', 'f4'), ('lat', 'f4'), ('numb', 'i2')]
+    records = []
+    # zwj add 20210830
+    if station_number==0:
+        return None
+
+    for i in range(station_number):
+        record_head = np.frombuffer(
+            byteArray[ind:(ind+14)], dtype=record_head_dtype)
+        ind += 14
+        record = {
+            'ID': record_head['ID'][0], 'lon': record_head['lon'][0],
+            'lat': record_head['lat'][0]}
+        for j in range(record_head['numb'][0]):    # the record element number is not same, missing value is not included.
+            element_id = str(
+                np.frombuffer(byteArray[ind:(ind + 2)], dtype='i2')[0])
+            ind += 2
+            element_type = element_map[element_id]
+            if element_type == 'S':                # if the element type is string, we need get the length of string
+                str_len = np.frombuffer(byteArray[ind:(ind + 2)], dtype='i2')[0]
+                ind += 2
+                element_type = element_type + str(str_len)
+            element_len = int(element_type[1:])
+            record[element_id] = np.frombuffer(
+                byteArray[ind:(ind + element_len)],
+                dtype=element_type)[0]
+            ind += element_len
+        records += [record]
+
+    # convert to pandas data frame
+    records = pd.DataFrame(records)
+    records.set_index('ID')
+
+    # get time
+    time = datetime(
+        head_info['year'][0], head_info['month'][0],
+        head_info['day'][0], head_info['hour'][0],
+        head_info['minute'][0], head_info['second'][0])
+    records['time'] = time
+
+    # change column name for common observation
+    cur_dict=dict([(str(gds_element_id_dict[key]),key) for  key in gds_element_id_dict.keys()])
+
+    records.rename(columns=cur_dict,inplace=True)
+    
+    # drop all NaN columns
+    # if dropna:
+    #     records = records.dropna(axis=1, how='all')
+
+    return records
+
+def get_fy_awx(directory, filename=None, suffix="*.AWX", units='', cache=True):
+    """
+    Retrieve FY satellite cloud awx format file.
+    The awx file format is refered to “气象卫星分发产品及其格式规范AWX2.1”
+    http://satellite.nsmc.org.cn/PortalSite/StaticContent/DocumentDownload.aspx?TypeID=10
+    :param directory: the data directory on the service
+    :param filename: the data filename, if none, will be the latest file.
+    :param suffix: the filename filter pattern which will be used to
+                   find the specified file.
+    :param units: data units, default is ''.
+    :param cache: cache retrieved data to local directory, default is True.
+    :return: satellite information and data.
+    :Examples:
+    >>> directory = "SATELLITE/FY2E/L1/IR1/EQUAL"
+    >>> data = get_fy_awx(directory)
+    """
+
+    # get data file name
+    if filename is None:
+        try:
+            # connect to data service
+            service = GDSDataService()
+            if service.berror == True:
+                return None
+            status, response = service.getLatestDataName(directory, suffix)
+        except ValueError:
+            print('Can not retrieve data from ' + directory)
+            return None
+        StringResult = DataBlock_pb2.StringResult()
+        if status == 200:
+            StringResult.ParseFromString(response)
+            if StringResult is not None:
+                filename = StringResult.name
+                if filename == '':
+                    return None
+            else:
+                return None
+
+    # retrieve data from cached file
+    if cache:
+        cache_file = CONFIG.get_cache_file(directory, filename, name="MICAPS_DATA")
+        if cache_file.is_file():
+            with open(cache_file, 'rb') as f:
+                data = pickle.load(f)
+                return data
+
+    # get data contents
+    try:
+        service = GDSDataService()
+        if service.berror == True:
+            return None
+        status, response = service.getData(directory, filename)
+    except ValueError:
+        print('Can not retrieve data' + filename + ' from ' + directory)
+        return None
+    ByteArrayResult = DataBlock_pb2.ByteArrayResult()
+    if status == 200:
+        ByteArrayResult.ParseFromString(response)
+        if ByteArrayResult is not None:
+            byteArray = ByteArrayResult.byteArray
+            if byteArray == b'':
+                print('There is no data ' + filename + ' in ' + directory)
+                return None
+            CODE1 = 'B'
+            CODE2 = 'H'
+            INT1 = 'B'
+            INT2 = 'H'
+            INT4 = 'I'
+            REAL4 = 'f'
+            REAL8 = 'd'
+            SINT1 = 'b'
+            SINT2 = 'h'
+            SINT4 = 'i'
+            # the first class file head
+            head1_dtype = [
+                ('SAT96', '12s'),                    # SAT96 filename
+                ('byteSequence', SINT2),              # 整型数的字节顺序, 0 低字节在前, 高字节在后; !=0 高字节在前, 低字节在后.
+                ('firstClassHeadLength', SINT2),      # 第一节文件头长度
+                ('secondClassHeadLength', SINT2),     # 第二节文件头长度
+                ('padDataLength', SINT2),             # 填充段数据长度
+                ('recordLength', SINT2),              # 记录长度(字节), 图像产品: 记录长度=图形宽度, 格点场产品: 记录长度=横向格点数x格点数据字长
+                ('headRecordNumber', SINT2),          # 文件头占用记录数, 一级文件头、二填充段扩展以及的所占用总记录个数
+                ('dataRecordNumber', SINT2),          # 产品数据占用记录数
+                ('productCategory', SINT2),           # 产品类别, 1：静止, 2：极轨, 3：格点定量, 4：离散, 5: 图形和分析
+                ('compressMethod', SINT2),            # 压缩方式, 0: 未压缩; 1 行程编码压缩; 2 LZW方式压缩; 3 特点方式压缩
+                ('formatString', '8s'),              # 格式说明字符串, 'SAT2004'
+                ('qualityFlag', SINT2)]               # 产品数据质量标记, 1 完全可靠; 2 基本可靠; 3 有缺值, 可用; 4 不可用
+            # head1_info = np.frombuffer(byteArray[0:40], dtype=head1_dtype)
+            head1_info = _unpack_from_buf(byteArray,0, head1_dtype)   # zwj modified 20210311
+            head1_info['SAT96'] = 'SAT2004      '
+            head1_info['formatString'] = 'SAT2004 '
+
+            ind = 40
+
+            if head1_info['productCategory']:
+                # the second class file head
+                head2_dtype = [
+                    ('satelliteName', '8s'),                 # 卫星名
+                    ('year', SINT2), ('month', SINT2),
+                    ('day', SINT2), ('hour', SINT2),
+                    ('minute', SINT2),
+                    ('channel', SINT2),                       # 通道号, 1红外, 2水汽, 3红外分裂, 4可见光, 5中红外, 6备用
+                    ('flagOfProjection', SINT2),              # 投影, 0为投影, 1兰勃托, 2麦卡托, 3极射, 4等经纬度, 5等面积
+                    ('widthOfImage', SINT2),
+                    ('heightOfImage', SINT2),
+                    ('scanLineNumberOfImageTopLeft', SINT2),
+                    ('pixelNumberOfImageTopLeft', SINT2),
+                    ('sampleRatio', SINT2),
+                    ('latitudeOfNorth', SINT2),
+                    ('latitudeOfSouth', SINT2),
+                    ('longitudeOfWest', SINT2),
+                    ('longitudeOfEast', SINT2),
+                    ('centerLatitudeOfProjection', SINT2),
+                    ('centerLongitudeOfProjection', SINT2),
+                    ('standardLatitude1', SINT2),
+                    ('standardLatitude2', SINT2),
+                    ('horizontalResolution', SINT2),
+                    ('verticalResolution', SINT2),
+                    ('overlapFlagGeoGrid', SINT2),
+                    ('overlapValueGeoGrid', SINT2),
+                    ('dataLengthOfColorTable', SINT2),
+                    ('dataLengthOfCalibration', SINT2),
+                    ('dataLengthOfGeolocation', SINT2),
+                    ('reserved', SINT2)]
+
+                # head2_info = np.frombuffer(byteArray[ind:(ind+64)], dtype=head2_dtype)
+                head2_info = _unpack_from_buf(byteArray,ind, head2_dtype)   # zwj modified 20210311
+                head2_info['satelliteName'] = 'SAT2004 '
+
+
+                ind += 64
+
+                # color table
+                if head2_info['dataLengthOfColorTable'] != 0:
+                    table_R =  np.frombuffer(byteArray[ind:(ind + 256)], dtype='u1')
+                    ind += 256
+                    table_G =  np.frombuffer(byteArray[ind:(ind + 256)], dtype='u1')
+                    ind += 256
+                    table_B =  np.frombuffer(byteArray[ind:(ind + 256)], dtype='u1')
+                    ind += 256
+                
+                # calibration table
+                calibration_table = None
+                if head2_info['dataLengthOfCalibration'] != 0:
+                    calibration_table = np.frombuffer(byteArray[ind:(ind + 2048)], dtype='i2')
+                    calibration_table = calibration_table * 0.01
+                    if (np.array_equal(calibration_table[0::4], calibration_table[1::4]) and
+                        np.array_equal(calibration_table[0::4], calibration_table[2::4]) and
+                        np.array_equal(calibration_table[0::4], calibration_table[3::4])):
+                        # This is a trick, refer to http://bbs.06climate.com/forum.php?mod=viewthread&tid=89296
+                        calibration_table = calibration_table[0::4]
+                    ind += 2048
+
+                # geolocation table
+                if head2_info['dataLengthOfGeolocation'] != 0:
+                    geolocation_dtype = [
+                         ('coordinate', SINT2),
+                         ('source', SINT2),
+                         ('delta', SINT2),
+                         ('left_top_lat', SINT2),
+                         ('left_top_lon', SINT2),
+                         ('horizontalNumber', SINT2),
+                         ('verticalNumber', SINT2),
+                         ('reserved', SINT2)]
+                    # geolocation_info = np.frombuffer(byteArray[ind:(ind+16)], dtype=geolocation_dtype)
+                    geolocation_info = _unpack_from_buf(byteArray,ind, geolocation_info)   # zwj modified 20210311
+                    ind += 16
+                    geolocation_length = geolocation_info['horizontal_number'] * geolocation_info['vertical_number'] * 2
+                    geolocation_table = np.frombuffer(byteArray[ind:(ind+geolocation_length)], dtype='i2')
+                    ind += geolocation_length
+
+                # pad field
+                pad_field = np.frombuffer(byteArray[ind:(ind+head1_info['padDataLength'])], dtype='u1')
+                ind += head1_info['padDataLength']
+
+                 # retrieve data records
+                data_len = (int(head1_info['dataRecordNumber'])*
+                            head1_info['recordLength'])
+                data = np.frombuffer(byteArray[ind:(ind + data_len)], dtype='u1', count=data_len)
+                if calibration_table is not None:
+                    data = calibration_table[data]
+                data.shape = (head1_info['dataRecordNumber'], head1_info['recordLength'])
+
+                    # construct longitude and latitude coordinates
+                lat = (
+                    head2_info['latitudeOfNorth']/100. - 
+                    np.arange(head2_info['heightOfImage'])*head2_info['verticalResolution']/100.)
+                lon = (
+                    head2_info['longitudeOfWest']/100. + 
+                    np.arange(head2_info['widthOfImage'])*head2_info['horizontalResolution']/100.)
+                
+                # construct time
+                time = datetime(
+                    head2_info['year'], head2_info['month'],
+                    head2_info['day'], head2_info['hour'], head2_info['minute'])
+                time = np.array([time], dtype='datetime64[ms]')
+
+                # define coordinates
+                time_coord = ('time', time)
+                lon_coord = ('lon', lon, {
+                    'long_name':'longitude', 'units':'degrees_east', '_CoordinateAxisType':'Lon'})
+                lat_coord = ('lat', lat, {
+                    'long_name':'latitude', 'units':'degrees_north', '_CoordinateAxisType':'Lat'})
+                channel_coord = ('channel', head2_info['channel'], {'long_name':'channel', 'units':''})
+
+                # create xarray
+                data = data[np.newaxis, ...]
+                varattrs = {
+                    'productCategory': head1_info['productCategory'],   # 产品类型, 1:静止, 2:极轨, 3:格点, 4:离散, 5:图形和分析
+                    'formatString': head1_info['formatString'],         # 产品格式名称
+                    'qualityFlag': head1_info['qualityFlag'],           # 产品质量标识
+                    'satelliteName': head2_info['satelliteName'],       # 卫星名称
+                    'flagOfProjection': head2_info['flagOfProjection'], # 投影方式, 0:未投影, 1:兰勃托, 2:麦卡托, 3:极射, 4:等经纬, 5:等面积
+                    'units': units}
+                data = xr.Dataset({
+                    'image':(['time', 'lat', 'lon'], data, varattrs)},
+                    coords={ 'time':time_coord, 'lat':lat_coord, 'lon':lon_coord})
+
+                # add attributes
+                data.attrs['Conventions'] = "CF-1.6"
+                data.attrs['Origin'] = 'MICAPS Cassandra Server'
+
+                # cache data
+                if cache:
+                    with open(cache_file, 'wb') as f:
+                        pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+                # return
+                return data
+            else:
+                print("The productCategory is not supported.")
+                return None
+        else:
+            return None
+    else:
+        return None
+
+
+def get_fy_awxs(directory, filenames, allExists=True, pbar=False, **kargs):
+    """
+    Retrieve multiple satellite images from MICAPS cassandra service.
+    
+    Args:
+        directory (string): the data directory on the service.
+        filenames (list): the list of filenames.
+        allExists (boolean): all files should exist, or return None.
+        pbar (boolean): Show progress bar, default to False.
+        **kargs: key arguments passed to get_fy_awx function.
+    """
+
+    dataset = []
+    if pbar:
+        tqdm_filenames = tqdm(filenames, desc=directory + ": ")
+    else:
+        tqdm_filenames = filenames
+    for filename in tqdm_filenames:
+        data = get_fy_awx(directory, filename=filename, **kargs)
+        if data:
+            dataset.append(data)
+        else:
+            if allExists:
+                warnings.warn("{} doese not exists.".format(directory+'/'+filename))
+                return None
+    
+    return xr.concat(dataset, dim='time')
+
+
+def get_radar_mosaic(directory, filename=None, suffix="*.BIN", cache=True):
+    """
+    该程序主要用于读取和处理中国气象局CRaMS系统的雷达回波全国拼图数据.
+    :param directory: the data directory on the service
+    :param filename: the data filename, if none, will be the latest file.
+    :param suffix: the filename filter pattern which will be used to
+                   find the specified file.
+    :param cache: cache retrieved data to local directory, default is True.
+    :return: xarray object.
+    :Example:
+    >>> data = get_radar_mosaic("RADARMOSAIC/CREF/")
+    """
+
+    # get data file name
+    if filename is None:
+        try:
+            # connect to data service
+            service = GDSDataService()
+            if service.berror == True:
+                return None
+            status, response = service.getLatestDataName(directory, suffix)
+        except ValueError:
+            print('Can not retrieve data from ' + directory)
+            return None
+        StringResult = DataBlock_pb2.StringResult()
+        if status == 200:
+            StringResult.ParseFromString(response)
+            if StringResult is not None:
+                filename = StringResult.name
+                if filename == '':
+                    return None
+            else:
+                return None
+
+    # retrieve data from cached file
+    if cache:
+        cache_file = CONFIG.get_cache_file(directory, filename, name="MICAPS_DATA")
+        if cache_file.is_file():
+            with open(cache_file, 'rb') as f:
+                data = pickle.load(f)
+                return data
+
+    # get data contents
+    try:
+        service = GDSDataService()
+        if service.berror == True:
+            return None
+        status, response = service.getData(directory, filename)
+    except ValueError:
+        print('Can not retrieve data' + filename + ' from ' + directory)
+        return None
+    ByteArrayResult = DataBlock_pb2.ByteArrayResult()
+    if status == 200:
+        ByteArrayResult.ParseFromString(response)
+        if ByteArrayResult is not None:
+            byteArray = ByteArrayResult.byteArray
+            if byteArray == b'':
+                print('There is no data ' + filename + ' in ' + directory)
+                return None
+            return byteArray
+            # # define head structure
+            # head_dtype = [
+            #     ('description', 'S128'),
+            #     # product name,  QREF=基本反射率, CREF=组合反射率,
+            #     # VIL=液态水含量, OHP=一小时降水
+            #     ('name', 'S32'),
+            #     ('organization', 'S16'),
+            #     ('grid_flag', 'u2'),  # 经纬网格数据标识，固定值19532
+            #     ('data_byte', 'i2'),  # 数据单元字节数，固定值2
+            #     ('slat', 'f4'),       # 数据区的南纬（度）
+            #     ('wlon', 'f4'),       # 数据区的西经（度）
+            #     ('nlat', 'f4'),       # 数据区的北纬（度）
+            #     ('elon', 'f4'),       # 数据区的东经（度）
+            #     ('clat', 'f4'),       # 数据区中心纬度（度）
+            #     ('clon', 'f4'),       # 数据区中心经度（度）
+            #     ('rows', 'i4'),       # 数据区的行数
+            #     ('cols', 'i4'),       # 每行数据的列数
+            #     ('dlat', 'f4'),       # 纬向分辨率（度）
+            #     ('dlon', 'f4'),       # 经向分辨率（度）
+            #     ('nodata', 'f4'),     # 无数据区的编码值
+            #     ('levelbybtes', 'i4'),  # 单层数据字节数
+            #     ('levelnum', 'i2'),   # 数据层个数
+            #     ('amp', 'i2'),        # 数值放大系数
+            #     ('compmode', 'i2'),   # 数据压缩存储时为1，否则为0
+            #     ('dates', 'u2'),      # 数据观测时间，为1970年1月1日以来的天数
+            #     ('seconds', 'i4'),    # 数据观测时间的秒数
+            #     ('min_value', 'i2'),  # 放大后的数据最小取值
+            #     ('max_value', 'i2'),  # 放大后的数据最大取值
+            #     ('reserved', 'i2', 6)  # 保留字节
+            # ]
+
+            # # read head information
+            # head_info = np.frombuffer(byteArray[0:256], dtype=head_dtype)
+            # ind = 256
+
+            # # get data information
+            # varname = head_info['name'][0].decode("utf-8", 'ignore').rsplit('\x00')[0]
+            # longname = {'CREF': 'Composite Reflectivity', 'QREF': 'Basic Reflectivity',
+            #             'VIL': 'Vertically Integrated Liquid', 'OHP': 'One Hour Precipitation'}
+            # units = head_info['organization'][0].decode("utf-8", 'ignore').rsplit('\x00')[0]
+            # amp = head_info['amp'][0]
+
+            # # define data variable
+            # rows = head_info['rows'][0]
+            # cols = head_info['cols'][0]
+            # dlat = head_info['dlat'][0]
+            # dlon = head_info['dlon'][0]
+            # data = np.full(rows*cols, -9999, dtype=np.int32)
+
+            # # put data into array
+            # while ind < len(byteArray):
+            #     irow = np.frombuffer(byteArray[ind:(ind + 2)], dtype='i2')[0]
+            #     ind += 2
+            #     icol = np.frombuffer(byteArray[ind:(ind + 2)], dtype='i2')[0]
+            #     ind += 2
+            #     if irow == -1 or icol == -1:
+            #         break
+            #     nrec = np.frombuffer(byteArray[ind:(ind + 2)], dtype='i2')[0]
+            #     ind += 2
+            #     recd = np.frombuffer(
+            #         byteArray[ind:(ind + 2*nrec)], dtype='i2', count=nrec)
+            #     ind += 2*nrec
+            #     position = (irow-1)*cols+icol-1
+            #     data[position:(position+nrec)] = recd
+
+            # # reshape data
+            # data.shape = (rows, cols)
+
+            # # deal missing data and restore values
+            # data = data.astype(np.float32)
+            # data[data < 0] = np.nan
+            # data /= amp
+
+            # # set longitude and latitude coordinates
+            # lat = head_info['nlat'][0] - np.arange(rows)*dlat - dlat/2.0
+            # lon = head_info['wlon'][0] + np.arange(cols)*dlon - dlon/2.0
+
+            # # reverse latitude axis
+            # data = np.flip(data, 0)
+            # lat = lat[::-1]
+
+            # # set time coordinates
+            # time = datetime(1970, 1, 1) + timedelta(
+            #     days=head_info['dates'][0].astype(np.float64),
+            #     seconds=head_info['seconds'][0].astype(np.float64))
+            # time = np.array([time], dtype='datetime64[m]')
+            # data = np.expand_dims(data, axis=0)
+
+            # # define coordinates
+            # time_coord = ('time', time)
+            # lon_coord = ('lon', lon, {
+            #     'long_name':'longitude', 'units':'degrees_east', '_CoordinateAxisType':'Lon'})
+            # lat_coord = ('lat', lat, {
+            #     'long_name':'latitude', 'units':'degrees_north', '_CoordinateAxisType':'Lat'})
+
+            # # create xarray
+            # varattrs = {'long_name': longname.get(varname, 'radar mosaic'), 
+            #             'short_name': varname, 'units': units}
+            # data = xr.Dataset({'data':(['time', 'lat', 'lon'], data, varattrs)},
+            #     coords={'time':time_coord, 'lat':lat_coord, 'lon':lon_coord})
+
+            # # add attributes
+            # data.attrs['Conventions'] = "CF-1.6"
+            # data.attrs['Origin'] = 'MICAPS Cassandra Server'
+
+            # # cache data
+            # if cache:
+            #     with open(cache_file, 'wb') as f:
+            #         pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+            # # return
+            # return data
+        else:
+            return None
+    else:
+        return None
+
+# 朱文剑 获取标准格式雷达基数据产品
+def get_radar_fmt(directory, filename=None, suffix="*.BZ2", cache=True):
+    """
+    该程序主要用于读取标准格式的FMT雷达基数据.
+    :param directory: the data directory on the service
+    :param filename: the data filename, if none, will be the latest file.
+    :param suffix: the filename filter pattern which will be used to
+                   find the specified file.
+    :param cache: cache retrieved data to local directory, default is True.
+    :return: xarray object.
+    :Example:
+    >>> data = get_radar_fmt("SINGLERADAR/ARCHIVES/PRE_QC/北京大兴")
+    """
+
+    # get data file name
+    if filename is None:
+        try:
+            # connect to data service
+            service = GDSDataService()
+            if service.berror == True:
+                return None
+            status, response = service.getLatestDataName(directory, suffix)
+        except ValueError:
+            print('Can not retrieve data from ' + directory)
+            return None
+        StringResult = DataBlock_pb2.StringResult()
+        if status == 200:
+            StringResult.ParseFromString(response)
+            if StringResult is not None:
+                filename = StringResult.name
+                if filename == '':
+                    return None
+            else:
+                return None
+
+    # retrieve data from cached file
+    if cache:
+        cache_file = CONFIG.get_cache_file(directory, filename, name="MICAPS_DATA")
+        if cache_file.is_file():
+            with open(cache_file, 'rb') as f:
+                data = pickle.load(f)
+                return data
+
+    # get data contents
+    try:
+        service = GDSDataService()
+        if service.berror == True:
+            return None
+        status, response = service.getData(directory, filename)
+    except ValueError:
+        print('Can not retrieve data' + filename + ' from ' + directory)
+        return None
+    ByteArrayResult = DataBlock_pb2.ByteArrayResult()
+    if status == 200:
+        ByteArrayResult.ParseFromString(response)
+        if ByteArrayResult is not None:
+            byteArray = ByteArrayResult.byteArray
+            if byteArray == b'':
+                print('There is no data ' + filename + ' in ' + directory)
+                return None
+
+            return byteArray
+            
+        else:
+            return None
+    else:
+        return None
+
+def get_radar_mosaics(directory, filenames, allExists=True, pbar=False, **kargs):
+    """
+    Retrieve multiple radar mosaics from MICAPS cassandra service.
+    
+    Args:
+        directory (string): the data directory on the service.
+        filenames (list): the list of filenames.
+        allExists (boolean): all files should exist, or return None.
+        pbar (boolean): Show progress bar, default to False.
+        **kargs: key arguments passed to get_fy_awx function.
+    """
+
+    dataset = []
+    if pbar:
+        tqdm_filenames = tqdm(filenames, desc=directory + ": ")
+    else:
+        tqdm_filenames = filenames
+    for filename in tqdm_filenames:
+        data = get_radar_mosaic(directory, filename=filename, **kargs)
+        if data:
+            dataset.append(data)
+        else:
+            if allExists:
+                warnings.warn("{} doese not exists.".format(directory+'/'+filename))
+                return None
+    
+    return xr.concat(dataset, dim='time')
+
+
+def get_tlogp(directory, filename=None, suffix="*.000", cache=True):
+    """
+    该程序用于读取micaps服务器上TLOGP数据信息, 文件格式与MICAPS第5类格式相同.
+    :param directory: the data directory on the service
+    :param filename: the data filename, if none, will be the latest file.
+    :param suffix: the filename filter pattern which will be used to
+                   find the specified file.
+    :param cache: cache retrieved data to local directory, default is True.
+    :return: pandas DataFrame object.
+    >>> data = get_tlogp("UPPER_AIR/TLOGP/")
+    """
+
+    # get data file name
+    if filename is None:
+        try:
+            # connect to data service
+            service = GDSDataService()
+            if service.berror == True:
+                return None
+            status, response = service.getLatestDataName(directory, suffix)
+        except ValueError:
+            print('Can not retrieve data from ' + directory)
+            return None
+        StringResult = DataBlock_pb2.StringResult()
+        if status == 200:
+            StringResult.ParseFromString(response)
+            if StringResult is not None:
+                filename = StringResult.name
+                if filename == '':
+                    return None
+            else:
+                return None
+
+    # retrieve data from cached file
+    if cache:
+        cache_file = CONFIG.get_cache_file(directory, filename, name="MICAPS_DATA")
+        if cache_file.is_file():
+            with open(cache_file, 'rb') as f:
+                records = pickle.load(f)
+                return records
+
+    # get data contents
+    try:
+        service = GDSDataService()
+        if service.berror == True:
+            return None
+        status, response = service.getData(directory, filename)
+    except ValueError:
+        print('Can not retrieve data' + filename + ' from ' + directory)
+        return None
+    ByteArrayResult = DataBlock_pb2.ByteArrayResult()
+    if status == 200:
+        ByteArrayResult.ParseFromString(response)
+        if ByteArrayResult is not None:
+            byteArray = ByteArrayResult.byteArray
+            if byteArray == b'':
+                print('There is no data ' + filename + ' in ' + directory)
+                return None
+
+            # decode bytes to string
+            txt = byteArray.decode("utf-8")
+            txt = list(filter(None, re.split(' |\n', txt)))
+
+            # observation date and time
+            if len(txt[3]) < 4:
+                year = int(txt[3]) + 2000
+            else:
+                year = int(txt[3])
+            month = int(txt[4])
+            day = int(txt[5])
+            hour = int(txt[6])
+            time = datetime(year, month, day, hour)
+
+            # the number of records
+            number = int(txt[7])
+            if number < 1:
+                return None
+
+            # cut the data
+            txt = txt[8:]
+
+            # put the data into dictionary
+            index = 0
+            records = []
+            while index < len(txt):
+                # get the record information
+                ID = txt[index].strip()
+                lon = float(txt[index+1])
+                lat = float(txt[index+2])
+                alt = float(txt[index+3])
+                number = int(int(txt[index+4])/6)
+                index += 5
+
+                # get the sounding records
+                for i in range(number):
+                    record = {
+                        'ID': ID, 'lon': lon, 'lat': lat, 'alt': alt,
+                        'time': time,
+                        'p': float(txt[index]), 'h': float(txt[index+1]),
+                        't': float(txt[index+2]), 'td': float(txt[index+3]),
+                        'wind_angle': float(txt[index+4]),
+                        'wind_speed': float(txt[index+5])}
+                    records.append(record)
+                    index += 6
+
+            # transform to pandas data frame
+            records = pd.DataFrame(records)
+            records.set_index('ID')
+
+            # cache data
+            if cache:
+                with open(cache_file, 'wb') as f:
+                    pickle.dump(records, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+            # return
+            return records
+        else:
+            return None
+    else:
+        return None
+
+
+def get_tlogps(directory, filenames, allExists=True, pbar=False, **kargs):
+    """
+    Retrieve multiple tlog observation from MICAPS cassandra service.
+    
+    Args:
+        directory (string): the data directory on the service.
+        filenames (list): the list of filenames.
+        allExists (boolean): all files should exist, or return None.
+        pbar (boolean): Show progress bar, default to False.
+        **kargs: key arguments passed to get_fy_awx function.
+    """
+
+    dataset = []
+    if pbar:
+        tqdm_filenames = tqdm(filenames, desc=directory + ": ")
+    else:
+        tqdm_filenames = filenames
+    for filename in tqdm_filenames:
+        data = get_tlogp(directory, filename=filename, **kargs)
+        if data:
+            dataset.append(data)
+        else:
+            if allExists:
+                warnings.warn("{} doese not exists.".format(directory+'/'+filename))
+                return None
+    
+    return pd.concat(dataset)
+
+
+def get_swan_radar(directory, filename=None, suffix="*.000", scale=[0.1, 0], 
+                   varattrs={'long_name': 'quantitative_precipitation_forecast', 'short_name': 'QPF', 'units': 'mm'},
+                   cache=True, attach_forecast_period=True):
+    """
+    该程序用于读取micaps服务器上SWAN的D131格点数据格式.
+    refer to https://www.taodocs.com/p-274692126.html
+    :param directory: the data directory on the service
+    :param filename: the data filename, if none, will be the latest file.
+    :param suffix: the filename filter pattern which will be used to
+                   find the specified file.
+    :param scale: data value will be scaled = (data + scale[1]) * scale[0], normally,
+                  CREF, CAPPI: [0.5, -66]
+                  radar echo height, VIL, OHP, ...: [0.1, 0]
+    :param varattrs: dictionary, variable attributes.
+    :param cache: cache retrieved data to local directory, default is True.
+    :return: pandas DataFrame object.
+    >>> data = get_swan_radar("RADARMOSAIC/EXTRAPOLATION/QPF/")
+    """
+
+    # get data file name
+    if filename is None:
+        try:
+            # connect to data service
+            service = GDSDataService()
+            if service.berror == True:
+                return None
+            status, response = service.getLatestDataName(directory, suffix)
+        except ValueError:
+            print('Can not retrieve data from ' + directory)
+            return None
+        StringResult = DataBlock_pb2.StringResult()
+        if status == 200:
+            StringResult.ParseFromString(response)
+            if StringResult is not None:
+                filename = StringResult.name
+                if filename == '':
+                    return None
+            else:
+                return None
+
+    # retrieve data from cached file
+    if cache:
+        cache_file = CONFIG.get_cache_file(directory, filename, name="MICAPS_DATA")
+        if cache_file.is_file():
+            with open(cache_file, 'rb') as f:
+                data = pickle.load(f)
+                return data
+
+    # get data contents
+    try:
+        service = GDSDataService()
+        if service.berror == True:
+            return None
+        status, response = service.getData(directory, filename)
+    except ValueError:
+        print('Can not retrieve data' + filename + ' from ' + directory)
+        return None
+    ByteArrayResult = DataBlock_pb2.ByteArrayResult()
+    if status == 200:
+        ByteArrayResult.ParseFromString(response)
+        if ByteArrayResult is not None:
+            byteArray = ByteArrayResult.byteArray
+            if byteArray == b'':
+                print('There is no data ' + filename + ' in ' + directory)
+                return None
+
+            # define head structure
+            head_dtype = [
+                ('ZonName', 'S12'),
+                ('DataName', 'S38'),
+                ('Flag', 'S8'),
+                ('Version', 'S8'),
+                ('year', 'i2'),
+                ('month', 'i2'),     
+                ('day', 'i2'),     
+                ('hour', 'i2'),    
+                ('minute', 'i2'),
+                ('interval', 'i2'),
+                ('XNumGrids', 'i2'),
+                ('YNumGrids', 'i2'),
+                ('ZNumGrids', 'i2'),
+                ('RadarCount', 'i4'),
+                ('StartLon', 'f4'),
+                ('StartLat', 'f4'),
+                ('CenterLon', 'f4'),
+                ('CenterLat', 'f4'),
+                ('XReso', 'f4'),
+                ('YReso', 'f4'),
+                ('ZhighGrids', 'f4', 40),
+                ('RadarStationName', 'S20', 16),
+                ('RadarLongitude', 'f4', 20),
+                ('RadarLatitude', 'f4', 20),
+                ('RadarAltitude', 'f4', 20),
+                ('MosaicFlag', 'S1', 20),
+                ('m_iDataType', 'i2'),
+                ('m_iLevelDimension', 'i2'),
+                ('Reserved', 'S168')]
+
+            # read head information
+            head_info = np.frombuffer(byteArray[0:1024], dtype=head_dtype)
+            ind = 1024
+
+            # get coordinates
+            nlon = head_info['XNumGrids'][0].astype(np.int64)
+            nlat = head_info['YNumGrids'][0].astype(np.int64)
+            nlev = head_info['ZNumGrids'][0].astype(np.int64)
+            dlon = head_info['XReso'][0].astype(np.float)
+            dlat = head_info['YReso'][0].astype(np.float)
+            lat = head_info['StartLat'][0] - np.arange(nlat)*dlat - dlat/2.0
+            lon = head_info['StartLon'][0] + np.arange(nlon)*dlon - dlon/2.0
+            level = head_info['ZhighGrids'][0][0:nlev]
+
+            # retrieve data records
+            data_type = ['u1', 'u1', 'u2', 'i2']
+            data_type = data_type[head_info['m_iDataType'][0]]
+            data_len = (nlon * nlat * nlev)
+            data = np.frombuffer(
+                byteArray[ind:(ind + data_len*int(data_type[1]))], 
+                dtype=data_type, count=data_len)
+
+            # convert data type
+            data.shape = (nlev, nlat, nlon)
+            data = data.astype(np.float32)
+            data = (data + scale[1]) * scale[0]
+
+             # reverse latitude axis
+            data = np.flip(data, 1)
+            lat = lat[::-1]
+
+            # set time coordinates
+            init_time = datetime(
+                head_info['year'][0], head_info['month'][0], 
+                head_info['day'][0], head_info['hour'][0], head_info['minute'][0])
+            if attach_forecast_period:
+                fhour = int(filename.split('.')[1])/60.0
+            else:
+                fhour = 0
+            fhour = np.array([fhour], dtype=np.float)
+            time = init_time + timedelta(hours=fhour[0])
+            init_time = np.array([init_time], dtype='datetime64[ms]')
+            time = np.array([time], dtype='datetime64[ms]')
+
+            # define coordinates
+            time_coord = ('time', time)
+            lon_coord = ('lon', lon, {
+                'long_name':'longitude', 'units':'degrees_east', '_CoordinateAxisType':'Lon'})
+            lat_coord = ('lat', lat, {
+                'long_name':'latitude', 'units':'degrees_north', '_CoordinateAxisType':'Lat'})
+            level_coord = ('level', level, {
+                'long_name':'height', 'units':'m'})
+
+            # create xarray
+            data = np.expand_dims(data, axis=0)
+            data = xr.Dataset({'data':(['time', 'level', 'lat', 'lon'], data, varattrs)},
+                coords={'time':time_coord, 'level':level_coord, 'lat':lat_coord, 'lon':lon_coord})
+
+            # add time coordinates
+            data.coords['forecast_reference_time'] = init_time[0]
+            data.coords['forecast_period'] = ('time', fhour, {
+                'long_name':'forecast_period', 'units':'hour'})
+
+            # add attributes
+            data.attrs['Conventions'] = "CF-1.6"
+            data.attrs['Origin'] = 'MICAPS Cassandra Server'
+
+            # cache data
+            if cache:
+                with open(cache_file, 'wb') as f:
+                    pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
+
+            # return
+            return data
+        else:
+            return None
+    else:
+        return None
+
+
+def get_swan_radars(directory, filenames, allExists=True, pbar=False, **kargs):
+    """
+    Retrieve multiple swan 131 radar from MICAPS cassandra service.
+    
+    Args:
+        directory (string): the data directory on the service.
+        filenames (list): the list of filenames.
+        allExists (boolean): all files should exist, or return None.
+        pbar (boolean): Show progress bar, default to False.
+        **kargs: key arguments passed to get_fy_awx function.
+    """
+
+    dataset = []
+    if pbar:
+        tqdm_filenames = tqdm(filenames, desc=directory + ": ")
+    else:
+        tqdm_filenames = filenames
+    for filename in tqdm_filenames:
+        data = get_swan_radar(directory, filename=filename, **kargs)
+        if data:
+            dataset.append(data)
+        else:
+            if allExists:
+                warnings.warn("{} doese not exists.".format(directory+'/'+filename))
+                return None
+    
+    return xr.concat(dataset, dim='time')
+