AeroViz 0.1.12__tar.gz → 0.1.13__tar.gz

{aeroviz-0.1.12 → aeroviz-0.1.13}/AeroViz/dataProcess/Optical/_IMPROVE.py

@@ -41,7 +41,7 @@ def _revised(_df_mass, _df_RH):
 
         _df['AS'] = 2.2 * _frhs * _df_mass['S_AS'] + 4.8 * _frhl * _df_mass['L_AS']
         _df['AN'] = 2.4 * _frhs * _df_mass['S_AN'] + 5.1 * _frhl * _df_mass['L_AN']
-        _df['OM'] = 2.8 * _df_mass['S_OM'] + 6.1 * _frhl * _df_mass['L_OM']
+        _df['OM'] = 2.8 * _df_mass['S_OM'] + 6.1 * _df_mass['L_OM']
         _df['Soil'] = _df_mass['Soil']
         _df['SS'] = 1.7 * _frhss * _df_mass['SS']
         _df['EC'] = 10 * _df_mass['EC']

{aeroviz-0.1.12 → aeroviz-0.1.13}/AeroViz/rawDataReader/script/APS.py

@@ -22,19 +22,23 @@ class Reader(AbstractReader):
     # QC data
     def _QC(self, _df):
         _df = _df.copy()
+        _index = _df.index.copy()
 
         # mask out the data size lower than 7
-        _df['total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.keys().to_numpy(float)))).mean()
-        _df_size = _df['total'].dropna().resample('1h').size().resample(_df.index.freq).ffill()
-        _df = _df.mask(_df_size < 7)
+        _df.loc[:, 'total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.keys().to_numpy(float)))).mean()
 
-        # remove total conc. lower than 700
-        _df = _df.mask(_df['total'] > 700)
+        hourly_counts = (_df['total']
+                         .dropna()
+                         .resample('h')
+                         .size()
+                         .resample('6min')
+                         .ffill()
+                         .reindex(_df.index, method='ffill', tolerance='6min'))
 
-        # not confirmed
-        # remove the bin over 4000 nm which num. conc. larger than 1
-        # _df_remv_ky = _df.keys()[:-2][_df.keys()[:-2]>=4.]
+        # Remove data with less than 6 data per hour
+        _df = _df.mask(hourly_counts < 6)
 
-        # _df_1hr[_df_remv_ky] = _df_1hr[_df_remv_ky].copy().mask(_df_1hr[_df_remv_ky]>1.)
+        # remove total conc. lower than 700 or lower than 1
+        _df = _df.mask((_df['total'] > 700) | (_df['total'] < 1))
 
         return _df[_df.keys()[:-1]]

{aeroviz-0.1.12 → aeroviz-0.1.13}/AeroViz/rawDataReader/script/OCEC.py

@@ -9,12 +9,12 @@ class Reader(AbstractReader):
 
     def _raw_reader(self, file):
         with open(file, 'r', encoding='utf-8', errors='ignore') as f:
-            _df = read_csv(f, skiprows=3)
+            _df = read_csv(f, skiprows=3, on_bad_lines='skip')
 
             _df['Start Date/Time'] = _df['Start Date/Time'].str.strip()
             _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %I:%M:%S %p', errors='coerce')
 
-            if _df['time'].isna().any():
+            if _df['time'].isna().all():
                 _df['time'] = to_datetime(_df['Start Date/Time'], format='%m/%d/%Y %H:%M:%S', errors='coerce')
 
             _df = _df.set_index('time')

{aeroviz-0.1.12 → aeroviz-0.1.13}/AeroViz/rawDataReader/script/SMPS.py

@@ -59,27 +59,42 @@ class Reader(AbstractReader):
             if _df_smps.columns[0] != self.size_range[0] or _df_smps.columns[-1] != self.size_range[1]:
                 self.logger.info(f'\tSMPS file: {file.name} is not match the default size range {self.size_range}, '
                                  f'it is ({_df_smps.columns[0]}, {_df_smps.columns[-1]})')
+                return None
 
             return _df_smps.apply(to_numeric, errors='coerce')
 
     # QC data
     def _QC(self, _df):
-        df = _df.copy()
+        _df = _df.copy()
+        _index = _df.index.copy()
 
-        size_range_mask = (df.columns.astype(float) >= self.size_range[0]) & (
-                df.columns.astype(float) <= self.size_range[1])
-        df = df.loc[:, size_range_mask]
+        size_range_mask = (_df.columns.astype(float) >= self.size_range[0]) & (
+                _df.columns.astype(float) <= self.size_range[1])
+        _df = _df.loc[:, size_range_mask]
 
         # mask out the data size lower than 7
-        df.loc[:, 'total'] = df.sum(axis=1, min_count=1) * (np.diff(np.log(df.columns[:-1].to_numpy(float)))).mean()
-        _df_size = df['total'].dropna().resample('1h').size().resample(df.index.freq).ffill()
-        df = df.mask(_df_size < 7)
+        _df.loc[:, 'total'] = _df.sum(axis=1, min_count=1) * (np.diff(np.log(_df.columns[:-1].to_numpy(float)))).mean()
 
-        # remove total conc. lower than 2000
-        df = df.mask(df['total'] < 2000)
+        hourly_counts = (_df['total']
+                         .dropna()
+                         .resample('h')
+                         .size()
+                         .resample('6min')
+                         .ffill()
+                         .reindex(_df.index, method='ffill', tolerance='6min'))
+
+        # Remove data with less than 6 data per hour
+        _df = _df.mask(hourly_counts < 6)
+
+        # remove total conc. (dN) lower than 2000
+        _df = _df.mask(_df['total'] < 2000)
+        _df = _df.drop('total', axis=1)
+
+        # remove single bin conc. (dN/dlogdp) larger than 1e6
+        _df = _df.mask((_df > 1e6).any(axis=1))
 
         # remove the bin over 400 nm which num. conc. larger than 4000
-        _df_remv_ky = df.keys()[:-1][df.keys()[:-1] >= 400.]
-        df[_df_remv_ky] = df[_df_remv_ky].copy().mask(df[_df_remv_ky] > 4000.)
+        large_bins = _df.columns[_df.columns.astype(float) >= 400]
+        _df = _df.mask((_df[large_bins] > 4000).any(axis=1))
 
-        return df[df.keys()[:-1]]
+        return _df

{aeroviz-0.1.12 → aeroviz-0.1.13}/AeroViz.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AeroViz
-Version: 0.1.12
+Version: 0.1.13
 Summary: Aerosol science
 Author-email: alex <alex870521@gmail.com>
 License: MIT

{aeroviz-0.1.12 → aeroviz-0.1.13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: AeroViz
-Version: 0.1.12
+Version: 0.1.13
 Summary: Aerosol science
 Author-email: alex <alex870521@gmail.com>
 License: MIT

{aeroviz-0.1.12 → aeroviz-0.1.13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "AeroViz"
-version = "0.1.12"
+version = "0.1.13"
 description = "Aerosol science"
 authors = [{ name = "alex", email = "alex870521@gmail.com" }]
 license = { text = "MIT" }