bsmcmp 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bsmcmp-0.0.1/LICENSE +21 -0
- bsmcmp-0.0.1/PKG-INFO +79 -0
- bsmcmp-0.0.1/README.md +35 -0
- bsmcmp-0.0.1/bsmcmp/__init__.py +1 -0
- bsmcmp-0.0.1/bsmcmp/__main__.py +19 -0
- bsmcmp-0.0.1/bsmcmp/ascii.py +50 -0
- bsmcmp-0.0.1/bsmcmp/base.py +132 -0
- bsmcmp-0.0.1/bsmcmp/netcdf.py +173 -0
- bsmcmp-0.0.1/bsmcmp/version.py +9 -0
- bsmcmp-0.0.1/bsmcmp.egg-info/PKG-INFO +79 -0
- bsmcmp-0.0.1/bsmcmp.egg-info/SOURCES.txt +15 -0
- bsmcmp-0.0.1/bsmcmp.egg-info/dependency_links.txt +1 -0
- bsmcmp-0.0.1/bsmcmp.egg-info/entry_points.txt +2 -0
- bsmcmp-0.0.1/bsmcmp.egg-info/requires.txt +10 -0
- bsmcmp-0.0.1/bsmcmp.egg-info/top_level.txt +2 -0
- bsmcmp-0.0.1/pyproject.toml +37 -0
- bsmcmp-0.0.1/setup.cfg +4 -0
bsmcmp-0.0.1/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Tianzhu Qiao
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
bsmcmp-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bsmcmp
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: bsmcmp is a tool to compare files
|
|
5
|
+
Author-email: Tianzhu Qiao <tq@feiyilin.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2025 Tianzhu Qiao
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/tianzhuqiao/bsmcmp
|
|
29
|
+
Classifier: Operating System :: OS Independent
|
|
30
|
+
Requires-Python: >=3.0
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
License-File: LICENSE
|
|
33
|
+
Requires-Dist: numpy
|
|
34
|
+
Requires-Dist: scipy
|
|
35
|
+
Requires-Dist: click>=8.1
|
|
36
|
+
Requires-Dist: pandas
|
|
37
|
+
Requires-Dist: pyulog
|
|
38
|
+
Requires-Dist: netCDF4
|
|
39
|
+
Requires-Dist: ply
|
|
40
|
+
Requires-Dist: charset_normalizer
|
|
41
|
+
Requires-Dist: h5py
|
|
42
|
+
Requires-Dist: packaging
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# bsmcmp
|
|
46
|
+
**bsmcmp** is a tool to compare data files.
|
|
47
|
+
|
|
48
|
+
## Install
|
|
49
|
+
```
|
|
50
|
+
$ pip install bsmcmp
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Supported file formats
|
|
54
|
+
- ASCII
|
|
55
|
+
- [netCDF](https://unidata.github.io/netcdf4-python/)
|
|
56
|
+
|
|
57
|
+
## Usage
|
|
58
|
+
For example, to compare two files:
|
|
59
|
+
```
|
|
60
|
+
$ bsmcmp netcdf --file1 file1.nc --file2 file2.nc
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
To compare all files in two folders (assume both folders have the same structure):
|
|
64
|
+
```
|
|
65
|
+
$ bsmcmp netcdf --folder1 file1.nc --folder file2.nc
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
% bsmcmp --help
|
|
70
|
+
Usage: bsmcmp [OPTIONS] COMMAND [ARGS]...
|
|
71
|
+
|
|
72
|
+
Options:
|
|
73
|
+
--version Show the version and exit.
|
|
74
|
+
--help Show this message and exit.
|
|
75
|
+
|
|
76
|
+
Commands:
|
|
77
|
+
ascii
|
|
78
|
+
netcdf
|
|
79
|
+
```
|
bsmcmp-0.0.1/README.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# bsmcmp
|
|
2
|
+
**bsmcmp** is a tool to compare data files.
|
|
3
|
+
|
|
4
|
+
## Install
|
|
5
|
+
```
|
|
6
|
+
$ pip install bsmcmp
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
## Supported file formats
|
|
10
|
+
- ASCII
|
|
11
|
+
- [netCDF](https://unidata.github.io/netcdf4-python/)
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
For example, to compare two files:
|
|
15
|
+
```
|
|
16
|
+
$ bsmcmp netcdf --file1 file1.nc --file2 file2.nc
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
To compare all files in two folders (assume both folders have the same structure):
|
|
20
|
+
```
|
|
21
|
+
$ bsmcmp netcdf --folder1 file1.nc --folder file2.nc
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
% bsmcmp --help
|
|
26
|
+
Usage: bsmcmp [OPTIONS] COMMAND [ARGS]...
|
|
27
|
+
|
|
28
|
+
Options:
|
|
29
|
+
--version Show the version and exit.
|
|
30
|
+
--help Show this message and exit.
|
|
31
|
+
|
|
32
|
+
Commands:
|
|
33
|
+
ascii
|
|
34
|
+
netcdf
|
|
35
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .version import __version__
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import click
|
|
2
|
+
|
|
3
|
+
from .version import __version__
|
|
4
|
+
from .ascii import test_ascii
|
|
5
|
+
|
|
6
|
+
@click.group()
|
|
7
|
+
@click.version_option(__version__)
|
|
8
|
+
def cli():
|
|
9
|
+
pass
|
|
10
|
+
|
|
11
|
+
cli.add_command(test_ascii)
|
|
12
|
+
try:
|
|
13
|
+
from .netcdf import test_netcdf
|
|
14
|
+
cli.add_command(test_netcdf)
|
|
15
|
+
except:
|
|
16
|
+
pass
|
|
17
|
+
|
|
18
|
+
if __name__ == '__main__':
|
|
19
|
+
cli()
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import click
|
|
2
|
+
from charset_normalizer import detect
|
|
3
|
+
|
|
4
|
+
from .base import TestBase, common_options
|
|
5
|
+
|
|
6
|
+
class TestAscii(TestBase):
|
|
7
|
+
NAME = "ascii"
|
|
8
|
+
def __init__(self):
|
|
9
|
+
super().__init__()
|
|
10
|
+
self.ext = ".asc"
|
|
11
|
+
|
|
12
|
+
def open_ascii(self, filename):
|
|
13
|
+
with open(filename.strip(), 'rb') as fp:
|
|
14
|
+
raw = fp.read()
|
|
15
|
+
encoding = detect(raw)['encoding']
|
|
16
|
+
|
|
17
|
+
data = None
|
|
18
|
+
with open(filename, 'r', encoding=encoding) as fp:
|
|
19
|
+
data = fp.readlines()
|
|
20
|
+
if data is None:
|
|
21
|
+
self.error(f'Failed to open {filename}')
|
|
22
|
+
return data
|
|
23
|
+
|
|
24
|
+
def test(self, file1, file2):
|
|
25
|
+
super().test(file1, file2)
|
|
26
|
+
|
|
27
|
+
d1 = self.open_ascii(file1)
|
|
28
|
+
d2 = self.open_ascii(file2)
|
|
29
|
+
|
|
30
|
+
match = d1 == d2
|
|
31
|
+
|
|
32
|
+
if not match:
|
|
33
|
+
self.mismatch_count += 1
|
|
34
|
+
return match
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@click.command('ascii', context_settings={'show_default': True})
|
|
38
|
+
@common_options('.asc', 'ASCII')
|
|
39
|
+
def test_ascii(**kwargs):
|
|
40
|
+
test = TestAscii()
|
|
41
|
+
kwargs = test.load_config(**kwargs)
|
|
42
|
+
if kwargs['file1'] is not None and kwargs['file2'] is not None:
|
|
43
|
+
test.stop_on_mismatch = False
|
|
44
|
+
test.verbose = True
|
|
45
|
+
test.test(kwargs['file1'], kwargs['file2'])
|
|
46
|
+
test.show_result()
|
|
47
|
+
return
|
|
48
|
+
if kwargs['folder1'] is not None and kwargs['folder2'] is not None:
|
|
49
|
+
test.verbose = False
|
|
50
|
+
test.test_all(kwargs['folder1'], kwargs['folder2'])
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import traceback
|
|
3
|
+
import glob
|
|
4
|
+
import re
|
|
5
|
+
import functools
|
|
6
|
+
import click
|
|
7
|
+
from click.core import ParameterSource
|
|
8
|
+
import yaml
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class TestBase:
|
|
12
|
+
NAME = "base"
|
|
13
|
+
def __init__(self):
|
|
14
|
+
self.verbose = True
|
|
15
|
+
self.ext = ".*"
|
|
16
|
+
self.stop_on_mismatch = True
|
|
17
|
+
self.file_count = 0
|
|
18
|
+
self.mismatch_count = 0
|
|
19
|
+
self._stop = False
|
|
20
|
+
self.ignore_pattern = []
|
|
21
|
+
self.recursive = True
|
|
22
|
+
|
|
23
|
+
def echo(self, *args, **kwargs):
|
|
24
|
+
verbose = kwargs.pop('verbose', False) or self.verbose
|
|
25
|
+
if verbose:
|
|
26
|
+
click.secho(*args, **kwargs)
|
|
27
|
+
|
|
28
|
+
def error(self, *args, **kwargs):
|
|
29
|
+
if 'fg' not in kwargs:
|
|
30
|
+
kwargs['fg'] = 'red'
|
|
31
|
+
self.echo(*args, **kwargs)
|
|
32
|
+
|
|
33
|
+
def warning(self, *args, **kwargs):
|
|
34
|
+
if 'fg' not in kwargs:
|
|
35
|
+
kwargs['fg'] = 'blue'
|
|
36
|
+
self.echo(*args, **kwargs)
|
|
37
|
+
|
|
38
|
+
def success(self, *args, **kwargs):
|
|
39
|
+
if 'fg' not in kwargs:
|
|
40
|
+
kwargs['fg'] = 'green'
|
|
41
|
+
self.echo(*args, **kwargs)
|
|
42
|
+
|
|
43
|
+
def test(self, file1, file2):
|
|
44
|
+
self.file_count += 1
|
|
45
|
+
return False
|
|
46
|
+
|
|
47
|
+
def shall_stop(self):
|
|
48
|
+
return self._stop
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def has_pattern(self, value, patterns):
|
|
52
|
+
for item in patterns:
|
|
53
|
+
if re.search(item, value) is not None:
|
|
54
|
+
return True
|
|
55
|
+
return False
|
|
56
|
+
|
|
57
|
+
def shall_ignore(self, filename):
|
|
58
|
+
return self.has_pattern(filename, self.ignore_pattern)
|
|
59
|
+
|
|
60
|
+
def show_result(self):
|
|
61
|
+
self.echo(f'{self.file_count} files checked!', verbose=True)
|
|
62
|
+
self.echo(f' mismatch: {self.mismatch_count}', verbose=True)
|
|
63
|
+
|
|
64
|
+
def test_all(self, folder1, folder2):
|
|
65
|
+
self._stop = False
|
|
66
|
+
if folder1 is not None and folder2 is not None:
|
|
67
|
+
|
|
68
|
+
for filename in glob.iglob(f'{folder1}/**/*{self.ext}', recursive=self.recursive):
|
|
69
|
+
if self.shall_stop():
|
|
70
|
+
break
|
|
71
|
+
file1 = filename
|
|
72
|
+
file2 = filename.replace(folder1, folder2)
|
|
73
|
+
|
|
74
|
+
if self.shall_ignore(filename):
|
|
75
|
+
continue
|
|
76
|
+
|
|
77
|
+
if not os.path.isfile(file2):
|
|
78
|
+
self.warning(f"can't find file: {file2}")
|
|
79
|
+
continue
|
|
80
|
+
try:
|
|
81
|
+
match = self.test(file1, file2)
|
|
82
|
+
if not match:
|
|
83
|
+
self.echo(file1, verbose=True)
|
|
84
|
+
self.echo(file2, verbose=True)
|
|
85
|
+
except:
|
|
86
|
+
self.echo(file1, verbose=True)
|
|
87
|
+
self.echo(file2, verbose=True)
|
|
88
|
+
traceback.print_exc()
|
|
89
|
+
break
|
|
90
|
+
|
|
91
|
+
self.show_result()
|
|
92
|
+
return True
|
|
93
|
+
return False
|
|
94
|
+
|
|
95
|
+
def load_config(self, **kwargs):
|
|
96
|
+
if 'config' in kwargs and os.path.isfile(kwargs['config']):
|
|
97
|
+
try:
|
|
98
|
+
with open(kwargs['config'], 'r', encoding='utf-8') as fp:
|
|
99
|
+
cfg = yaml.safe_load(fp)
|
|
100
|
+
if self.NAME in cfg:
|
|
101
|
+
cfg = cfg[self.NAME]
|
|
102
|
+
for option in kwargs:
|
|
103
|
+
src = click.get_current_context().get_parameter_source(option)
|
|
104
|
+
if src == ParameterSource.COMMANDLINE:
|
|
105
|
+
continue
|
|
106
|
+
kwargs[option] = cfg.get(option, kwargs[option])
|
|
107
|
+
except:
|
|
108
|
+
self.error(f"Fail to load {kwargs['config']}", verbose=True)
|
|
109
|
+
traceback.print_exc()
|
|
110
|
+
|
|
111
|
+
self.ext = kwargs.get('ext', self.ext)
|
|
112
|
+
self.stop_on_mismatch = kwargs.get('stop_on_mismatch', self.stop_on_mismatch)
|
|
113
|
+
self.ignore_pattern = kwargs.get('ignore_pattern', self.ignore_pattern)
|
|
114
|
+
self.recursive = kwargs.get('recursive', self.recursive)
|
|
115
|
+
return kwargs
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def common_options(ext, name):
|
|
119
|
+
def _common_options(f):
|
|
120
|
+
options = [
|
|
121
|
+
click.option('--ext', default=ext, help=f'the {name} file extention'),
|
|
122
|
+
click.option('--file1', type=click.Path(exists=True, dir_okay=False), help=f'1nd {name} file.'),
|
|
123
|
+
click.option('--file2', type=click.Path(exists=True, dir_okay=False), help=f'2nd {name} file.'),
|
|
124
|
+
click.option('--folder1', type=click.Path(exists=True, file_okay=False), help=f'1st top folder contains the {name} files.'),
|
|
125
|
+
click.option('--folder2', type=click.Path(exists=True, file_okay=False), help=f'2nd top folder contains the {name} files. folder1 and folder2 shall have the same structure'),
|
|
126
|
+
click.option('--stop_on_mismatch/--no-stop_on_mismatch', is_flag=True, default=True, help='Stop when see any data mismatch'),
|
|
127
|
+
click.option('--ignore_pattern', '-i', multiple=True, help='filename pattern to be ignored'),
|
|
128
|
+
click.option('--recursive/--no-recursive', default=True, is_flag=True, help='the netcef file extention'),
|
|
129
|
+
click.option('--config', default='file_compare.yml', type=click.Path(exists=False, dir_okay=False), help='the configuation in yaml file'),
|
|
130
|
+
]
|
|
131
|
+
return functools.reduce(lambda x, opt: opt(x), reversed(options), f)
|
|
132
|
+
return _common_options
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import numpy as np
|
|
2
|
+
import netCDF4
|
|
3
|
+
from netCDF4 import Dataset
|
|
4
|
+
import click
|
|
5
|
+
|
|
6
|
+
from .base import TestBase, common_options
|
|
7
|
+
|
|
8
|
+
class TestNetcdf(TestBase):
|
|
9
|
+
NAME = 'netcdf'
|
|
10
|
+
def __init__(self):
|
|
11
|
+
super().__init__()
|
|
12
|
+
self.ext = ".nc"
|
|
13
|
+
self.mismatch_attr = 0
|
|
14
|
+
self.stop_on_attr_mismatch = False
|
|
15
|
+
self.ignore_variables = []
|
|
16
|
+
self.ignore_attributes = []
|
|
17
|
+
|
|
18
|
+
def check_attr(self, d1, d2, indent=''):
|
|
19
|
+
match = True
|
|
20
|
+
if sorted(d1.ncattrs()) != sorted(d2.ncattrs()):
|
|
21
|
+
self.error(f"{indent}different attributes")
|
|
22
|
+
self.echo(f"{indent} {sorted(d1.ncattrs())}")
|
|
23
|
+
self.echo(f"{indent} {sorted(d2.ncattrs())}")
|
|
24
|
+
|
|
25
|
+
for att in d1.ncattrs():
|
|
26
|
+
if att not in d2.ncattrs():
|
|
27
|
+
continue
|
|
28
|
+
if self.has_pattern(att, self.ignore_attributes):
|
|
29
|
+
self.warning(f"{indent}{att}: ignore")
|
|
30
|
+
continue
|
|
31
|
+
if d1.getncattr(att) != d2.getncattr(att):
|
|
32
|
+
match = False
|
|
33
|
+
self.error(f"{indent}{att}: fail")
|
|
34
|
+
self.echo(f"{indent} {d1.getncattr(att)}")
|
|
35
|
+
self.echo(f"{indent} {d2.getncattr(att)}")
|
|
36
|
+
else:
|
|
37
|
+
self.success(f"{indent}{att}: pass")
|
|
38
|
+
return match
|
|
39
|
+
|
|
40
|
+
def check_data(self, d1, d2, indent=''):
|
|
41
|
+
d1 = np.asarray(d1[:])
|
|
42
|
+
d2 = np.asarray(d2[:])
|
|
43
|
+
match = True
|
|
44
|
+
if d1.shape == d2.shape:
|
|
45
|
+
if not d1.shape:
|
|
46
|
+
# empty variable
|
|
47
|
+
match = True
|
|
48
|
+
else:
|
|
49
|
+
match = np.array_equal(d1, d2, equal_nan=True)
|
|
50
|
+
else:
|
|
51
|
+
match = False
|
|
52
|
+
|
|
53
|
+
if not match:
|
|
54
|
+
self.error(f"{indent}data: fail")
|
|
55
|
+
err = np.abs(d1[:]-d2[:])
|
|
56
|
+
w = np.argwhere(err == np.max(err))
|
|
57
|
+
n_zero_err = np.sum((err.flatten() == 0))
|
|
58
|
+
n_all = len(err.flatten())
|
|
59
|
+
|
|
60
|
+
self.error(f"{indent} max error: {np.nanmax(err)} at {np.argwhere(err == np.max(err))}")
|
|
61
|
+
self.error(f"{indent} d1: {d1[:][tuple(w[0])]}")
|
|
62
|
+
self.error(f"{indent} d2: {d2[:][tuple(w[0])]}")
|
|
63
|
+
self.error(f"{indent} avg error: {np.nanmean(err)}")
|
|
64
|
+
self.error(f"{indent} std error: {np.nanstd(err)}")
|
|
65
|
+
self.error(f"{indent} 0 error(%): {n_zero_err}/{n_all} = {n_zero_err/n_all}")
|
|
66
|
+
|
|
67
|
+
else:
|
|
68
|
+
self.success(f"{indent}data: pass")
|
|
69
|
+
|
|
70
|
+
return match
|
|
71
|
+
|
|
72
|
+
def check_group(self, group1, group2, indent=""):
|
|
73
|
+
|
|
74
|
+
# check attribute
|
|
75
|
+
match_attr = self.check_attr(group1, group2)
|
|
76
|
+
|
|
77
|
+
# check data and its attributes
|
|
78
|
+
match_data = True
|
|
79
|
+
match_data = len(group1.variables) == len(group2.variables)
|
|
80
|
+
for k in group1.variables:
|
|
81
|
+
self.echo(k)
|
|
82
|
+
if self.has_pattern(k, self.ignore_variables):
|
|
83
|
+
self.warning(f"{indent} ignore")
|
|
84
|
+
continue
|
|
85
|
+
|
|
86
|
+
if k not in group2.variables:
|
|
87
|
+
self.error(f'group "{k}" not found in 2nd file')
|
|
88
|
+
match_data = False
|
|
89
|
+
continue
|
|
90
|
+
|
|
91
|
+
d1 = group1.variables[k]
|
|
92
|
+
d2 = group2.variables[k]
|
|
93
|
+
if not self.check_attr(d1, d2, indent+' '):
|
|
94
|
+
match_attr = False
|
|
95
|
+
if not self.check_data(d1, d2, indent+' '):
|
|
96
|
+
match_data = False
|
|
97
|
+
|
|
98
|
+
# check groups
|
|
99
|
+
if len(group1.groups) != len(group2.groups):
|
|
100
|
+
match_data = False
|
|
101
|
+
for k, g in group1.groups.items():
|
|
102
|
+
if k not in group2.groups:
|
|
103
|
+
match_data = False
|
|
104
|
+
self.error(f"{k} not found in 2nd file", bg='red')
|
|
105
|
+
group_data, group_attr = self.check_group(g, group2[k], indent+' ')
|
|
106
|
+
match_data = match_data and group_data
|
|
107
|
+
match_attr = match_data and group_attr
|
|
108
|
+
|
|
109
|
+
return match_data, match_attr
|
|
110
|
+
|
|
111
|
+
def test(self, file1, file2):
|
|
112
|
+
super().test(file1, file2)
|
|
113
|
+
|
|
114
|
+
nc_p = Dataset(file1)
|
|
115
|
+
nc_m = Dataset(file2)
|
|
116
|
+
match_data, match_attr = self.check_group(nc_p, nc_m)
|
|
117
|
+
nc_p.close()
|
|
118
|
+
nc_m.close()
|
|
119
|
+
if not match_data:
|
|
120
|
+
self.mismatch_count += 1
|
|
121
|
+
|
|
122
|
+
if not match_attr:
|
|
123
|
+
self.mismatch_attr += 1
|
|
124
|
+
if self.stop_on_attr_mismatch and not match_attr:
|
|
125
|
+
self._stop = True
|
|
126
|
+
self.error("attribute mismatch")
|
|
127
|
+
|
|
128
|
+
if self.stop_on_mismatch and not match_data:
|
|
129
|
+
self._stop = True
|
|
130
|
+
self.error("data mismatch")
|
|
131
|
+
|
|
132
|
+
self.echo("--------")
|
|
133
|
+
self.echo("overall:")
|
|
134
|
+
if match_data:
|
|
135
|
+
self.success(' data: pass')
|
|
136
|
+
else:
|
|
137
|
+
self.error(' data: fail')
|
|
138
|
+
if match_attr:
|
|
139
|
+
self.success(' attribute: pass')
|
|
140
|
+
else:
|
|
141
|
+
self.error(' attribute: fail')
|
|
142
|
+
return match_data
|
|
143
|
+
|
|
144
|
+
def test_all(self, folder1, folder2):
|
|
145
|
+
if super().test_all(folder1, folder2):
|
|
146
|
+
self.echo(f' attr mismatch: {self.mismatch_attr}', verbose=True)
|
|
147
|
+
|
|
148
|
+
def load_config(self, **kwargs):
|
|
149
|
+
kwargs = super().load_config(**kwargs)
|
|
150
|
+
self.stop_on_attr_mismatch = kwargs.get('stop_on_attr_mismatch', self.stop_on_attr_mismatch)
|
|
151
|
+
self.ignore_variables = kwargs.get('ignore_var', [])
|
|
152
|
+
self.ignore_attributes = kwargs.get('ignore_att', [])
|
|
153
|
+
return kwargs
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
@click.command('netcdf', context_settings={'show_default': True})
|
|
157
|
+
@common_options('.nc', 'netCDF')
|
|
158
|
+
@click.option('--stop_on_attr_mismatch', is_flag=True, default=False, help='Stop when see any attribute mismatch')
|
|
159
|
+
@click.option('--ignore_var', multiple=True, help='variables to be ignored')
|
|
160
|
+
@click.option('--ignore_att', multiple=True, help='attributes to be ignored')
|
|
161
|
+
def test_netcdf(**kwargs):
|
|
162
|
+
test = TestNetcdf()
|
|
163
|
+
kwargs = test.load_config(**kwargs)
|
|
164
|
+
if kwargs['file1'] is not None and kwargs['file2'] is not None:
|
|
165
|
+
test.stop_on_mismatch = False
|
|
166
|
+
test.stop_on_attr_mismatch = False
|
|
167
|
+
test.verbose = True
|
|
168
|
+
test.test(kwargs['file1'], kwargs['file2'])
|
|
169
|
+
test.show_result()
|
|
170
|
+
return
|
|
171
|
+
if kwargs['folder1'] is not None and kwargs['folder2'] is not None:
|
|
172
|
+
test.verbose = False
|
|
173
|
+
test.test_all(kwargs['folder1'], kwargs['folder2'])
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: bsmcmp
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: bsmcmp is a tool to compare files
|
|
5
|
+
Author-email: Tianzhu Qiao <tq@feiyilin.com>
|
|
6
|
+
License: MIT License
|
|
7
|
+
|
|
8
|
+
Copyright (c) 2025 Tianzhu Qiao
|
|
9
|
+
|
|
10
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
11
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
12
|
+
in the Software without restriction, including without limitation the rights
|
|
13
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
14
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
15
|
+
furnished to do so, subject to the following conditions:
|
|
16
|
+
|
|
17
|
+
The above copyright notice and this permission notice shall be included in all
|
|
18
|
+
copies or substantial portions of the Software.
|
|
19
|
+
|
|
20
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
21
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
22
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
23
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
24
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
25
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
26
|
+
SOFTWARE.
|
|
27
|
+
|
|
28
|
+
Project-URL: Homepage, https://github.com/tianzhuqiao/bsmcmp
|
|
29
|
+
Classifier: Operating System :: OS Independent
|
|
30
|
+
Requires-Python: >=3.0
|
|
31
|
+
Description-Content-Type: text/markdown
|
|
32
|
+
License-File: LICENSE
|
|
33
|
+
Requires-Dist: numpy
|
|
34
|
+
Requires-Dist: scipy
|
|
35
|
+
Requires-Dist: click>=8.1
|
|
36
|
+
Requires-Dist: pandas
|
|
37
|
+
Requires-Dist: pyulog
|
|
38
|
+
Requires-Dist: netCDF4
|
|
39
|
+
Requires-Dist: ply
|
|
40
|
+
Requires-Dist: charset_normalizer
|
|
41
|
+
Requires-Dist: h5py
|
|
42
|
+
Requires-Dist: packaging
|
|
43
|
+
Dynamic: license-file
|
|
44
|
+
|
|
45
|
+
# bsmcmp
|
|
46
|
+
**bsmcmp** is a tool to compare data files.
|
|
47
|
+
|
|
48
|
+
## Install
|
|
49
|
+
```
|
|
50
|
+
$ pip install bsmcmp
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Supported file formats
|
|
54
|
+
- ASCII
|
|
55
|
+
- [netCDF](https://unidata.github.io/netcdf4-python/)
|
|
56
|
+
|
|
57
|
+
## Usage
|
|
58
|
+
For example, to compare two files:
|
|
59
|
+
```
|
|
60
|
+
$ bsmcmp netcdf --file1 file1.nc --file2 file2.nc
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
To compare all files in two folders (assume both folders have the same structure):
|
|
64
|
+
```
|
|
65
|
+
$ bsmcmp netcdf --folder1 file1.nc --folder file2.nc
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
```
|
|
69
|
+
% bsmcmp --help
|
|
70
|
+
Usage: bsmcmp [OPTIONS] COMMAND [ARGS]...
|
|
71
|
+
|
|
72
|
+
Options:
|
|
73
|
+
--version Show the version and exit.
|
|
74
|
+
--help Show this message and exit.
|
|
75
|
+
|
|
76
|
+
Commands:
|
|
77
|
+
ascii
|
|
78
|
+
netcdf
|
|
79
|
+
```
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
bsmcmp/__init__.py
|
|
5
|
+
bsmcmp/__main__.py
|
|
6
|
+
bsmcmp/ascii.py
|
|
7
|
+
bsmcmp/base.py
|
|
8
|
+
bsmcmp/netcdf.py
|
|
9
|
+
bsmcmp/version.py
|
|
10
|
+
bsmcmp.egg-info/PKG-INFO
|
|
11
|
+
bsmcmp.egg-info/SOURCES.txt
|
|
12
|
+
bsmcmp.egg-info/dependency_links.txt
|
|
13
|
+
bsmcmp.egg-info/entry_points.txt
|
|
14
|
+
bsmcmp.egg-info/requires.txt
|
|
15
|
+
bsmcmp.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=67.6.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "bsmcmp"
|
|
7
|
+
authors = [
|
|
8
|
+
{ name="Tianzhu Qiao", email="tq@feiyilin.com" },
|
|
9
|
+
]
|
|
10
|
+
license = {file = "LICENSE"}
|
|
11
|
+
description='bsmcmp is a tool to compare files'
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
requires-python = ">=3.0"
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Operating System :: OS Independent",
|
|
16
|
+
]
|
|
17
|
+
dependencies = [
|
|
18
|
+
'numpy', 'scipy', 'click>=8.1', 'pandas', 'pyulog', 'netCDF4', 'ply',
|
|
19
|
+
'charset_normalizer', 'h5py', 'packaging'
|
|
20
|
+
]
|
|
21
|
+
dynamic = ["version"]
|
|
22
|
+
|
|
23
|
+
[tool.setuptools.dynamic]
|
|
24
|
+
version = {attr = "bsmcmp.__version__"}
|
|
25
|
+
|
|
26
|
+
[project.urls]
|
|
27
|
+
"Homepage" = "https://github.com/tianzhuqiao/bsmcmp"
|
|
28
|
+
|
|
29
|
+
[project.gui-scripts]
|
|
30
|
+
bsmcmp = "bsmcmp.__main__:cli"
|
|
31
|
+
|
|
32
|
+
[tool.setuptools.packages]
|
|
33
|
+
find = {}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
[options]
|
|
37
|
+
include_package_data = true
|
bsmcmp-0.0.1/setup.cfg
ADDED