nctoolkit 1.2.6__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/PKG-INFO +2 -2
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/README.md +1 -1
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/anomaly.py +6 -13
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/api.py +4 -9
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/cleanup.py +62 -85
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/distgrid.py +6 -8
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/nco_command.py +0 -5
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/runners.py +24 -32
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/runthis.py +24 -32
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/session.py +5 -13
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/split.py +6 -8
- nctoolkit-1.3.0/nctoolkit/temp_file.py +46 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/verticals.py +5 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/setup.py +2 -2
- nctoolkit-1.2.6/nctoolkit/temp_file.py +0 -68
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/CITATION.cff +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/CODE_OF_CONDUCT.md +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/CONTRIBUTIONS.md +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/LICENSE +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/MANIFEST.in +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/benchmarks/benchmark_nctoolkit_versus_xarray.ipynb +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/cheatsheet/nctoolkit_cheatsheet.pdf +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/cheatsheet/nctoolkit_cheatsheet.pptx +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/checklists/api_checker.ipynb +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/data/geotiff.tif +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/data/pubplot_test.png +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/data/test1.html +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/data/test2.html +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/__init__.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/add_etc.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/append.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/assign.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/cdo_command.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/cellareas.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/centres.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/checks.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/clear.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/compare.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/compare_data.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/corr.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/create_ensemble.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/crop.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/deprecated.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/drop.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/ensembles.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/esoteric.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/experimental.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/fill.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/flatten.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/fldstat.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/format.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/generate_grid.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/inttime.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/invert.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/masking.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/matchpoint.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/mergers.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/meridonials.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/mp_adders.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/mp_matchers.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/mp_matchups.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/phenology.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/plot.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/reduce.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/reduce_grid.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/regrid.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/remove.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/rename.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/resample.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/rollstat.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/run.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/set.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/setters.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/shape.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/shift.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/show.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/static_plot.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/strip_vars.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/subset.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/sumall.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/temporal_stat.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/temporals.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/thresholds.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/to_lonlat.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/to_nc.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/toxarray.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/tozlev.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/unify.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/utils.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/validator.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/validator_funs.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/zip.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit/zonals.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/nctoolkit.egg-info/SOURCES.txt +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/pyproject.toml +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/requirements.txt +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/setup.cfg +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/testing/test_parallel.py +0 -0
- {nctoolkit-1.2.6 → nctoolkit-1.3.0}/testing/test_scripting.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: nctoolkit
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: A Python package for netCDF analysis and post-processing
|
|
5
5
|
Home-page: https://github.com/pmlmodelling/nctoolkit
|
|
6
6
|
Author: Robert Wilson
|
|
@@ -52,7 +52,7 @@ Dynamic: summary
|
|
|
52
52
|
|
|
53
53
|
|
|
54
54
|
|
|
55
|
-
**nctoolkit** is a comprehensive Python (3.
|
|
55
|
+
**nctoolkit** is a comprehensive Python (3.10 and above) package for analyzing netCDF data on Linux and macOS.
|
|
56
56
|
|
|
57
57
|
Core abilities of nctoolkit include:
|
|
58
58
|
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
# nctoolkit - A Python package for netCDF analysis and post-processing
|
|
19
19
|
|
|
20
|
-
nctoolkit is a comprehensive Python (3.8 and above) package for analyzing netCDF data on Linux
|
|
20
|
+
nctoolkit is a comprehensive Python (3.8 and above) package for analyzing netCDF data on Linux.
|
|
21
21
|
|
|
22
22
|
Core abilities include:
|
|
23
23
|
- Clipping to spatial regions
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import copy
|
|
2
|
-
import platform
|
|
3
2
|
import os
|
|
4
3
|
|
|
5
4
|
from nctoolkit.api import open_data
|
|
@@ -11,19 +10,13 @@ from nctoolkit.session import remove_safe, session_info, nc_safe_par, nc_safe
|
|
|
11
10
|
from nctoolkit.api import options
|
|
12
11
|
from nctoolkit.runners import ann_anomaly
|
|
13
12
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
from multiprocessing import Manager
|
|
13
|
+
import multiprocessing as mp
|
|
14
|
+
#start = mp.get_start_method()
|
|
15
|
+
#if start != "spawn":
|
|
16
|
+
# mp.set_start_method("spawn")
|
|
17
|
+
from multiprocessing import Manager
|
|
20
18
|
|
|
21
|
-
|
|
22
|
-
else:
|
|
23
|
-
import multiprocess as mp
|
|
24
|
-
from multiprocess import Manager
|
|
25
|
-
|
|
26
|
-
manager = Manager()
|
|
19
|
+
manager = Manager()
|
|
27
20
|
|
|
28
21
|
|
|
29
22
|
|
|
@@ -10,7 +10,6 @@ import string
|
|
|
10
10
|
import subprocess
|
|
11
11
|
import warnings
|
|
12
12
|
import urllib.request
|
|
13
|
-
import platform
|
|
14
13
|
|
|
15
14
|
from netCDF4 import Dataset
|
|
16
15
|
|
|
@@ -143,19 +142,15 @@ session_info["coast"] = None
|
|
|
143
142
|
session_info["interactive"] = sys.__stdin__.isatty()
|
|
144
143
|
|
|
145
144
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
session_info["size"] = result.f_frsize * result.f_bavail
|
|
149
|
-
else:
|
|
150
|
-
session_info["size"] = 0
|
|
145
|
+
result = os.statvfs("/tmp/")
|
|
146
|
+
session_info["size"] = result.f_frsize * result.f_bavail
|
|
151
147
|
|
|
152
148
|
|
|
153
149
|
session_info["latest_size"] = 0
|
|
154
150
|
session_info["cores"] = 1
|
|
155
151
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
append_tempdirs("/var/tmp")
|
|
152
|
+
append_tempdirs("/tmp")
|
|
153
|
+
append_tempdirs("/var/tmp")
|
|
159
154
|
|
|
160
155
|
|
|
161
156
|
def update_options(kwargs):
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import glob
|
|
2
2
|
import shutil
|
|
3
3
|
import os
|
|
4
|
-
import platform
|
|
5
4
|
import tempfile
|
|
6
5
|
import time
|
|
7
6
|
import stat
|
|
@@ -66,19 +65,17 @@ def cleanup():
|
|
|
66
65
|
if os.path.exists(dd):
|
|
67
66
|
nc_remove(dd)
|
|
68
67
|
|
|
69
|
-
# only update the session size on linux
|
|
70
68
|
|
|
71
69
|
if session_info["user_dir"] is False:
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
if
|
|
76
|
-
if session_info["temp_dir"] == "/var/tmp/":
|
|
77
|
-
session_info["temp_dir"] = "/tmp/"
|
|
78
|
-
session_info["size"] = result
|
|
79
|
-
|
|
80
|
-
if session_info["size"] > (1.5 * session_info["latest_size"]):
|
|
70
|
+
result = os.statvfs("/tmp/")
|
|
71
|
+
result = result.f_frsize * result.f_bavail
|
|
72
|
+
if result > session_info["size"]:
|
|
73
|
+
if session_info["temp_dir"] == "/var/tmp/":
|
|
81
74
|
session_info["temp_dir"] = "/tmp/"
|
|
75
|
+
session_info["size"] = result
|
|
76
|
+
|
|
77
|
+
if session_info["size"] > (1.5 * session_info["latest_size"]):
|
|
78
|
+
session_info["temp_dir"] = "/tmp/"
|
|
82
79
|
|
|
83
80
|
|
|
84
81
|
def clean_all(x = None, y = None):
|
|
@@ -140,43 +137,25 @@ def temp_check():
|
|
|
140
137
|
Function to check temp files
|
|
141
138
|
"""
|
|
142
139
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
if len(mylist)
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
else:
|
|
164
|
-
temp_folder = tempfile.gettempdir()
|
|
165
|
-
|
|
166
|
-
mylist = [f for f in glob.glob(f"{temp_folder}/*")]
|
|
167
|
-
mylist = [f for f in mylist if "nctoolkit" in f]
|
|
168
|
-
|
|
169
|
-
if len(mylist) > 0:
|
|
170
|
-
if len(mylist) == 1:
|
|
171
|
-
print(
|
|
172
|
-
f"{len(mylist)} temporary file was created by nctoolkit in prior or current "
|
|
173
|
-
f"sessions. Consider running deep_clean!"
|
|
174
|
-
)
|
|
175
|
-
else:
|
|
176
|
-
print(
|
|
177
|
-
f"{len(mylist)} temporary files were created by nctoolkit in prior or "
|
|
178
|
-
f"current sessions. Consider running deep_clean!"
|
|
179
|
-
)
|
|
140
|
+
mylist = [f for f in glob.glob("/tmp/*")]
|
|
141
|
+
mylist = mylist + [f for f in glob.glob("/var/tmp/*")]
|
|
142
|
+
mylist = mylist + [f for f in glob.glob("/usr/tmp/*")]
|
|
143
|
+
mylist = [f for f in mylist if "nctoolkit" in f]
|
|
144
|
+
|
|
145
|
+
session_info["old_files"] = mylist
|
|
146
|
+
|
|
147
|
+
if len(mylist) > 0:
|
|
148
|
+
if len(mylist) == 1:
|
|
149
|
+
print(
|
|
150
|
+
f"{len(mylist)} temporary file was created by nctoolkit in prior or current "
|
|
151
|
+
f"sessions. Consider running deep_clean!"
|
|
152
|
+
)
|
|
153
|
+
else:
|
|
154
|
+
print(
|
|
155
|
+
f"{len(mylist)} temporary files were created by nctoolkit in prior or current"
|
|
156
|
+
f" sessions. Consider running deep_clean!"
|
|
157
|
+
)
|
|
158
|
+
|
|
180
159
|
|
|
181
160
|
|
|
182
161
|
def disk_clean(self):
|
|
@@ -187,46 +166,44 @@ def disk_clean(self):
|
|
|
187
166
|
if session_info["user_dir"]:
|
|
188
167
|
return None
|
|
189
168
|
|
|
190
|
-
#
|
|
191
|
-
if platform.system() == "Linux":
|
|
192
|
-
# get files as a list
|
|
169
|
+
# get files as a list
|
|
193
170
|
|
|
194
|
-
|
|
171
|
+
ff_list = self.current
|
|
195
172
|
|
|
196
|
-
|
|
197
|
-
|
|
173
|
+
# First step is to figure out how much space is in /tmp
|
|
174
|
+
# Do nothing if it is less than 0.5 GB
|
|
198
175
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
176
|
+
if session_info["temp_dir"] == "/tmp/":
|
|
177
|
+
result = os.statvfs("/tmp/")
|
|
178
|
+
result = result.f_frsize * result.f_bavail
|
|
202
179
|
|
|
203
|
-
|
|
204
|
-
return None
|
|
205
|
-
|
|
206
|
-
if session_info["temp_dir"] not in ["/tmp/", "/var/tmp", "/usr/tmp"]:
|
|
180
|
+
if result > 0.5 * 1e9:
|
|
207
181
|
return None
|
|
208
182
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
183
|
+
if session_info["temp_dir"] not in ["/tmp/", "/var/tmp", "/usr/tmp"]:
|
|
184
|
+
return None
|
|
185
|
+
|
|
186
|
+
# at this point we want to change the temp dir,
|
|
187
|
+
# though it probably has been already
|
|
188
|
+
session_info["temp_dir"] = "/var/tmp/"
|
|
189
|
+
# get a list of the new file names
|
|
190
|
+
|
|
191
|
+
# loop through the existing ones
|
|
192
|
+
for ff in ff_list:
|
|
193
|
+
# check if the file is in /var/tmp
|
|
194
|
+
# if it is, keep it that way
|
|
195
|
+
# check the space remaining the /tmp
|
|
196
|
+
result = os.statvfs("/tmp/")
|
|
197
|
+
result = result.f_frsize * result.f_bavail
|
|
198
|
+
# if there is less than 0.5 GB left, move the file to /var/tmp
|
|
199
|
+
if result < 0.5 * 1e9:
|
|
200
|
+
if ff.startswith("/tmp/"):
|
|
201
|
+
new_ff = ff.replace("/tmp/", "/var/tmp/")
|
|
202
|
+
append_safe(new_ff)
|
|
203
|
+
remove_safe(ff)
|
|
204
|
+
shutil.copyfile(ff, new_ff)
|
|
205
|
+
self.current = [
|
|
206
|
+
new_ff if file == ff else file for file in self.current
|
|
207
|
+
]
|
|
208
|
+
|
|
209
|
+
cleanup()
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import glob
|
|
2
2
|
import copy
|
|
3
3
|
import os
|
|
4
|
-
import platform
|
|
5
4
|
|
|
6
5
|
from nctoolkit.cleanup import cleanup
|
|
7
6
|
from nctoolkit.temp_file import temp_file
|
|
@@ -26,14 +25,13 @@ def dist_cdo(self, i=None, j=None):
|
|
|
26
25
|
# But, first we need to check if there is space in the output folder
|
|
27
26
|
# If there isn't, we need to switch to the /var/tmp
|
|
28
27
|
|
|
29
|
-
if
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
session_info["size"] = result
|
|
28
|
+
if session_info["temp_dir"] == "/tmp/":
|
|
29
|
+
result = os.statvfs("/tmp/")
|
|
30
|
+
result = result.f_frsize * result.f_bavail
|
|
31
|
+
session_info["size"] = result
|
|
34
32
|
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
if os.path.getsize(ff) * 2 > session_info["size"]:
|
|
34
|
+
session_info["temp_dir"] = "/var/tmp/"
|
|
37
35
|
|
|
38
36
|
split_base = temp_file()
|
|
39
37
|
bases.append(split_base)
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import copy
|
|
2
2
|
import multiprocessing
|
|
3
|
-
import platform
|
|
4
3
|
import warnings
|
|
5
4
|
|
|
6
5
|
from nctoolkit.cleanup import cleanup
|
|
@@ -36,10 +35,6 @@ def nco_command(self, command=None, ensemble=False):
|
|
|
36
35
|
|
|
37
36
|
cores = session_info["cores"]
|
|
38
37
|
|
|
39
|
-
if platform.system() != "Linux":
|
|
40
|
-
if cores > 1:
|
|
41
|
-
warnings.warn("This cannot run in parallel on macOS")
|
|
42
|
-
cores = 1
|
|
43
38
|
|
|
44
39
|
# First, check that the command is valid
|
|
45
40
|
if command is None:
|
|
@@ -3,7 +3,6 @@ import copy
|
|
|
3
3
|
import os
|
|
4
4
|
import re
|
|
5
5
|
import subprocess
|
|
6
|
-
import platform
|
|
7
6
|
import warnings
|
|
8
7
|
|
|
9
8
|
import signal
|
|
@@ -109,19 +108,18 @@ def run_nco(command, target, out_file=None, overwrite=False):
|
|
|
109
108
|
if overwrite is False:
|
|
110
109
|
raise ValueError("Attempting to overwrite an opened file")
|
|
111
110
|
|
|
112
|
-
if
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
result = result.f_frsize * result.f_bavail
|
|
111
|
+
if session_info["temp_dir"] == "/tmp/":
|
|
112
|
+
result = os.statvfs("/tmp/")
|
|
113
|
+
result = result.f_frsize * result.f_bavail
|
|
116
114
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
115
|
+
if result < 1 * 1e9:
|
|
116
|
+
session_info["temp_dir"] == "/var/tmp/"
|
|
117
|
+
if target.startswith("/tmp"):
|
|
118
|
+
new_target = target.replace("/tmp/", "/var/tmp/")
|
|
119
|
+
command = command.replace(target, new_target)
|
|
120
|
+
remove_safe(target)
|
|
121
|
+
target = new_target
|
|
122
|
+
append_safe(target)
|
|
125
123
|
|
|
126
124
|
out = subprocess.Popen(
|
|
127
125
|
command,
|
|
@@ -209,17 +207,16 @@ def run_cdo(command=None, target=None, out_file=None, overwrite=False, precision
|
|
|
209
207
|
if overwrite is False:
|
|
210
208
|
raise ValueError("Attempting to overwrite file")
|
|
211
209
|
|
|
212
|
-
if
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
result = result.f_frsize * result.f_bavail
|
|
210
|
+
if session_info["temp_dir"] == "/tmp/":
|
|
211
|
+
result = os.statvfs("/tmp/")
|
|
212
|
+
result = result.f_frsize * result.f_bavail
|
|
216
213
|
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
214
|
+
if result < 1 * 1e9:
|
|
215
|
+
session_info["temp_dir"] == "/var/tmp/"
|
|
216
|
+
if target.startswith("/tmp"):
|
|
217
|
+
new_target = target.replace("/tmp/", "/var/tmp/")
|
|
218
|
+
command = command.replace(target, new_target)
|
|
219
|
+
target = target.replace("/tmp/", "/var/tmp/")
|
|
223
220
|
|
|
224
221
|
if command.startswith("cdo ") is False:
|
|
225
222
|
raise ValueError("The command does not start with cdo!")
|
|
@@ -850,13 +847,8 @@ def run_cdo(command=None, target=None, out_file=None, overwrite=False, precision
|
|
|
850
847
|
# detect regex pattern, based on above
|
|
851
848
|
|
|
852
849
|
for ww in w:
|
|
853
|
-
if
|
|
854
|
-
if ww.message
|
|
855
|
-
|
|
856
|
-
session_warnings.append(ww.message)
|
|
857
|
-
else:
|
|
858
|
-
warnings.warn(ww.message)
|
|
859
|
-
#print(session_warnings)
|
|
850
|
+
if ww.message not in session_warnings:
|
|
851
|
+
if ignore_warning(str(ww.message)) is False:
|
|
852
|
+
session_warnings.append(ww.message)
|
|
860
853
|
|
|
861
|
-
|
|
862
|
-
return target
|
|
854
|
+
return target
|
|
@@ -2,14 +2,10 @@ import copy
|
|
|
2
2
|
import os
|
|
3
3
|
import re
|
|
4
4
|
import subprocess
|
|
5
|
-
import platform
|
|
6
5
|
import warnings
|
|
7
6
|
from unittest.mock import patch
|
|
8
7
|
|
|
9
|
-
|
|
10
|
-
import multiprocessing as mp
|
|
11
|
-
else:
|
|
12
|
-
import multiprocess as mp
|
|
8
|
+
import multiprocessing as mp
|
|
13
9
|
|
|
14
10
|
import signal
|
|
15
11
|
|
|
@@ -247,9 +243,9 @@ def run_this(os_command, self, output="one", out_file=None, suppress=False):
|
|
|
247
243
|
if self._zip:
|
|
248
244
|
ff_command = ff_command.replace("cdo ", "cdo -z zip ")
|
|
249
245
|
|
|
250
|
-
if session_info["cores"] > 1:
|
|
251
|
-
|
|
252
|
-
|
|
246
|
+
#if session_info["cores"] > 1:
|
|
247
|
+
# if len(self) == 1:
|
|
248
|
+
# ff_command = ff_command.replace("cdo ", f"cdo -P {session_info['cores']} ")
|
|
253
249
|
new_history.append(ff_command)
|
|
254
250
|
|
|
255
251
|
#warnings.simplefilter("ignore")
|
|
@@ -291,11 +287,10 @@ def run_this(os_command, self, output="one", out_file=None, suppress=False):
|
|
|
291
287
|
if progress_bar:
|
|
292
288
|
if not suppress:
|
|
293
289
|
pbar.update(1)
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
session_warnings.remove(mm)
|
|
290
|
+
for mm in session_warnings:
|
|
291
|
+
warnings.warn(mm)
|
|
292
|
+
if mm in session_warnings:
|
|
293
|
+
session_warnings.remove(mm)
|
|
299
294
|
|
|
300
295
|
self.history = copy.deepcopy(new_history)
|
|
301
296
|
self.current = copy.deepcopy(target_list)
|
|
@@ -317,11 +312,10 @@ def run_this(os_command, self, output="one", out_file=None, suppress=False):
|
|
|
317
312
|
|
|
318
313
|
self._format = None
|
|
319
314
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
session_warnings.remove(mm)
|
|
315
|
+
for mm in session_warnings:
|
|
316
|
+
warnings.warn(mm)
|
|
317
|
+
if mm in session_warnings:
|
|
318
|
+
session_warnings.remove(mm)
|
|
325
319
|
|
|
326
320
|
return None
|
|
327
321
|
|
|
@@ -334,18 +328,17 @@ def run_this(os_command, self, output="one", out_file=None, suppress=False):
|
|
|
334
328
|
os_command = os_command.replace(" ", " ")
|
|
335
329
|
|
|
336
330
|
# ensure there is sufficient space in /tmp if it is to be used
|
|
337
|
-
|
|
338
|
-
all_sizes = 0
|
|
331
|
+
all_sizes = 0
|
|
339
332
|
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
333
|
+
for ff in self:
|
|
334
|
+
if file_size(ff) is not None:
|
|
335
|
+
all_sizes += file_size(ff)
|
|
343
336
|
|
|
344
|
-
|
|
345
|
-
|
|
337
|
+
result = os.statvfs("/tmp/")
|
|
338
|
+
result = result.f_frsize * result.f_bavail
|
|
346
339
|
|
|
347
|
-
|
|
348
|
-
|
|
340
|
+
if result < (2 * all_sizes):
|
|
341
|
+
session_info["temp_dir"] == "/var/tmp/"
|
|
349
342
|
|
|
350
343
|
target = temp_file("nc")
|
|
351
344
|
|
|
@@ -431,11 +424,10 @@ def run_this(os_command, self, output="one", out_file=None, suppress=False):
|
|
|
431
424
|
target = run_cdo(
|
|
432
425
|
os_command, target, out_file, precision=self._precision
|
|
433
426
|
)
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
session_warnings.remove(mm)
|
|
427
|
+
for mm in session_warnings:
|
|
428
|
+
warnings.warn(mm)
|
|
429
|
+
if mm in session_warnings:
|
|
430
|
+
session_warnings.remove(mm)
|
|
439
431
|
|
|
440
432
|
remove_safe(target)
|
|
441
433
|
|
|
@@ -1,20 +1,12 @@
|
|
|
1
1
|
import glob
|
|
2
2
|
import os
|
|
3
|
-
import platform
|
|
4
3
|
|
|
5
4
|
session_info = dict()
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
nc_protected_par = Manager().list()
|
|
12
|
-
else:
|
|
13
|
-
from multiprocess import Manager
|
|
14
|
-
|
|
15
|
-
nc_safe_par = Manager().list()
|
|
16
|
-
temp_dirs_par = Manager().list()
|
|
17
|
-
nc_protected_par = Manager().list()
|
|
5
|
+
from multiprocessing import Manager
|
|
6
|
+
|
|
7
|
+
nc_safe_par = Manager().list()
|
|
8
|
+
temp_dirs_par = Manager().list()
|
|
9
|
+
nc_protected_par = Manager().list()
|
|
18
10
|
|
|
19
11
|
nc_safe = list()
|
|
20
12
|
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import glob
|
|
2
2
|
import copy
|
|
3
3
|
import os
|
|
4
|
-
import platform
|
|
5
4
|
|
|
6
5
|
from nctoolkit.cleanup import cleanup
|
|
7
6
|
from nctoolkit.temp_file import temp_file
|
|
@@ -26,14 +25,13 @@ def split_cdo(self, method="year"):
|
|
|
26
25
|
# But, first we need to check if there is space in the output folder
|
|
27
26
|
# If there isn't, we need to switch to the /var/tmp
|
|
28
27
|
|
|
29
|
-
if
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
session_info["size"] = result
|
|
28
|
+
if session_info["temp_dir"] == "/tmp/":
|
|
29
|
+
result = os.statvfs("/tmp/")
|
|
30
|
+
result = result.f_frsize * result.f_bavail
|
|
31
|
+
session_info["size"] = result
|
|
34
32
|
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
if os.path.getsize(ff) * 2 > session_info["size"]:
|
|
34
|
+
session_info["temp_dir"] = "/var/tmp/"
|
|
37
35
|
|
|
38
36
|
split_base = temp_file()
|
|
39
37
|
bases.append(split_base)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import tempfile
|
|
3
|
+
|
|
4
|
+
from nctoolkit.session import session_info
|
|
5
|
+
from nctoolkit.session import append_tempdirs
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def temp_file(ext=""):
|
|
9
|
+
"""
|
|
10
|
+
Function to create a temporary file.
|
|
11
|
+
This accounts for OS
|
|
12
|
+
|
|
13
|
+
Parameters
|
|
14
|
+
--------------------
|
|
15
|
+
ext : str
|
|
16
|
+
File extension
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
# check space left in temp dir and switch it if there isn't much
|
|
20
|
+
if session_info["temp_dir"] == "/tmp/":
|
|
21
|
+
result = os.statvfs("/tmp/")
|
|
22
|
+
result = result.f_frsize * result.f_bavail
|
|
23
|
+
|
|
24
|
+
if result < 0.5 * 1e9:
|
|
25
|
+
if session_info["temp_dir"] == "/tmp/":
|
|
26
|
+
session_info["temp_dir"] = "/var/tmp/"
|
|
27
|
+
|
|
28
|
+
actual_temp = session_info["temp_dir"]
|
|
29
|
+
actual_temp = actual_temp + "/"
|
|
30
|
+
actual_temp = actual_temp.replace("//", "/")
|
|
31
|
+
|
|
32
|
+
target = tempfile.NamedTemporaryFile().name
|
|
33
|
+
target = target.replace("tmp/", "tmp/" + session_info["stamp"])
|
|
34
|
+
|
|
35
|
+
target = actual_temp + os.path.basename(target)
|
|
36
|
+
if not isinstance(ext, str):
|
|
37
|
+
raise TypeError("Extension supplied is not a str")
|
|
38
|
+
if ext.startswith("."):
|
|
39
|
+
target = target + ext
|
|
40
|
+
else:
|
|
41
|
+
target = f"{target}.{ext}"
|
|
42
|
+
|
|
43
|
+
append_tempdirs(os.path.dirname(target))
|
|
44
|
+
|
|
45
|
+
return target
|
|
46
|
+
|
|
@@ -314,6 +314,11 @@ def vertical_mean(self, thickness=None, depth_range=None, fixed=None):
|
|
|
314
314
|
ds_thick = open_data(thickness)
|
|
315
315
|
if len(ds_thick.variables) != 1:
|
|
316
316
|
raise ValueError("Please provide a thickness file with 1 variable!")
|
|
317
|
+
ds_mask = open_data(self[0])
|
|
318
|
+
ds_mask.subset(variables=ds_mask.variables[0], time = 0)
|
|
319
|
+
ds_mask.abs()
|
|
320
|
+
ds_mask > 0
|
|
321
|
+
ds_thick * ds_mask
|
|
317
322
|
|
|
318
323
|
thick_var = ds_thick.variables[0]
|
|
319
324
|
# modify the depth if it is a list
|
|
@@ -4,7 +4,7 @@ import sys
|
|
|
4
4
|
DESCRIPTION = "A Python package for netCDF analysis and post-processing"
|
|
5
5
|
LONG_DESCRIPTION = """
|
|
6
6
|
|
|
7
|
-
**nctoolkit** is a comprehensive Python (3.
|
|
7
|
+
**nctoolkit** is a comprehensive Python (3.10 and above) package for analyzing netCDF data on Linux and macOS.
|
|
8
8
|
|
|
9
9
|
Core abilities of nctoolkit include:
|
|
10
10
|
|
|
@@ -46,7 +46,7 @@ extras_require["complete"] = ["geoviews", "rioxarray", "cfchecker", "geocube", "
|
|
|
46
46
|
REQUIREMENTS = [i.strip() for i in open("requirements.txt").readlines()]
|
|
47
47
|
|
|
48
48
|
setup(name='nctoolkit',
|
|
49
|
-
version='1.
|
|
49
|
+
version='1.3.0',
|
|
50
50
|
description=DESCRIPTION,
|
|
51
51
|
description_content_type='text/plain',
|
|
52
52
|
long_description=LONG_DESCRIPTION,
|
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
import tempfile
|
|
3
|
-
import platform
|
|
4
|
-
|
|
5
|
-
from nctoolkit.session import session_info
|
|
6
|
-
from nctoolkit.session import append_tempdirs
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def temp_file(ext=""):
|
|
10
|
-
"""
|
|
11
|
-
Function to create a temporary file.
|
|
12
|
-
This accounts for OS
|
|
13
|
-
|
|
14
|
-
Parameters
|
|
15
|
-
--------------------
|
|
16
|
-
ext : str
|
|
17
|
-
File extension
|
|
18
|
-
"""
|
|
19
|
-
|
|
20
|
-
# this needs to work differently on Linux
|
|
21
|
-
if platform.system() == "Linux":
|
|
22
|
-
# check space left in temp dir and switch it if there isn't much
|
|
23
|
-
if session_info["temp_dir"] == "/tmp/":
|
|
24
|
-
result = os.statvfs("/tmp/")
|
|
25
|
-
result = result.f_frsize * result.f_bavail
|
|
26
|
-
|
|
27
|
-
if result < 0.5 * 1e9:
|
|
28
|
-
if session_info["temp_dir"] == "/tmp/":
|
|
29
|
-
session_info["temp_dir"] = "/var/tmp/"
|
|
30
|
-
|
|
31
|
-
actual_temp = session_info["temp_dir"]
|
|
32
|
-
actual_temp = actual_temp + "/"
|
|
33
|
-
actual_temp = actual_temp.replace("//", "/")
|
|
34
|
-
|
|
35
|
-
target = tempfile.NamedTemporaryFile().name
|
|
36
|
-
target = target.replace("tmp/", "tmp/" + session_info["stamp"])
|
|
37
|
-
|
|
38
|
-
target = actual_temp + os.path.basename(target)
|
|
39
|
-
if not isinstance(ext, str):
|
|
40
|
-
raise TypeError("Extension supplied is not a str")
|
|
41
|
-
if ext.startswith("."):
|
|
42
|
-
target = target + ext
|
|
43
|
-
else:
|
|
44
|
-
target = f"{target}.{ext}"
|
|
45
|
-
|
|
46
|
-
append_tempdirs(os.path.dirname(target))
|
|
47
|
-
|
|
48
|
-
return target
|
|
49
|
-
|
|
50
|
-
# osx approach
|
|
51
|
-
else:
|
|
52
|
-
target = tempfile.NamedTemporaryFile().name
|
|
53
|
-
|
|
54
|
-
if not isinstance(ext, str):
|
|
55
|
-
raise TypeError("Extension supplied is not a str")
|
|
56
|
-
if ext.startswith("."):
|
|
57
|
-
target = target + ext
|
|
58
|
-
else:
|
|
59
|
-
target = f"{target}.{ext}"
|
|
60
|
-
|
|
61
|
-
append_tempdirs(os.path.dirname(target))
|
|
62
|
-
|
|
63
|
-
target = target.replace(
|
|
64
|
-
os.path.dirname(target) + "/",
|
|
65
|
-
os.path.dirname(target) + "/" + session_info["stamp"],
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
return target
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|