cht_utils 2.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cht_utils/__init__.py +28 -0
- cht_utils/cog/__init__.py +6 -0
- cht_utils/cog/geotiff_to_cog.py +79 -0
- cht_utils/cog/netcdf_to_cog.py +85 -0
- cht_utils/cog/xyz_to_cog.py +86 -0
- cht_utils/colors/__init__.py +6 -0
- cht_utils/colors/colors.py +117 -0
- cht_utils/fileio/__init__.py +21 -0
- cht_utils/fileio/deltares_ini.py +326 -0
- cht_utils/fileio/json_js.py +72 -0
- cht_utils/fileio/pli_file.py +233 -0
- cht_utils/fileio/tekal.py +234 -0
- cht_utils/fileio/xml.py +184 -0
- cht_utils/fileio/yaml.py +39 -0
- cht_utils/fileops/__init__.py +25 -0
- cht_utils/fileops/fileops.py +344 -0
- cht_utils/interpolation/__init__.py +5 -0
- cht_utils/interpolation/interpolation.py +152 -0
- cht_utils/maps/__init__.py +2 -0
- cht_utils/maps/fileops.py +191 -0
- cht_utils/maps/flood_map.py +1231 -0
- cht_utils/maps/topobathy_map.py +463 -0
- cht_utils/maps/utils.py +700 -0
- cht_utils/physics/__init__.py +8 -0
- cht_utils/physics/deshoal.py +63 -0
- cht_utils/physics/disper.py +91 -0
- cht_utils/physics/runup_vo21.py +229 -0
- cht_utils/physics/waves.py +59 -0
- cht_utils/probabilistic/__init__.py +5 -0
- cht_utils/probabilistic/prob_maps.py +263 -0
- cht_utils/remote/__init__.py +4 -0
- cht_utils/remote/s3.py +380 -0
- cht_utils/remote/sftp.py +192 -0
- cht_utils-2.0.0.dist-info/METADATA +30 -0
- cht_utils-2.0.0.dist-info/RECORD +39 -0
- cht_utils-2.0.0.dist-info/WHEEL +5 -0
- cht_utils-2.0.0.dist-info/licenses/LICENSE +21 -0
- cht_utils-2.0.0.dist-info/top_level.txt +1 -0
- cht_utils-2.0.0.dist-info/zip-safe +1 -0
cht_utils/remote/s3.py
ADDED
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
"""AWS S3 client wrapper for file and folder operations."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import pathlib
|
|
5
|
+
import tarfile
|
|
6
|
+
from multiprocessing.pool import ThreadPool
|
|
7
|
+
from typing import List
|
|
8
|
+
|
|
9
|
+
import boto3
|
|
10
|
+
from botocore.exceptions import ClientError
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class S3Session:
|
|
14
|
+
"""AWS S3 session with upload, download, and management operations.
|
|
15
|
+
|
|
16
|
+
Parameters
|
|
17
|
+
----------
|
|
18
|
+
access_key : str
|
|
19
|
+
AWS access key ID.
|
|
20
|
+
secret_key : str
|
|
21
|
+
AWS secret access key.
|
|
22
|
+
region : str
|
|
23
|
+
AWS region name.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, access_key: str, secret_key: str, region: str) -> None:
|
|
27
|
+
self.ready = True
|
|
28
|
+
session = boto3.Session(
|
|
29
|
+
aws_access_key_id=access_key,
|
|
30
|
+
aws_secret_access_key=secret_key,
|
|
31
|
+
region_name=region,
|
|
32
|
+
)
|
|
33
|
+
self.s3_client = session.client("s3")
|
|
34
|
+
|
|
35
|
+
def upload_file(
|
|
36
|
+
self, bucket_name: str, file: str, s3_folder: str, quiet: bool = True
|
|
37
|
+
) -> None:
|
|
38
|
+
"""Upload a local file to S3.
|
|
39
|
+
|
|
40
|
+
Parameters
|
|
41
|
+
----------
|
|
42
|
+
bucket_name : str
|
|
43
|
+
S3 bucket name.
|
|
44
|
+
file : str
|
|
45
|
+
Local file path.
|
|
46
|
+
s3_folder : str
|
|
47
|
+
Remote folder prefix.
|
|
48
|
+
quiet : bool
|
|
49
|
+
Suppress progress output.
|
|
50
|
+
"""
|
|
51
|
+
s3_key = os.path.join(s3_folder, os.path.basename(file)).replace("\\", "/")
|
|
52
|
+
self.s3_client.upload_file(file, bucket_name, s3_key)
|
|
53
|
+
if not quiet:
|
|
54
|
+
print(f"Uploaded {os.path.basename(file)}")
|
|
55
|
+
|
|
56
|
+
def download_file(
|
|
57
|
+
self,
|
|
58
|
+
bucket_name: str,
|
|
59
|
+
s3_folder: str,
|
|
60
|
+
file: str,
|
|
61
|
+
local_folder: str,
|
|
62
|
+
quiet: bool = True,
|
|
63
|
+
) -> None:
|
|
64
|
+
"""Download a file from S3.
|
|
65
|
+
|
|
66
|
+
Parameters
|
|
67
|
+
----------
|
|
68
|
+
bucket_name : str
|
|
69
|
+
S3 bucket name.
|
|
70
|
+
s3_folder : str
|
|
71
|
+
Remote folder prefix.
|
|
72
|
+
file : str
|
|
73
|
+
File name to download.
|
|
74
|
+
local_folder : str
|
|
75
|
+
Local destination directory.
|
|
76
|
+
quiet : bool
|
|
77
|
+
Suppress progress output.
|
|
78
|
+
"""
|
|
79
|
+
s3_key = os.path.join(s3_folder, os.path.basename(file)).replace("\\", "/")
|
|
80
|
+
local_path = os.path.join(local_folder, os.path.basename(file))
|
|
81
|
+
self.s3_client.download_file(bucket_name, s3_key, local_path)
|
|
82
|
+
if not quiet:
|
|
83
|
+
print(f"Downloaded {os.path.basename(file)}")
|
|
84
|
+
|
|
85
|
+
def download_files(
|
|
86
|
+
self,
|
|
87
|
+
bucket_name: str,
|
|
88
|
+
key_list: List[str],
|
|
89
|
+
file_list: List[str],
|
|
90
|
+
quiet: bool = True,
|
|
91
|
+
) -> None:
|
|
92
|
+
"""Download multiple files in parallel.
|
|
93
|
+
|
|
94
|
+
Parameters
|
|
95
|
+
----------
|
|
96
|
+
bucket_name : str
|
|
97
|
+
S3 bucket name.
|
|
98
|
+
key_list : List[str]
|
|
99
|
+
List of S3 keys.
|
|
100
|
+
file_list : List[str]
|
|
101
|
+
Corresponding local file paths.
|
|
102
|
+
quiet : bool
|
|
103
|
+
Suppress progress output.
|
|
104
|
+
"""
|
|
105
|
+
pool = ThreadPool()
|
|
106
|
+
pool.starmap(
|
|
107
|
+
self.download_file_parallel,
|
|
108
|
+
[(bucket_name, key, file, quiet) for key, file in zip(key_list, file_list)],
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
def download_file_parallel(
|
|
112
|
+
self, bucket_name: str, s3_key: str, file: str, quiet: bool = True
|
|
113
|
+
) -> None:
|
|
114
|
+
"""Download a single file (used for parallel execution).
|
|
115
|
+
|
|
116
|
+
Parameters
|
|
117
|
+
----------
|
|
118
|
+
bucket_name : str
|
|
119
|
+
S3 bucket name.
|
|
120
|
+
s3_key : str
|
|
121
|
+
Full S3 key.
|
|
122
|
+
file : str
|
|
123
|
+
Local file path.
|
|
124
|
+
quiet : bool
|
|
125
|
+
Suppress progress output.
|
|
126
|
+
"""
|
|
127
|
+
self.s3_client.download_file(bucket_name, s3_key, file)
|
|
128
|
+
if not quiet:
|
|
129
|
+
print(f"Downloaded {os.path.basename(file)}")
|
|
130
|
+
|
|
131
|
+
def delete_file(
|
|
132
|
+
self, bucket_name: str, s3_folder: str, file: str, quiet: bool = True
|
|
133
|
+
) -> None:
|
|
134
|
+
"""Delete a file from S3.
|
|
135
|
+
|
|
136
|
+
Parameters
|
|
137
|
+
----------
|
|
138
|
+
bucket_name : str
|
|
139
|
+
S3 bucket name.
|
|
140
|
+
s3_folder : str
|
|
141
|
+
Remote folder prefix.
|
|
142
|
+
file : str
|
|
143
|
+
File name to delete.
|
|
144
|
+
quiet : bool
|
|
145
|
+
Suppress progress output.
|
|
146
|
+
"""
|
|
147
|
+
s3_key = os.path.join(s3_folder, os.path.basename(file)).replace("\\", "/")
|
|
148
|
+
self.s3_client.delete_object(Bucket=bucket_name, Key=s3_key)
|
|
149
|
+
if not quiet:
|
|
150
|
+
print(f"Deleted {os.path.basename(file)}")
|
|
151
|
+
|
|
152
|
+
def make_folder(self, bucket_name: str, s3_folder: str, quiet: bool = True) -> None:
|
|
153
|
+
"""Create a folder (prefix) in S3.
|
|
154
|
+
|
|
155
|
+
Parameters
|
|
156
|
+
----------
|
|
157
|
+
bucket_name : str
|
|
158
|
+
S3 bucket name.
|
|
159
|
+
s3_folder : str
|
|
160
|
+
Folder prefix to create.
|
|
161
|
+
quiet : bool
|
|
162
|
+
Suppress progress output.
|
|
163
|
+
"""
|
|
164
|
+
self.s3_client.put_object(Bucket=bucket_name, Key=f"{s3_folder}/")
|
|
165
|
+
if not quiet:
|
|
166
|
+
print(f"Made folder: {s3_folder}")
|
|
167
|
+
|
|
168
|
+
def upload_folder(
|
|
169
|
+
self,
|
|
170
|
+
bucket_name: str,
|
|
171
|
+
local_folder: str,
|
|
172
|
+
s3_folder: str,
|
|
173
|
+
parallel: bool = True,
|
|
174
|
+
quiet: bool = True,
|
|
175
|
+
) -> None:
|
|
176
|
+
"""Recursively upload a local directory to S3.
|
|
177
|
+
|
|
178
|
+
Parameters
|
|
179
|
+
----------
|
|
180
|
+
bucket_name : str
|
|
181
|
+
S3 bucket name.
|
|
182
|
+
local_folder : str
|
|
183
|
+
Local directory to upload.
|
|
184
|
+
s3_folder : str
|
|
185
|
+
Remote folder prefix.
|
|
186
|
+
parallel : bool
|
|
187
|
+
Use parallel uploads.
|
|
188
|
+
quiet : bool
|
|
189
|
+
Suppress progress output.
|
|
190
|
+
"""
|
|
191
|
+
local_folder = local_folder.replace("\\", "/")
|
|
192
|
+
flist = _list_all_files(local_folder)
|
|
193
|
+
args = [
|
|
194
|
+
(f, local_folder, s3_folder, bucket_name, self.s3_client, quiet)
|
|
195
|
+
for f in flist
|
|
196
|
+
]
|
|
197
|
+
if parallel:
|
|
198
|
+
pool = ThreadPool()
|
|
199
|
+
pool.starmap(_upload_file, args)
|
|
200
|
+
else:
|
|
201
|
+
for a in args:
|
|
202
|
+
_upload_file(*a)
|
|
203
|
+
|
|
204
|
+
def download_folder(
|
|
205
|
+
self,
|
|
206
|
+
bucket_name: str,
|
|
207
|
+
s3_folder: str,
|
|
208
|
+
local_folder: str,
|
|
209
|
+
parallel: bool = True,
|
|
210
|
+
quiet: bool = True,
|
|
211
|
+
) -> None:
|
|
212
|
+
"""Download all files under an S3 prefix to a local directory.
|
|
213
|
+
|
|
214
|
+
Parameters
|
|
215
|
+
----------
|
|
216
|
+
bucket_name : str
|
|
217
|
+
S3 bucket name.
|
|
218
|
+
s3_folder : str
|
|
219
|
+
Remote folder prefix.
|
|
220
|
+
local_folder : str
|
|
221
|
+
Local destination directory.
|
|
222
|
+
parallel : bool
|
|
223
|
+
Currently unused (sequential download).
|
|
224
|
+
quiet : bool
|
|
225
|
+
Suppress progress output.
|
|
226
|
+
"""
|
|
227
|
+
os.makedirs(local_folder, exist_ok=True)
|
|
228
|
+
objects = self.s3_client.list_objects(Bucket=bucket_name, Prefix=s3_folder)
|
|
229
|
+
for obj in objects.get("Contents", []):
|
|
230
|
+
s3_key = obj["Key"]
|
|
231
|
+
local_path = os.path.join(local_folder, os.path.basename(s3_key))
|
|
232
|
+
self.s3_client.download_file(bucket_name, s3_key, local_path)
|
|
233
|
+
if not quiet:
|
|
234
|
+
print(f"Downloaded {os.path.basename(s3_key)}")
|
|
235
|
+
|
|
236
|
+
def delete_folder(self, bucket_name: str, s3_folder: str) -> None:
|
|
237
|
+
"""Delete all objects under an S3 prefix.
|
|
238
|
+
|
|
239
|
+
Parameters
|
|
240
|
+
----------
|
|
241
|
+
bucket_name : str
|
|
242
|
+
S3 bucket name.
|
|
243
|
+
s3_folder : str
|
|
244
|
+
Remote folder prefix.
|
|
245
|
+
"""
|
|
246
|
+
if not s3_folder.endswith("/"):
|
|
247
|
+
s3_folder += "/"
|
|
248
|
+
objects = self.s3_client.list_objects(Bucket=bucket_name, Prefix=s3_folder)
|
|
249
|
+
for obj in objects.get("Contents", []):
|
|
250
|
+
self.s3_client.delete_object(Bucket=bucket_name, Key=obj["Key"])
|
|
251
|
+
|
|
252
|
+
def list_folders(self, bucket_name: str, s3_folder: str) -> List[str]:
|
|
253
|
+
"""List immediate subfolders under an S3 prefix.
|
|
254
|
+
|
|
255
|
+
Parameters
|
|
256
|
+
----------
|
|
257
|
+
bucket_name : str
|
|
258
|
+
S3 bucket name.
|
|
259
|
+
s3_folder : str
|
|
260
|
+
Remote folder prefix.
|
|
261
|
+
|
|
262
|
+
Returns
|
|
263
|
+
-------
|
|
264
|
+
List[str]
|
|
265
|
+
Subfolder names.
|
|
266
|
+
"""
|
|
267
|
+
if not s3_folder.endswith("/"):
|
|
268
|
+
s3_folder += "/"
|
|
269
|
+
folders = []
|
|
270
|
+
paginator = self.s3_client.get_paginator("list_objects_v2")
|
|
271
|
+
for page in paginator.paginate(
|
|
272
|
+
Bucket=bucket_name, Prefix=s3_folder, Delimiter="/"
|
|
273
|
+
):
|
|
274
|
+
for subfolder in page.get("CommonPrefixes", []):
|
|
275
|
+
folders.append(subfolder["Prefix"].rstrip("/").split("/")[-1])
|
|
276
|
+
return folders
|
|
277
|
+
|
|
278
|
+
def list_files(self, bucket_name: str, s3_folder: str) -> List[str]:
|
|
279
|
+
"""List all file keys under an S3 prefix.
|
|
280
|
+
|
|
281
|
+
Parameters
|
|
282
|
+
----------
|
|
283
|
+
bucket_name : str
|
|
284
|
+
S3 bucket name.
|
|
285
|
+
s3_folder : str
|
|
286
|
+
Remote folder prefix.
|
|
287
|
+
|
|
288
|
+
Returns
|
|
289
|
+
-------
|
|
290
|
+
List[str]
|
|
291
|
+
Full S3 keys.
|
|
292
|
+
"""
|
|
293
|
+
all_files = []
|
|
294
|
+
paginator = self.s3_client.get_paginator("list_objects_v2")
|
|
295
|
+
for page in paginator.paginate(Bucket=bucket_name, Prefix=s3_folder):
|
|
296
|
+
for obj in page.get("Contents", []):
|
|
297
|
+
all_files.append(obj["Key"])
|
|
298
|
+
return all_files
|
|
299
|
+
|
|
300
|
+
def download_and_extract_tgz(
|
|
301
|
+
self, bucket_name: str, s3_folder: str, local_folder: str
|
|
302
|
+
) -> None:
|
|
303
|
+
"""Download and extract a ``.tgz`` archive from S3.
|
|
304
|
+
|
|
305
|
+
Parameters
|
|
306
|
+
----------
|
|
307
|
+
bucket_name : str
|
|
308
|
+
S3 bucket name.
|
|
309
|
+
s3_folder : str
|
|
310
|
+
S3 key of the ``.tgz`` file.
|
|
311
|
+
local_folder : str
|
|
312
|
+
Local directory to extract into.
|
|
313
|
+
"""
|
|
314
|
+
local_tgz_path = os.path.join("/tmp", os.path.basename(s3_folder))
|
|
315
|
+
self.s3_client.download_file(bucket_name, s3_folder, local_tgz_path)
|
|
316
|
+
with tarfile.open(local_tgz_path, "r:gz") as tar:
|
|
317
|
+
tar.extractall(path=local_folder)
|
|
318
|
+
os.remove(local_tgz_path)
|
|
319
|
+
|
|
320
|
+
def check_file_exists(self, bucket_name: str, s3_key: str) -> bool:
|
|
321
|
+
"""Check if a file exists in S3.
|
|
322
|
+
|
|
323
|
+
Parameters
|
|
324
|
+
----------
|
|
325
|
+
bucket_name : str
|
|
326
|
+
S3 bucket name.
|
|
327
|
+
s3_key : str
|
|
328
|
+
Full S3 key.
|
|
329
|
+
|
|
330
|
+
Returns
|
|
331
|
+
-------
|
|
332
|
+
bool
|
|
333
|
+
"""
|
|
334
|
+
try:
|
|
335
|
+
self.s3_client.head_object(Bucket=bucket_name, Key=s3_key)
|
|
336
|
+
return True
|
|
337
|
+
except ClientError as e:
|
|
338
|
+
if e.response["Error"]["Code"] == "404":
|
|
339
|
+
return False
|
|
340
|
+
raise
|
|
341
|
+
|
|
342
|
+
def check_folder_exists(self, bucket_name: str, s3_key: str) -> bool:
|
|
343
|
+
"""Check if a folder (prefix) exists in S3.
|
|
344
|
+
|
|
345
|
+
Parameters
|
|
346
|
+
----------
|
|
347
|
+
bucket_name : str
|
|
348
|
+
S3 bucket name.
|
|
349
|
+
s3_key : str
|
|
350
|
+
S3 prefix to check.
|
|
351
|
+
|
|
352
|
+
Returns
|
|
353
|
+
-------
|
|
354
|
+
bool
|
|
355
|
+
"""
|
|
356
|
+
response = self.s3_client.list_objects_v2(
|
|
357
|
+
Bucket=bucket_name, Prefix=s3_key, Delimiter="/"
|
|
358
|
+
)
|
|
359
|
+
return "CommonPrefixes" in response
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _list_all_files(src: str) -> List[str]:
|
|
363
|
+
"""Recursively list all files in a directory."""
|
|
364
|
+
return [str(f) for f in pathlib.Path(src).rglob("*") if f.is_file()]
|
|
365
|
+
|
|
366
|
+
|
|
367
|
+
def _upload_file(
|
|
368
|
+
file: str,
|
|
369
|
+
local_folder: str,
|
|
370
|
+
s3_folder: str,
|
|
371
|
+
bucket_name: str,
|
|
372
|
+
s3_client: object,
|
|
373
|
+
quiet: bool,
|
|
374
|
+
) -> None:
|
|
375
|
+
"""Upload a single file, preserving relative path structure."""
|
|
376
|
+
relative = file.replace("\\", "/").replace(local_folder, "")
|
|
377
|
+
s3_key = s3_folder + relative
|
|
378
|
+
s3_client.upload_file(file, bucket_name, s3_key)
|
|
379
|
+
if not quiet:
|
|
380
|
+
print(f"Uploaded {file}")
|
cht_utils/remote/sftp.py
ADDED
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
"""SSH/SFTP client for remote file operations."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import posixpath
|
|
5
|
+
import socket
|
|
6
|
+
from stat import S_ISDIR
|
|
7
|
+
from typing import Optional
|
|
8
|
+
|
|
9
|
+
import paramiko
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class SSHSession:
|
|
13
|
+
"""SSH session with SFTP file transfer capabilities.
|
|
14
|
+
|
|
15
|
+
Parameters
|
|
16
|
+
----------
|
|
17
|
+
hostname : str
|
|
18
|
+
Remote host address.
|
|
19
|
+
username : str
|
|
20
|
+
SSH username.
|
|
21
|
+
key_file : str or None
|
|
22
|
+
Path to private key file (not yet implemented).
|
|
23
|
+
password : str or None
|
|
24
|
+
SSH password.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(
|
|
28
|
+
self,
|
|
29
|
+
hostname: str,
|
|
30
|
+
username: str = "root",
|
|
31
|
+
key_file: Optional[str] = None,
|
|
32
|
+
password: Optional[str] = None,
|
|
33
|
+
) -> None:
|
|
34
|
+
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
35
|
+
self.sock.connect((hostname, 22))
|
|
36
|
+
self.t = paramiko.Transport(self.sock)
|
|
37
|
+
self.t.start_client()
|
|
38
|
+
self.t.auth_password(username, password, fallback=False)
|
|
39
|
+
self.sftp = paramiko.SFTPClient.from_transport(self.t)
|
|
40
|
+
|
|
41
|
+
def command(self, cmd: str) -> str:
|
|
42
|
+
"""Execute a remote command.
|
|
43
|
+
|
|
44
|
+
Parameters
|
|
45
|
+
----------
|
|
46
|
+
cmd : str
|
|
47
|
+
Command string (may contain newlines for multiple commands).
|
|
48
|
+
|
|
49
|
+
Returns
|
|
50
|
+
-------
|
|
51
|
+
str
|
|
52
|
+
Combined server response.
|
|
53
|
+
"""
|
|
54
|
+
chan = self.t.open_session()
|
|
55
|
+
chan.get_pty()
|
|
56
|
+
chan.invoke_shell()
|
|
57
|
+
chan.settimeout(20.0)
|
|
58
|
+
ret = ""
|
|
59
|
+
try:
|
|
60
|
+
ret += chan.recv(1024).decode()
|
|
61
|
+
except Exception:
|
|
62
|
+
chan.send("\n")
|
|
63
|
+
ret += chan.recv(1024).decode()
|
|
64
|
+
for line in cmd.split("\n"):
|
|
65
|
+
chan.send(line.strip() + "\n")
|
|
66
|
+
ret += chan.recv(1024).decode()
|
|
67
|
+
return ret
|
|
68
|
+
|
|
69
|
+
def put(self, localfile: str, remotefile: str) -> None:
|
|
70
|
+
"""Upload a file.
|
|
71
|
+
|
|
72
|
+
Parameters
|
|
73
|
+
----------
|
|
74
|
+
localfile : str
|
|
75
|
+
Local file path.
|
|
76
|
+
remotefile : str
|
|
77
|
+
Remote destination path.
|
|
78
|
+
"""
|
|
79
|
+
self.sftp.put(localfile, remotefile)
|
|
80
|
+
|
|
81
|
+
def put_all(self, localpath: str, remotepath: str) -> None:
|
|
82
|
+
"""Recursively upload a directory.
|
|
83
|
+
|
|
84
|
+
Parameters
|
|
85
|
+
----------
|
|
86
|
+
localpath : str
|
|
87
|
+
Local directory to upload.
|
|
88
|
+
remotepath : str
|
|
89
|
+
Remote destination directory.
|
|
90
|
+
"""
|
|
91
|
+
current_path = os.getcwd()
|
|
92
|
+
os.chdir(os.path.split(localpath)[0])
|
|
93
|
+
parent = os.path.split(localpath)[1]
|
|
94
|
+
for walker in os.walk(parent):
|
|
95
|
+
try:
|
|
96
|
+
self.sftp.mkdir(remotepath + "/" + walker[0].replace("\\", "/"))
|
|
97
|
+
except OSError:
|
|
98
|
+
pass
|
|
99
|
+
for file in walker[2]:
|
|
100
|
+
remote_file = f"{remotepath}/{walker[0]}/{file}".replace("\\", "/")
|
|
101
|
+
self.put(os.path.join(walker[0], file), remote_file)
|
|
102
|
+
os.chdir(current_path)
|
|
103
|
+
|
|
104
|
+
def get(self, remotefile: str, localfile: str) -> None:
|
|
105
|
+
"""Download a file.
|
|
106
|
+
|
|
107
|
+
Parameters
|
|
108
|
+
----------
|
|
109
|
+
remotefile : str
|
|
110
|
+
Remote file path.
|
|
111
|
+
localfile : str
|
|
112
|
+
Local destination path.
|
|
113
|
+
"""
|
|
114
|
+
self.sftp.get(remotefile, localfile)
|
|
115
|
+
|
|
116
|
+
def sftp_walk(self, remotepath: str):
|
|
117
|
+
"""Walk a remote directory tree (like ``os.walk``).
|
|
118
|
+
|
|
119
|
+
Parameters
|
|
120
|
+
----------
|
|
121
|
+
remotepath : str
|
|
122
|
+
Remote directory path.
|
|
123
|
+
|
|
124
|
+
Yields
|
|
125
|
+
------
|
|
126
|
+
tuple of (str, list, list)
|
|
127
|
+
``(path, folders, files)`` for each directory level.
|
|
128
|
+
"""
|
|
129
|
+
files = []
|
|
130
|
+
folders = []
|
|
131
|
+
for f in self.sftp.listdir_attr(remotepath):
|
|
132
|
+
if S_ISDIR(f.st_mode):
|
|
133
|
+
folders.append(f.filename)
|
|
134
|
+
else:
|
|
135
|
+
files.append(f.filename)
|
|
136
|
+
yield remotepath, folders, files
|
|
137
|
+
for folder in folders:
|
|
138
|
+
new_path = os.path.join(remotepath, folder).replace("\\", "/")
|
|
139
|
+
yield from self.sftp_walk(new_path)
|
|
140
|
+
|
|
141
|
+
def get_all(self, remotepath: str, localpath: str) -> None:
|
|
142
|
+
"""Recursively download a directory.
|
|
143
|
+
|
|
144
|
+
Parameters
|
|
145
|
+
----------
|
|
146
|
+
remotepath : str
|
|
147
|
+
Remote directory path.
|
|
148
|
+
localpath : str
|
|
149
|
+
Local destination directory.
|
|
150
|
+
"""
|
|
151
|
+
self.sftp.chdir(os.path.split(remotepath)[0])
|
|
152
|
+
parent = os.path.split(remotepath)[1]
|
|
153
|
+
os.makedirs(localpath, exist_ok=True)
|
|
154
|
+
for walker in self.sftp_walk(parent):
|
|
155
|
+
local_dir = os.path.join(localpath, walker[0])
|
|
156
|
+
os.makedirs(local_dir, exist_ok=True)
|
|
157
|
+
for file in walker[2]:
|
|
158
|
+
self.get(
|
|
159
|
+
os.path.join(walker[0], file),
|
|
160
|
+
os.path.join(localpath, walker[0], file),
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
def write_command(self, text: str, remotefile: str) -> None:
|
|
164
|
+
"""Write text to a remote file and make it executable.
|
|
165
|
+
|
|
166
|
+
Parameters
|
|
167
|
+
----------
|
|
168
|
+
text : str
|
|
169
|
+
Script content.
|
|
170
|
+
remotefile : str
|
|
171
|
+
Remote file path.
|
|
172
|
+
"""
|
|
173
|
+
self.sftp.open(remotefile, "w").write(text)
|
|
174
|
+
self.sftp.chmod(remotefile, 0o755)
|
|
175
|
+
|
|
176
|
+
def rmtree(self, remotepath: str, level: int = 0) -> None:
|
|
177
|
+
"""Recursively delete a remote directory tree.
|
|
178
|
+
|
|
179
|
+
Parameters
|
|
180
|
+
----------
|
|
181
|
+
remotepath : str
|
|
182
|
+
Remote directory to remove.
|
|
183
|
+
level : int
|
|
184
|
+
Recursion depth (internal use).
|
|
185
|
+
"""
|
|
186
|
+
for f in self.sftp.listdir_attr(remotepath):
|
|
187
|
+
rpath = posixpath.join(remotepath, f.filename)
|
|
188
|
+
if S_ISDIR(f.st_mode):
|
|
189
|
+
self.rmtree(rpath, level=level + 1)
|
|
190
|
+
else:
|
|
191
|
+
self.sftp.remove(rpath)
|
|
192
|
+
self.sftp.rmdir(remotepath)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cht_utils
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Utility functions for the Coastal Hazards Toolkit
|
|
5
|
+
Author-email: Maarten van Ormondt <maarten.vanormondt@deltares-usa.us>
|
|
6
|
+
License: MIT
|
|
7
|
+
Classifier: Intended Audience :: Science/Research
|
|
8
|
+
Classifier: Topic :: Scientific/Engineering :: Hydrology
|
|
9
|
+
Requires-Python: >=3.10
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: boto3
|
|
13
|
+
Requires-Dist: contextily
|
|
14
|
+
Requires-Dist: geopandas
|
|
15
|
+
Requires-Dist: numpy
|
|
16
|
+
Requires-Dist: matplotlib
|
|
17
|
+
Requires-Dist: pandas
|
|
18
|
+
Requires-Dist: paramiko
|
|
19
|
+
Requires-Dist: pillow
|
|
20
|
+
Requires-Dist: pyproj
|
|
21
|
+
Requires-Dist: pyyaml
|
|
22
|
+
Requires-Dist: scipy
|
|
23
|
+
Requires-Dist: shapely
|
|
24
|
+
Requires-Dist: rioxarray
|
|
25
|
+
Requires-Dist: rasterio
|
|
26
|
+
Requires-Dist: xarray
|
|
27
|
+
Provides-Extra: tests
|
|
28
|
+
Requires-Dist: pytest; extra == "tests"
|
|
29
|
+
Requires-Dist: pytest-cov; extra == "tests"
|
|
30
|
+
Dynamic: license-file
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
cht_utils/__init__.py,sha256=Mq_iKuJxMRqFkQ9KhfWS7JR7y0ikebU5B0q0qYMWzik,779
|
|
2
|
+
cht_utils/cog/__init__.py,sha256=1PGmiW7V4OsJDM4PtRhwyNvZaetms6QDYtGIwjGFSy8,324
|
|
3
|
+
cht_utils/cog/geotiff_to_cog.py,sha256=j5xKZWMGBxPPu9UFfRCWsAAza3RS8Q_41Qrd4pKiLO4,1943
|
|
4
|
+
cht_utils/cog/netcdf_to_cog.py,sha256=PoODYmPq7QtdhCNqYWzaO2p9MEa56O1RV-viW2HizjQ,2304
|
|
5
|
+
cht_utils/cog/xyz_to_cog.py,sha256=f1sBksY4gLsoGya3L-jmq4mpoXI9dng0KoljAn_ope0,2380
|
|
6
|
+
cht_utils/colors/__init__.py,sha256=1flEinYzE711LC0jFgqt2ExLOjIak9A-CnwaJ4amyhE,323
|
|
7
|
+
cht_utils/colors/colors.py,sha256=jYHE4ZtyuPi3iQ79ABTHY4teAJK-W-9rBoJqSdj05vs,3021
|
|
8
|
+
cht_utils/fileio/__init__.py,sha256=iwD_-5oTaImACmGNx-3G9OyEgzwKix49P23FlgUq1p8,1212
|
|
9
|
+
cht_utils/fileio/deltares_ini.py,sha256=6yRW3xs4GRfnvabGOnFkh1PD5aJL1bb7vPfzikLzfNc,9908
|
|
10
|
+
cht_utils/fileio/json_js.py,sha256=uUhaBsf4vNZ9hUimOlRL3-nvCF50hLfQHgwimc8X2sM,1925
|
|
11
|
+
cht_utils/fileio/pli_file.py,sha256=nj0tNAERBicl1WZCNg8pF1NbYn382d6ZncZLQTUumfE,5940
|
|
12
|
+
cht_utils/fileio/tekal.py,sha256=jUuGxbTFZGVqX13rUQzTGIoNd3CGNA4H95Q_Ejy-Ytg,6487
|
|
13
|
+
cht_utils/fileio/xml.py,sha256=BTLScILYv4anmDdL5TReJ8CkQHbTD4hm0N4OkI2IMQI,5291
|
|
14
|
+
cht_utils/fileio/yaml.py,sha256=vlC2Z_MUTznZRnkyCSBc9WQLJHQGTm2NKA9Y_r6J0yk,932
|
|
15
|
+
cht_utils/fileops/__init__.py,sha256=lvwwPkSwckOO-_vZ_ROfpmVnwmxsBRjSnui3YWc5krU,1286
|
|
16
|
+
cht_utils/fileops/fileops.py,sha256=iiU8mS_2XhjZ1SCGBY2f-nZX4DofMVqmB-cucqqTjCU,9036
|
|
17
|
+
cht_utils/interpolation/__init__.py,sha256=vbXwLsvBtb27oV5BEJHTm09QxLdGcGSPcfhwwQldRNE,300
|
|
18
|
+
cht_utils/interpolation/interpolation.py,sha256=PVK_CpDpbITJqxjXitXNJ-CCVMWfNPbJJQWhclRzd10,3729
|
|
19
|
+
cht_utils/maps/__init__.py,sha256=EStqelML03gpH_HZ4FGHeQb_oJrp6bwLbIKc85_D9q4,82
|
|
20
|
+
cht_utils/maps/fileops.py,sha256=Dm4ZwZfzhr4d9VumZT9Oghy1Skg-6TtBozW2j7Xw_yY,4674
|
|
21
|
+
cht_utils/maps/flood_map.py,sha256=VBH9V5LQgNtopWB0hbhA0IJj0iMI7iyaSS1a8wMcft0,41993
|
|
22
|
+
cht_utils/maps/topobathy_map.py,sha256=0n11h5X6EphQiMS4t_6dKAsub8QDKp2yZjoLFlEkPCA,15807
|
|
23
|
+
cht_utils/maps/utils.py,sha256=S0pf4Mz996FCl3ENqtlyzFRiDlbwYYNP-vhOqJ5HuMg,20565
|
|
24
|
+
cht_utils/physics/__init__.py,sha256=IQy7Fkn8c8c6opZ9jZ6W4VdO9RacrkZWONlSygRXs1k,438
|
|
25
|
+
cht_utils/physics/deshoal.py,sha256=OA53NthjcrXdlgbP8KtehyVC9tDN-bQU5BgGHZJVVic,1444
|
|
26
|
+
cht_utils/physics/disper.py,sha256=e_dCgKmVFM2YREgueDIlFLuarsjcfQMW_s0gD-oAsas,1898
|
|
27
|
+
cht_utils/physics/runup_vo21.py,sha256=Nv4Uq3yHLnhNZsc-KxaIMJHr2QlOEWcKEDsCEkNlJwI,8053
|
|
28
|
+
cht_utils/physics/waves.py,sha256=VOfHz0_qiMFp3e0nocUBqalfTqJFu3h--5BiJqIHl0Q,1363
|
|
29
|
+
cht_utils/probabilistic/__init__.py,sha256=CqdfxE7tHPsnsZYXv_L3IN0rQXkYWG15BJ0h2oBAQmQ,293
|
|
30
|
+
cht_utils/probabilistic/prob_maps.py,sha256=UQacy6duEAmo903V8iRJMtanXQ2RJDD4a7xZZ-AxYzo,7643
|
|
31
|
+
cht_utils/remote/__init__.py,sha256=BYocLmXAnwiVIlMv8arm9nUCqVNiGKeLi3t7EDlaO0w,164
|
|
32
|
+
cht_utils/remote/s3.py,sha256=-bgHiM-fMyjbAz_42lXBo2FWPL0avc0ltky2VGsk80Q,11226
|
|
33
|
+
cht_utils/remote/sftp.py,sha256=TnPSWBM4dN-ZgoXFO0SuFO5x6uCJ8FZSVrkMjBirBrY,5671
|
|
34
|
+
cht_utils-2.0.0.dist-info/licenses/LICENSE,sha256=6KO9er0UAa3Z1wCOaIG9VXUYi9tyKTRtQtWHJ_U4Iwg,1074
|
|
35
|
+
cht_utils-2.0.0.dist-info/METADATA,sha256=G-bZ4NkB1RcnWVKrohVkJr-egY3JE2LRS2hzwgh3aKk,863
|
|
36
|
+
cht_utils-2.0.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
|
|
37
|
+
cht_utils-2.0.0.dist-info/top_level.txt,sha256=RNQ7P87dotxpkPVIDZ2hDGzcBOPWT1FJKCCctYCNzZ8,10
|
|
38
|
+
cht_utils-2.0.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
39
|
+
cht_utils-2.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 maartenvanormondt
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
cht_utils
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|