cci-os-worker 0.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- cci_os_worker/README.md +32 -0
- cci_os_worker/__init__.py +8 -0
- cci_os_worker/all_facets.py +211 -0
- cci_os_worker/deleter.py +115 -0
- cci_os_worker/directory.py +387 -0
- cci_os_worker/errors.py +31 -0
- cci_os_worker/filehandlers/__init__.py +2 -0
- cci_os_worker/filehandlers/generic_file.py +135 -0
- cci_os_worker/filehandlers/geojson.py +314 -0
- cci_os_worker/filehandlers/netcdf_file.py +311 -0
- cci_os_worker/filehandlers/util.py +679 -0
- cci_os_worker/log_status.py +134 -0
- cci_os_worker/path_tools.py +422 -0
- cci_os_worker/tests/__init__.py +0 -0
- cci_os_worker/tests/test_consistency.py +30 -0
- cci_os_worker/utils.py +178 -0
- cci_os_worker-0.5.0.dist-info/LICENSE +31 -0
- cci_os_worker-0.5.0.dist-info/METADATA +122 -0
- cci_os_worker-0.5.0.dist-info/RECORD +21 -0
- cci_os_worker-0.5.0.dist-info/WHEEL +4 -0
- cci_os_worker-0.5.0.dist-info/entry_points.txt +7 -0
cci_os_worker/README.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# CCI OS Worker Source
|
|
2
|
+
|
|
3
|
+
This package has now combined elements from the following packages, which have been dropped as dependencies:
|
|
4
|
+
- CEDA FBS
|
|
5
|
+
- Nappy
|
|
6
|
+
|
|
7
|
+
The source code provided here should accomplish all tasks related to these dependency packages. Contact [Daniel Westwood](daniel.westwood@stfc.ac.uk) for more details
|
|
8
|
+
|
|
9
|
+
## Entrypoint Scripts
|
|
10
|
+
It is recommended to use the entrypoint scripts to interface with this package once installed:
|
|
11
|
+
- facetscan
|
|
12
|
+
- fbi_update
|
|
13
|
+
|
|
14
|
+
NOTE: In a future update, these two scripts will be combined into a single workflow which generates the entire record for one dataset as a single process, rather than generating half the record in each case.
|
|
15
|
+
|
|
16
|
+
### Facetscan
|
|
17
|
+
|
|
18
|
+
Use this entrypoint to perform a facet scan given a set of dataset files and the worker config. This is automatically configured to create records in the elasticsearch index given by `facet_files_index` in the worker config.
|
|
19
|
+
- `datafile_path`: A path to a `datasets.txt` list of files.
|
|
20
|
+
- `conf`: Path to the worker config file.
|
|
21
|
+
- [OPTIONAL] `output`: Outputs failed datasets to a text file.
|
|
22
|
+
|
|
23
|
+
Use `facetscan --help` for more information.
|
|
24
|
+
|
|
25
|
+
### FBI Update
|
|
26
|
+
|
|
27
|
+
This entrypoint completes the second part of the Opensearch record for each dataset, with the same configuration options as above:
|
|
28
|
+
- `datafile_path`: A path to a `datasets.txt` list of files.
|
|
29
|
+
- `conf`: Path to the worker config file.
|
|
30
|
+
- [OPTIONAL] `output`: Outputs failed datasets to a text file.
|
|
31
|
+
|
|
32
|
+
Use `fbi_update --help` for more information.
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
__author__ = 'Daniel Westwood'
|
|
3
|
+
__date__ = '05 Nov 2024'
|
|
4
|
+
__copyright__ = 'Copyright 2024 United Kingdom Research and Innovation'
|
|
5
|
+
__license__ = 'BSD - see LICENSE file in top-level package directory'
|
|
6
|
+
__contact__ = 'daniel.westwood@stfc.ac.uk'
|
|
7
|
+
|
|
8
|
+
# Opens the file provided in the datasets folder.
|
|
9
|
+
# Iterate over all provided dataset paths.
|
|
10
|
+
|
|
11
|
+
# facet_scanner.get_handler('filepath')
|
|
12
|
+
# facets = handler.get_facets('filepath')
|
|
13
|
+
|
|
14
|
+
import logging
|
|
15
|
+
import hashlib
|
|
16
|
+
import argparse
|
|
17
|
+
import os
|
|
18
|
+
|
|
19
|
+
import asyncio
|
|
20
|
+
|
|
21
|
+
from cci_facet_scanner.core.facet_scanner import FacetScanner
|
|
22
|
+
from elasticsearch import Elasticsearch
|
|
23
|
+
|
|
24
|
+
from cci_os_worker.filehandlers.util import LDAPIdentifier
|
|
25
|
+
from cci_os_worker.filehandlers import NetCdfFile, GenericFile
|
|
26
|
+
from cci_os_worker import logstream
|
|
27
|
+
|
|
28
|
+
from .path_tools import PathTools
|
|
29
|
+
from .directory import check_timeout
|
|
30
|
+
from .utils import load_config, UpdateHandler, set_verbose
|
|
31
|
+
from .errors import HandlerError, DocMetadataError
|
|
32
|
+
|
|
33
|
+
logger = logging.getLogger(__name__)
|
|
34
|
+
logger.addHandler(logstream)
|
|
35
|
+
logger.propagate = False
|
|
36
|
+
|
|
37
|
+
class FacetUpdateHandler(UpdateHandler):
|
|
38
|
+
|
|
39
|
+
def __init__(self, conf: dict, dryrun: bool = False, test: bool = False):
|
|
40
|
+
"""
|
|
41
|
+
Initialise this class with the correct connections to
|
|
42
|
+
establish an elasticsearch client.
|
|
43
|
+
"""
|
|
44
|
+
logger.info('Loading Facet Updater')
|
|
45
|
+
|
|
46
|
+
super().__init__(conf, dryrun=dryrun, test=test)
|
|
47
|
+
|
|
48
|
+
self.facet_scanner = FacetScanner()
|
|
49
|
+
|
|
50
|
+
ldap_hosts = self._conf['ldap_configuration']['hosts']
|
|
51
|
+
self.ldap_interface = LDAPIdentifier(server=ldap_hosts, auto_bind=True)
|
|
52
|
+
|
|
53
|
+
self._spot_file = conf.get('spot_file',None)
|
|
54
|
+
self.pt = PathTools(spot_file=self._spot_file)
|
|
55
|
+
|
|
56
|
+
def _get_project_info(self, path):
|
|
57
|
+
"""
|
|
58
|
+
Get project info for a specific path
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
extension = os.path.splitext(path.split('/')[-1])[-1]
|
|
62
|
+
extension = extension.lower()
|
|
63
|
+
|
|
64
|
+
if extension == '.nc':
|
|
65
|
+
handler = NetCdfFile
|
|
66
|
+
else:
|
|
67
|
+
handler = GenericFile
|
|
68
|
+
|
|
69
|
+
calculate_md5 = self._conf.get('calculate_md5',False)
|
|
70
|
+
|
|
71
|
+
if handler is None:
|
|
72
|
+
raise HandlerError(filename=path)
|
|
73
|
+
|
|
74
|
+
handler_instance = handler(path, 3, calculate_md5=calculate_md5)
|
|
75
|
+
|
|
76
|
+
# FutureDetail: Remove manifest from 'doc' if unneeded (no indexing required.)
|
|
77
|
+
doc, phenomena, spatial = handler_instance.get_metadata()
|
|
78
|
+
|
|
79
|
+
if doc is None:
|
|
80
|
+
raise DocMetadataError(filename=path)
|
|
81
|
+
if len(doc) > 1:
|
|
82
|
+
doc = doc[0]
|
|
83
|
+
|
|
84
|
+
if phenomena:
|
|
85
|
+
doc['info']['phenomena'] = phenomena
|
|
86
|
+
if spatial:
|
|
87
|
+
doc['info']['spatial'] = spatial
|
|
88
|
+
|
|
89
|
+
spot = self.pt.spots.get_spot(path)
|
|
90
|
+
|
|
91
|
+
if spot is not None:
|
|
92
|
+
doc['info']['spot_name'] = spot
|
|
93
|
+
|
|
94
|
+
# Replace the UID and GID with name and group
|
|
95
|
+
uid = doc['info']['user']
|
|
96
|
+
gid = doc['info']['group']
|
|
97
|
+
|
|
98
|
+
doc['info']['user'] = self.ldap_interface.get_user(uid)
|
|
99
|
+
doc['info']['group'] = self.ldap_interface.get_group(gid)
|
|
100
|
+
|
|
101
|
+
return doc['info']
|
|
102
|
+
|
|
103
|
+
def _single_process_file(self, filepath: str, index: int = None, total: int = None):
|
|
104
|
+
"""
|
|
105
|
+
Perform facet scanning for a specific filepath
|
|
106
|
+
"""
|
|
107
|
+
|
|
108
|
+
logger.info('--------------------------------')
|
|
109
|
+
if index is None:
|
|
110
|
+
logger.info(f'Processing {filepath.split("/")[-1]}')
|
|
111
|
+
else:
|
|
112
|
+
logger.info(f'Processing {filepath.split("/")[-1]} ({index}/{total})')
|
|
113
|
+
|
|
114
|
+
# Get the handler for this filepath
|
|
115
|
+
handler = self.facet_scanner.get_handler(filepath, json_files=None)
|
|
116
|
+
|
|
117
|
+
# Extract the facets
|
|
118
|
+
facets = handler.get_facets(filepath)
|
|
119
|
+
|
|
120
|
+
# Build the project dictionary using the handlers project name attr
|
|
121
|
+
project = {
|
|
122
|
+
'info': self._get_project_info(filepath),
|
|
123
|
+
'projects': {
|
|
124
|
+
handler.project_name: facets
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if self._test:
|
|
129
|
+
index = self._conf['facet_files_test_index']['name']
|
|
130
|
+
else:
|
|
131
|
+
index = self._conf['facet_files_index']['name']
|
|
132
|
+
|
|
133
|
+
id = hashlib.sha1(filepath.encode(errors="ignore")).hexdigest()
|
|
134
|
+
|
|
135
|
+
# Send facets to elasticsearch
|
|
136
|
+
if not self._dryrun:
|
|
137
|
+
self.es.update(
|
|
138
|
+
index=index,
|
|
139
|
+
id=id,
|
|
140
|
+
body={'doc': project, 'doc_as_upsert': True}
|
|
141
|
+
)
|
|
142
|
+
else:
|
|
143
|
+
logger.info(f'DRYRUN: Skipped updating for {filepath.split("/")[-1]}')
|
|
144
|
+
|
|
145
|
+
self._local_cache(
|
|
146
|
+
filename=f'cache/{filepath.split("/")[-1]}-cache.json',
|
|
147
|
+
contents=project,
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
def _get_command_line_args():
|
|
151
|
+
"""
|
|
152
|
+
Get the command line arguments for the facet scan
|
|
153
|
+
"""
|
|
154
|
+
parser = argparse.ArgumentParser(description='Entrypoint for the CCI OS Worker on the CMD Line')
|
|
155
|
+
parser.add_argument('datafile_path', type=str, help='Path to the "datasets.txt" file')
|
|
156
|
+
parser.add_argument('conf', type=str, help='Path to Yaml config file for Elasticsearch')
|
|
157
|
+
|
|
158
|
+
parser.add_argument('-d','--dryrun', dest='dryrun', action='store_true', help='Perform in dryrun mode')
|
|
159
|
+
parser.add_argument('-t','--test', dest='test', action='store_true', help='Perform in test/staging mode')
|
|
160
|
+
parser.add_argument('-p','--prefix', dest='prefix', default='', help='Prefix to apply to all filenames')
|
|
161
|
+
parser.add_argument('-v','--verbose', action='count', default=2, help='Set level of verbosity for logs')
|
|
162
|
+
parser.add_argument('-f','--file-count', dest='file_count', type=int, help='Add limit to number of files to process.')
|
|
163
|
+
parser.add_argument('-o','--output', dest='output', default=None, help='Send fail list to an output file')
|
|
164
|
+
|
|
165
|
+
args = parser.parse_args()
|
|
166
|
+
|
|
167
|
+
return {
|
|
168
|
+
'datafile_path': args.datafile_path,
|
|
169
|
+
'conf': args.conf,
|
|
170
|
+
'dryrun': args.dryrun,
|
|
171
|
+
'test': args.test,
|
|
172
|
+
'prefix': args.prefix,
|
|
173
|
+
'verbose': args.verbose-1,
|
|
174
|
+
'file_count': args.file_count,
|
|
175
|
+
'output': args.output
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
def main(args: dict = None):
|
|
179
|
+
if args is None:
|
|
180
|
+
args = _get_command_line_args()
|
|
181
|
+
if isinstance(args['conf'], str):
|
|
182
|
+
conf = load_config(args['conf'])
|
|
183
|
+
|
|
184
|
+
if conf is None:
|
|
185
|
+
logger.error('Config file could not be loaded')
|
|
186
|
+
return
|
|
187
|
+
if not os.path.isfile(args['datafile_path']):
|
|
188
|
+
logger.error(f'Inaccessible Datafile - {args["datafile_path"]}')
|
|
189
|
+
return
|
|
190
|
+
|
|
191
|
+
if check_timeout():
|
|
192
|
+
logger.error('Check-timeout failed')
|
|
193
|
+
return
|
|
194
|
+
|
|
195
|
+
file_limit = conf.get('file_limit', None) or args.get('file_limit', None)
|
|
196
|
+
|
|
197
|
+
set_verbose(args['verbose'])
|
|
198
|
+
|
|
199
|
+
fs = FacetUpdateHandler(conf, dryrun=args['dryrun'], test=args['test'])
|
|
200
|
+
fail_list = fs.process_deposits(args['datafile_path'], args['prefix'], file_limit=file_limit)
|
|
201
|
+
|
|
202
|
+
logger.info('Failed items:')
|
|
203
|
+
for f in fail_list:
|
|
204
|
+
logger.info(f)
|
|
205
|
+
|
|
206
|
+
if args['output'] is not None and fail_list != []:
|
|
207
|
+
with open(args['output'],'w') as f:
|
|
208
|
+
f.write('\n'.join(fail_list))
|
|
209
|
+
|
|
210
|
+
if __name__ == '__main__':
|
|
211
|
+
main()
|
cci_os_worker/deleter.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# encoding: utf-8
|
|
2
|
+
__author__ = 'Daniel Westwood'
|
|
3
|
+
__date__ = '05 Nov 2024'
|
|
4
|
+
__copyright__ = 'Copyright 2024 United Kingdom Research and Innovation'
|
|
5
|
+
__license__ = 'BSD - see LICENSE file in top-level package directory'
|
|
6
|
+
__contact__ = 'daniel.westwood@stfc.ac.uk'
|
|
7
|
+
|
|
8
|
+
import logging
|
|
9
|
+
import hashlib
|
|
10
|
+
import argparse
|
|
11
|
+
import os
|
|
12
|
+
|
|
13
|
+
from typing import Union
|
|
14
|
+
|
|
15
|
+
from cci_facet_scanner.core.facet_scanner import FacetScanner
|
|
16
|
+
from ceda_elasticsearch_tools.elasticsearch import CEDAElasticsearchClient
|
|
17
|
+
from fbi_directory_check.utils import check_timeout
|
|
18
|
+
|
|
19
|
+
from .utils import load_config, UpdateHandler, set_verbose
|
|
20
|
+
|
|
21
|
+
from cci_os_worker import logstream
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger(__name__)
|
|
24
|
+
logger.addHandler(logstream)
|
|
25
|
+
logger.propagate = False
|
|
26
|
+
|
|
27
|
+
class ElasticsearchDeleter(UpdateHandler):
|
|
28
|
+
|
|
29
|
+
def __init__(self, conf: dict, dryrun: bool = False, test: bool = False):
|
|
30
|
+
"""
|
|
31
|
+
Initialise this class with the correct connections to
|
|
32
|
+
establish an elasticsearch client.
|
|
33
|
+
"""
|
|
34
|
+
logger.info('Loading ES Deleter')
|
|
35
|
+
|
|
36
|
+
super().__init__(conf, dryrun=dryrun, test=test)
|
|
37
|
+
|
|
38
|
+
def _remove_file(self, filepath):
|
|
39
|
+
"""
|
|
40
|
+
Delete all records for a specific dataset
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
index = self._conf['facet_files_index']['name']
|
|
44
|
+
|
|
45
|
+
id = hashlib.sha1(filepath.encode(errors="ignore")).hexdigest()
|
|
46
|
+
|
|
47
|
+
if not self._dryrun:
|
|
48
|
+
try:
|
|
49
|
+
self.es.delete(
|
|
50
|
+
index=index,
|
|
51
|
+
id=id,
|
|
52
|
+
)
|
|
53
|
+
return 0
|
|
54
|
+
except:
|
|
55
|
+
return 1
|
|
56
|
+
else:
|
|
57
|
+
logger.info(f'DRYRUN: Skipped deleting {filepath.split("/")[-1]}')
|
|
58
|
+
return 0
|
|
59
|
+
|
|
60
|
+
def _get_command_line_args():
|
|
61
|
+
"""
|
|
62
|
+
Get the command line arguments for the facet scan
|
|
63
|
+
"""
|
|
64
|
+
parser = argparse.ArgumentParser(description='Entrypoint for the CCI OS Worker on the CMD Line')
|
|
65
|
+
parser.add_argument('datafile_path', type=str, help='Path to the "datasets.txt" file')
|
|
66
|
+
parser.add_argument('conf', type=str, help='Path to Yaml config file for Elasticsearch')
|
|
67
|
+
|
|
68
|
+
parser.add_argument('-d','--dryrun', dest='dryrun', action='store_true', help='Perform in dryrun mode')
|
|
69
|
+
parser.add_argument('-t','--test', dest='test', action='store_true', help='Perform in test/staging mode')
|
|
70
|
+
parser.add_argument('-v','--verbose', action='count', default=2, help='Set level of verbosity for logs')
|
|
71
|
+
parser.add_argument('-o','--output', dest='output', default=None, help='Send fail list to an output file')
|
|
72
|
+
|
|
73
|
+
args = parser.parse_args()
|
|
74
|
+
|
|
75
|
+
return {
|
|
76
|
+
'datafile_path': args.datafile_path,
|
|
77
|
+
'conf': args.conf,
|
|
78
|
+
'dryrun': args.dryrun,
|
|
79
|
+
'test': args.test,
|
|
80
|
+
'verbose': args.verbose-1,
|
|
81
|
+
'output': args.output
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
def main(args: dict = None):
|
|
85
|
+
if args is None:
|
|
86
|
+
args = _get_command_line_args()
|
|
87
|
+
if isinstance(args['conf'], str):
|
|
88
|
+
conf = load_config(args['conf'])
|
|
89
|
+
|
|
90
|
+
if conf is None:
|
|
91
|
+
logger.error('Config file could not be loaded')
|
|
92
|
+
return
|
|
93
|
+
if not os.path.isfile(args['datafile_path']):
|
|
94
|
+
logger.error(f'Inaccessible Datafile - {args["datafile_path"]}')
|
|
95
|
+
return
|
|
96
|
+
|
|
97
|
+
if check_timeout():
|
|
98
|
+
logger.error('Check-timeout failed')
|
|
99
|
+
return
|
|
100
|
+
|
|
101
|
+
set_verbose(args['verbose'])
|
|
102
|
+
|
|
103
|
+
deleter = ElasticsearchDeleter(conf, dryrun=args['dryrun'], test=args['test'])
|
|
104
|
+
fail_list = deleter.process_removals(args['datafile_path'])
|
|
105
|
+
|
|
106
|
+
logger.info('Failed items:')
|
|
107
|
+
for f in fail_list:
|
|
108
|
+
logger.info(f)
|
|
109
|
+
|
|
110
|
+
if args['output'] is not None and fail_list != []:
|
|
111
|
+
with open(args['output'],'w') as f:
|
|
112
|
+
f.write('\n'.join(fail_list))
|
|
113
|
+
|
|
114
|
+
if __name__ == '__main__':
|
|
115
|
+
main()
|