cci-os-worker 0.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,32 @@
1
+ # CCI OS Worker Source
2
+
3
+ This package has now combined elements from the following packages, which have been dropped as dependencies:
4
+ - CEDA FBS
5
+ - Nappy
6
+
7
+ The source code provided here should accomplish all tasks related to these dependency packages. Contact [Daniel Westwood](daniel.westwood@stfc.ac.uk) for more details
8
+
9
+ ## Entrypoint Scripts
10
+ It is recommended to use the entrypoint scripts to interface with this package once installed:
11
+ - facetscan
12
+ - fbi_update
13
+
14
+ NOTE: In a future update, these two scripts will be combined into a single workflow which generates the entire record for one dataset as a single process, rather than generating half the record in each case.
15
+
16
+ ### Facetscan
17
+
18
+ Use this entrypoint to perform a facet scan given a set of dataset files and the worker config. This is automatically configured to create records in the elasticsearch index given by `facet_files_index` in the worker config.
19
+ - `datafile_path`: A path to a `datasets.txt` list of files.
20
+ - `conf`: Path to the worker config file.
21
+ - [OPTIONAL] `output`: Outputs failed datasets to a text file.
22
+
23
+ Use `facetscan --help` for more information.
24
+
25
+ ### FBI Update
26
+
27
+ This entrypoint completes the second part of the Opensearch record for each dataset, with the same configuration options as above:
28
+ - `datafile_path`: A path to a `datasets.txt` list of files.
29
+ - `conf`: Path to the worker config file.
30
+ - [OPTIONAL] `output`: Outputs failed datasets to a text file.
31
+
32
+ Use `fbi_update --help` for more information.
@@ -0,0 +1,8 @@
1
+ # Logger setup
2
+ import logging
3
+
4
+ logging.basicConfig(level=logging.DEBUG)
5
+ logstream = logging.StreamHandler()
6
+
7
+ formatter = logging.Formatter('%(levelname)s [%(name)s]: %(message)s')
8
+ logstream.setFormatter(formatter)
@@ -0,0 +1,211 @@
1
+ # encoding: utf-8
2
+ __author__ = 'Daniel Westwood'
3
+ __date__ = '05 Nov 2024'
4
+ __copyright__ = 'Copyright 2024 United Kingdom Research and Innovation'
5
+ __license__ = 'BSD - see LICENSE file in top-level package directory'
6
+ __contact__ = 'daniel.westwood@stfc.ac.uk'
7
+
8
+ # Opens the file provided in the datasets folder.
9
+ # Iterate over all provided dataset paths.
10
+
11
+ # facet_scanner.get_handler('filepath')
12
+ # facets = handler.get_facets('filepath')
13
+
14
+ import logging
15
+ import hashlib
16
+ import argparse
17
+ import os
18
+
19
+ import asyncio
20
+
21
+ from cci_facet_scanner.core.facet_scanner import FacetScanner
22
+ from elasticsearch import Elasticsearch
23
+
24
+ from cci_os_worker.filehandlers.util import LDAPIdentifier
25
+ from cci_os_worker.filehandlers import NetCdfFile, GenericFile
26
+ from cci_os_worker import logstream
27
+
28
+ from .path_tools import PathTools
29
+ from .directory import check_timeout
30
+ from .utils import load_config, UpdateHandler, set_verbose
31
+ from .errors import HandlerError, DocMetadataError
32
+
33
+ logger = logging.getLogger(__name__)
34
+ logger.addHandler(logstream)
35
+ logger.propagate = False
36
+
37
+ class FacetUpdateHandler(UpdateHandler):
38
+
39
+ def __init__(self, conf: dict, dryrun: bool = False, test: bool = False):
40
+ """
41
+ Initialise this class with the correct connections to
42
+ establish an elasticsearch client.
43
+ """
44
+ logger.info('Loading Facet Updater')
45
+
46
+ super().__init__(conf, dryrun=dryrun, test=test)
47
+
48
+ self.facet_scanner = FacetScanner()
49
+
50
+ ldap_hosts = self._conf['ldap_configuration']['hosts']
51
+ self.ldap_interface = LDAPIdentifier(server=ldap_hosts, auto_bind=True)
52
+
53
+ self._spot_file = conf.get('spot_file',None)
54
+ self.pt = PathTools(spot_file=self._spot_file)
55
+
56
+ def _get_project_info(self, path):
57
+ """
58
+ Get project info for a specific path
59
+ """
60
+
61
+ extension = os.path.splitext(path.split('/')[-1])[-1]
62
+ extension = extension.lower()
63
+
64
+ if extension == '.nc':
65
+ handler = NetCdfFile
66
+ else:
67
+ handler = GenericFile
68
+
69
+ calculate_md5 = self._conf.get('calculate_md5',False)
70
+
71
+ if handler is None:
72
+ raise HandlerError(filename=path)
73
+
74
+ handler_instance = handler(path, 3, calculate_md5=calculate_md5)
75
+
76
+ # FutureDetail: Remove manifest from 'doc' if unneeded (no indexing required.)
77
+ doc, phenomena, spatial = handler_instance.get_metadata()
78
+
79
+ if doc is None:
80
+ raise DocMetadataError(filename=path)
81
+ if len(doc) > 1:
82
+ doc = doc[0]
83
+
84
+ if phenomena:
85
+ doc['info']['phenomena'] = phenomena
86
+ if spatial:
87
+ doc['info']['spatial'] = spatial
88
+
89
+ spot = self.pt.spots.get_spot(path)
90
+
91
+ if spot is not None:
92
+ doc['info']['spot_name'] = spot
93
+
94
+ # Replace the UID and GID with name and group
95
+ uid = doc['info']['user']
96
+ gid = doc['info']['group']
97
+
98
+ doc['info']['user'] = self.ldap_interface.get_user(uid)
99
+ doc['info']['group'] = self.ldap_interface.get_group(gid)
100
+
101
+ return doc['info']
102
+
103
+ def _single_process_file(self, filepath: str, index: int = None, total: int = None):
104
+ """
105
+ Perform facet scanning for a specific filepath
106
+ """
107
+
108
+ logger.info('--------------------------------')
109
+ if index is None:
110
+ logger.info(f'Processing {filepath.split("/")[-1]}')
111
+ else:
112
+ logger.info(f'Processing {filepath.split("/")[-1]} ({index}/{total})')
113
+
114
+ # Get the handler for this filepath
115
+ handler = self.facet_scanner.get_handler(filepath, json_files=None)
116
+
117
+ # Extract the facets
118
+ facets = handler.get_facets(filepath)
119
+
120
+ # Build the project dictionary using the handlers project name attr
121
+ project = {
122
+ 'info': self._get_project_info(filepath),
123
+ 'projects': {
124
+ handler.project_name: facets
125
+ }
126
+ }
127
+
128
+ if self._test:
129
+ index = self._conf['facet_files_test_index']['name']
130
+ else:
131
+ index = self._conf['facet_files_index']['name']
132
+
133
+ id = hashlib.sha1(filepath.encode(errors="ignore")).hexdigest()
134
+
135
+ # Send facets to elasticsearch
136
+ if not self._dryrun:
137
+ self.es.update(
138
+ index=index,
139
+ id=id,
140
+ body={'doc': project, 'doc_as_upsert': True}
141
+ )
142
+ else:
143
+ logger.info(f'DRYRUN: Skipped updating for {filepath.split("/")[-1]}')
144
+
145
+ self._local_cache(
146
+ filename=f'cache/{filepath.split("/")[-1]}-cache.json',
147
+ contents=project,
148
+ )
149
+
150
+ def _get_command_line_args():
151
+ """
152
+ Get the command line arguments for the facet scan
153
+ """
154
+ parser = argparse.ArgumentParser(description='Entrypoint for the CCI OS Worker on the CMD Line')
155
+ parser.add_argument('datafile_path', type=str, help='Path to the "datasets.txt" file')
156
+ parser.add_argument('conf', type=str, help='Path to Yaml config file for Elasticsearch')
157
+
158
+ parser.add_argument('-d','--dryrun', dest='dryrun', action='store_true', help='Perform in dryrun mode')
159
+ parser.add_argument('-t','--test', dest='test', action='store_true', help='Perform in test/staging mode')
160
+ parser.add_argument('-p','--prefix', dest='prefix', default='', help='Prefix to apply to all filenames')
161
+ parser.add_argument('-v','--verbose', action='count', default=2, help='Set level of verbosity for logs')
162
+ parser.add_argument('-f','--file-count', dest='file_count', type=int, help='Add limit to number of files to process.')
163
+ parser.add_argument('-o','--output', dest='output', default=None, help='Send fail list to an output file')
164
+
165
+ args = parser.parse_args()
166
+
167
+ return {
168
+ 'datafile_path': args.datafile_path,
169
+ 'conf': args.conf,
170
+ 'dryrun': args.dryrun,
171
+ 'test': args.test,
172
+ 'prefix': args.prefix,
173
+ 'verbose': args.verbose-1,
174
+ 'file_count': args.file_count,
175
+ 'output': args.output
176
+ }
177
+
178
+ def main(args: dict = None):
179
+ if args is None:
180
+ args = _get_command_line_args()
181
+ if isinstance(args['conf'], str):
182
+ conf = load_config(args['conf'])
183
+
184
+ if conf is None:
185
+ logger.error('Config file could not be loaded')
186
+ return
187
+ if not os.path.isfile(args['datafile_path']):
188
+ logger.error(f'Inaccessible Datafile - {args["datafile_path"]}')
189
+ return
190
+
191
+ if check_timeout():
192
+ logger.error('Check-timeout failed')
193
+ return
194
+
195
+ file_limit = conf.get('file_limit', None) or args.get('file_limit', None)
196
+
197
+ set_verbose(args['verbose'])
198
+
199
+ fs = FacetUpdateHandler(conf, dryrun=args['dryrun'], test=args['test'])
200
+ fail_list = fs.process_deposits(args['datafile_path'], args['prefix'], file_limit=file_limit)
201
+
202
+ logger.info('Failed items:')
203
+ for f in fail_list:
204
+ logger.info(f)
205
+
206
+ if args['output'] is not None and fail_list != []:
207
+ with open(args['output'],'w') as f:
208
+ f.write('\n'.join(fail_list))
209
+
210
+ if __name__ == '__main__':
211
+ main()
@@ -0,0 +1,115 @@
1
+ # encoding: utf-8
2
+ __author__ = 'Daniel Westwood'
3
+ __date__ = '05 Nov 2024'
4
+ __copyright__ = 'Copyright 2024 United Kingdom Research and Innovation'
5
+ __license__ = 'BSD - see LICENSE file in top-level package directory'
6
+ __contact__ = 'daniel.westwood@stfc.ac.uk'
7
+
8
+ import logging
9
+ import hashlib
10
+ import argparse
11
+ import os
12
+
13
+ from typing import Union
14
+
15
+ from cci_facet_scanner.core.facet_scanner import FacetScanner
16
+ from ceda_elasticsearch_tools.elasticsearch import CEDAElasticsearchClient
17
+ from fbi_directory_check.utils import check_timeout
18
+
19
+ from .utils import load_config, UpdateHandler, set_verbose
20
+
21
+ from cci_os_worker import logstream
22
+
23
+ logger = logging.getLogger(__name__)
24
+ logger.addHandler(logstream)
25
+ logger.propagate = False
26
+
27
+ class ElasticsearchDeleter(UpdateHandler):
28
+
29
+ def __init__(self, conf: dict, dryrun: bool = False, test: bool = False):
30
+ """
31
+ Initialise this class with the correct connections to
32
+ establish an elasticsearch client.
33
+ """
34
+ logger.info('Loading ES Deleter')
35
+
36
+ super().__init__(conf, dryrun=dryrun, test=test)
37
+
38
+ def _remove_file(self, filepath):
39
+ """
40
+ Delete all records for a specific dataset
41
+ """
42
+
43
+ index = self._conf['facet_files_index']['name']
44
+
45
+ id = hashlib.sha1(filepath.encode(errors="ignore")).hexdigest()
46
+
47
+ if not self._dryrun:
48
+ try:
49
+ self.es.delete(
50
+ index=index,
51
+ id=id,
52
+ )
53
+ return 0
54
+ except:
55
+ return 1
56
+ else:
57
+ logger.info(f'DRYRUN: Skipped deleting {filepath.split("/")[-1]}')
58
+ return 0
59
+
60
+ def _get_command_line_args():
61
+ """
62
+ Get the command line arguments for the facet scan
63
+ """
64
+ parser = argparse.ArgumentParser(description='Entrypoint for the CCI OS Worker on the CMD Line')
65
+ parser.add_argument('datafile_path', type=str, help='Path to the "datasets.txt" file')
66
+ parser.add_argument('conf', type=str, help='Path to Yaml config file for Elasticsearch')
67
+
68
+ parser.add_argument('-d','--dryrun', dest='dryrun', action='store_true', help='Perform in dryrun mode')
69
+ parser.add_argument('-t','--test', dest='test', action='store_true', help='Perform in test/staging mode')
70
+ parser.add_argument('-v','--verbose', action='count', default=2, help='Set level of verbosity for logs')
71
+ parser.add_argument('-o','--output', dest='output', default=None, help='Send fail list to an output file')
72
+
73
+ args = parser.parse_args()
74
+
75
+ return {
76
+ 'datafile_path': args.datafile_path,
77
+ 'conf': args.conf,
78
+ 'dryrun': args.dryrun,
79
+ 'test': args.test,
80
+ 'verbose': args.verbose-1,
81
+ 'output': args.output
82
+ }
83
+
84
+ def main(args: dict = None):
85
+ if args is None:
86
+ args = _get_command_line_args()
87
+ if isinstance(args['conf'], str):
88
+ conf = load_config(args['conf'])
89
+
90
+ if conf is None:
91
+ logger.error('Config file could not be loaded')
92
+ return
93
+ if not os.path.isfile(args['datafile_path']):
94
+ logger.error(f'Inaccessible Datafile - {args["datafile_path"]}')
95
+ return
96
+
97
+ if check_timeout():
98
+ logger.error('Check-timeout failed')
99
+ return
100
+
101
+ set_verbose(args['verbose'])
102
+
103
+ deleter = ElasticsearchDeleter(conf, dryrun=args['dryrun'], test=args['test'])
104
+ fail_list = deleter.process_removals(args['datafile_path'])
105
+
106
+ logger.info('Failed items:')
107
+ for f in fail_list:
108
+ logger.info(f)
109
+
110
+ if args['output'] is not None and fail_list != []:
111
+ with open(args['output'],'w') as f:
112
+ f.write('\n'.join(fail_list))
113
+
114
+ if __name__ == '__main__':
115
+ main()