PyONDE 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- PyONDE/__init__.py +52 -0
- PyONDE/_version.py +24 -0
- PyONDE/bin/__init__.py +0 -0
- PyONDE/onde.py +3827 -0
- PyONDE/onde_versions/ONDE_fields_v0.9.1pre.csv +202 -0
- pyonde-0.1.2.dist-info/METADATA +46 -0
- pyonde-0.1.2.dist-info/RECORD +10 -0
- pyonde-0.1.2.dist-info/WHEEL +5 -0
- pyonde-0.1.2.dist-info/licenses/LICENSE.txt +202 -0
- pyonde-0.1.2.dist-info/top_level.txt +1 -0
PyONDE/onde.py
ADDED
|
@@ -0,0 +1,3827 @@
|
|
|
1
|
+
# Road map:
|
|
2
|
+
# *Need to implement _del_attr and _del_item on various classes.
|
|
3
|
+
|
|
4
|
+
import h5py
|
|
5
|
+
import sys
|
|
6
|
+
import os
|
|
7
|
+
import os.path
|
|
8
|
+
import posixpath
|
|
9
|
+
import threading
|
|
10
|
+
import ast
|
|
11
|
+
import csv
|
|
12
|
+
import copy
|
|
13
|
+
import numbers
|
|
14
|
+
import collections
|
|
15
|
+
import collections.abc
|
|
16
|
+
from dataclasses import dataclass, field
|
|
17
|
+
from typing import Optional, Any
|
|
18
|
+
import numpy as np
|
|
19
|
+
|
|
20
|
+
onde_basic_attrs = {
|
|
21
|
+
"ONDEValue":(
|
|
22
|
+
"_freeze",
|
|
23
|
+
"_frozen",
|
|
24
|
+
"_get_attr",
|
|
25
|
+
"_get_data",
|
|
26
|
+
"_set_attr",
|
|
27
|
+
"_set_data",
|
|
28
|
+
"_has_attr",
|
|
29
|
+
"value"
|
|
30
|
+
),
|
|
31
|
+
"ONDEArray":(
|
|
32
|
+
"_freeze",
|
|
33
|
+
"_frozen",
|
|
34
|
+
"_get_attr",
|
|
35
|
+
"_get_data",
|
|
36
|
+
"_set_attr",
|
|
37
|
+
"_set_data",
|
|
38
|
+
"_has_attr",
|
|
39
|
+
"value"
|
|
40
|
+
),
|
|
41
|
+
"ONDEReferenceArray":(
|
|
42
|
+
"_freeze",
|
|
43
|
+
"_frozen",
|
|
44
|
+
"_get_attr",
|
|
45
|
+
"_get_data",
|
|
46
|
+
"_get_item",
|
|
47
|
+
"_set_attr",
|
|
48
|
+
"_set_data",
|
|
49
|
+
"_set_item",
|
|
50
|
+
"_has_attr",
|
|
51
|
+
"refs"
|
|
52
|
+
),
|
|
53
|
+
"ONDEObject":(
|
|
54
|
+
"_freeze",
|
|
55
|
+
"_frozen",
|
|
56
|
+
"_get_attr",
|
|
57
|
+
"_set_attr",
|
|
58
|
+
"_has_attr",
|
|
59
|
+
"_list_attrs"
|
|
60
|
+
),
|
|
61
|
+
"ONDEFileGraphSnapshot":(
|
|
62
|
+
"_freeze",
|
|
63
|
+
"_frozen",
|
|
64
|
+
"_get_attr",
|
|
65
|
+
"_set_attr",
|
|
66
|
+
"_has_attr",
|
|
67
|
+
"_list_attrs"
|
|
68
|
+
)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
def repr_helper(indentation, classname, _extra_info = None, **attrdict):
|
|
72
|
+
lines = []
|
|
73
|
+
classline = " "*indentation + classname
|
|
74
|
+
if _extra_info is not None:
|
|
75
|
+
classline += " "+_extra_info
|
|
76
|
+
pass
|
|
77
|
+
lines.append(classline)
|
|
78
|
+
for attrname in attrdict:
|
|
79
|
+
lines.append(" "*(indentation+2) + f"{attrname:>18s}: {attrdict[attrname]:s}")
|
|
80
|
+
pass
|
|
81
|
+
return "\n".join(lines)
|
|
82
|
+
|
|
83
|
+
def onde_from_python(value,field):
|
|
84
|
+
"""field is an ONDEField or None. So far it is not used because we don't have code to parse the type specification"""
|
|
85
|
+
if isinstance(value,numbers.Integral) or isinstance(value, numbers.Real) or isinstance(value,numbers.Complex) or isinstance(value,str) or isinstance(value,np.str_):
|
|
86
|
+
return ONDEValue.new(value)
|
|
87
|
+
|
|
88
|
+
if isinstance(value,collections.abc.Sequence):
|
|
89
|
+
subvalues = []
|
|
90
|
+
all_string = True
|
|
91
|
+
all_numeric = True
|
|
92
|
+
any_complex = False
|
|
93
|
+
all_object = True
|
|
94
|
+
for subvalue in value:
|
|
95
|
+
if isinstance(value,str) or isinstance(value,np.str_):
|
|
96
|
+
all_numeric = False
|
|
97
|
+
all_object = False
|
|
98
|
+
pass
|
|
99
|
+
elif isinstance(value,numbers.Real):
|
|
100
|
+
all_string = False
|
|
101
|
+
all_object = False
|
|
102
|
+
pass
|
|
103
|
+
elif isinstance(value,numbers.Complex):
|
|
104
|
+
all_string = False
|
|
105
|
+
any_complex = True
|
|
106
|
+
all_object = False
|
|
107
|
+
pass
|
|
108
|
+
elif isinstance(value,collections.abc.Sequence):
|
|
109
|
+
all_string = False
|
|
110
|
+
all_numeric = False
|
|
111
|
+
value = onde_from_python(value)
|
|
112
|
+
pass
|
|
113
|
+
else:
|
|
114
|
+
raise ValueError(f"Could not convert object of type {value.__class__.__name__:s} into an ONDE object")
|
|
115
|
+
|
|
116
|
+
subvalues.append(value)
|
|
117
|
+
pass
|
|
118
|
+
|
|
119
|
+
if all_string and not(all_numeric) and not(all_object):
|
|
120
|
+
dtype = np.StringDType()
|
|
121
|
+
return ONDEArray.new(value = np.array(subvalues,dtype = dtype))
|
|
122
|
+
elif all_numeric and not(all_string) and not(all_object):
|
|
123
|
+
if any_complex:
|
|
124
|
+
dtype = "D"
|
|
125
|
+
pass
|
|
126
|
+
else:
|
|
127
|
+
dtype = "d"
|
|
128
|
+
pass
|
|
129
|
+
return ONDEArray.new(value = np.array(subvalues,dtype = dtype))
|
|
130
|
+
|
|
131
|
+
elif all_object and not(all_string) and not(all_numeric):
|
|
132
|
+
return ONDEReferenceArray.new(refs = np.array(subvalues,dtype = "O"))
|
|
133
|
+
else:
|
|
134
|
+
raise ValueError(f"Could not identify unique type for converting python object {str(value):s} into an ONDE object")
|
|
135
|
+
pass
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class TwoWayDictionary(object):
|
|
139
|
+
"""A dictionary that is indexable by strings using
|
|
140
|
+
getattr() or dot notation to get objects. Can be
|
|
141
|
+
reverse indexed by those objects using bracket notation
|
|
142
|
+
to get strings, which will return a tuple because
|
|
143
|
+
multiple index values can point to the same object."""
|
|
144
|
+
_bystrings = None # String index dictionary, contains objects
|
|
145
|
+
_byobjid = None # id(object) dictionary, contains frozensets of strings
|
|
146
|
+
_lock = None # threading.Lock object protecting _bystrings and _byobjid from changes. This lock is last in the locking order, ie you may not acquire any other lock while holding this lock.
|
|
147
|
+
_frozen = None # Boolean. Dictionary cannot be modified once frozen
|
|
148
|
+
|
|
149
|
+
def __init__(self, bystrings=None):
|
|
150
|
+
"""Pass an existing dictionary or None as bystrings"""
|
|
151
|
+
if bystrings is None:
|
|
152
|
+
bystrings = {}
|
|
153
|
+
pass
|
|
154
|
+
|
|
155
|
+
if isinstance(bystrings, TwoWayDictionary):
|
|
156
|
+
bystrings = object.__getattribute__(bystrings, "_bystrings")
|
|
157
|
+
pass
|
|
158
|
+
|
|
159
|
+
object.__setattr__(self, "_bystrings", dict(bystrings))
|
|
160
|
+
byobjid = dict()
|
|
161
|
+
for s in bystrings:
|
|
162
|
+
obj = bystrings[s]
|
|
163
|
+
|
|
164
|
+
if obj is not None:
|
|
165
|
+
if id(obj) in byobjid:
|
|
166
|
+
idx_set = byobjid[id(obj)]
|
|
167
|
+
pass
|
|
168
|
+
else:
|
|
169
|
+
idx_set = frozenset()
|
|
170
|
+
pass
|
|
171
|
+
|
|
172
|
+
byobjid[id(obj)] = idx_set | { s }
|
|
173
|
+
pass
|
|
174
|
+
pass
|
|
175
|
+
object.__setattr__(self, "_byobjid",byobjid)
|
|
176
|
+
object.__setattr__(self, "_lock", threading.Lock())
|
|
177
|
+
object.__setattr__(self, "_frozen", False)
|
|
178
|
+
pass
|
|
179
|
+
|
|
180
|
+
def __iter__(self):
|
|
181
|
+
_bystrings = object.__getattribute__(self, "_bystrings")
|
|
182
|
+
return _bystrings.__iter__() # Just use iterator of underlying dictionary
|
|
183
|
+
|
|
184
|
+
def __getattribute__(self, name):
|
|
185
|
+
if name.startswith("_"):
|
|
186
|
+
if name in {"_freeze", "_frozen", "_set_attr", "_get_attr", "_keys","_del_attr"}:
|
|
187
|
+
return object.__getattribute__(self, name)
|
|
188
|
+
raise IndexError("TwoWayDictionary: Indexes are not allowed to have leading underscores")
|
|
189
|
+
|
|
190
|
+
_get_attr = object.__getattribute__(self, "_get_attr")
|
|
191
|
+
|
|
192
|
+
return _get_attr(name)
|
|
193
|
+
|
|
194
|
+
def __call__(self, obj):
|
|
195
|
+
_byobjid = object.__getattribute__(self, "_byobjid")
|
|
196
|
+
objidx = _byobjid[id(obj)]
|
|
197
|
+
return objidx # Returns frozenset
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def __setattr__(self, name, value):
|
|
201
|
+
_set_attr = object.__getattribute__(self, "_set_attr")
|
|
202
|
+
_set_attr(name, value)
|
|
203
|
+
|
|
204
|
+
pass
|
|
205
|
+
|
|
206
|
+
def __setitem__(self,name,value):
|
|
207
|
+
_set_attr = object.__getattribute__(self, "_set_attr")
|
|
208
|
+
_set_attr(name, value)
|
|
209
|
+
|
|
210
|
+
pass
|
|
211
|
+
|
|
212
|
+
def _set_attr(self, name, value):
|
|
213
|
+
if name.startswith("_"):
|
|
214
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
215
|
+
_bystrings = object.__getattribute__(self, "_bystrings")
|
|
216
|
+
_byobjid = object.__getattribute__(self, "_byobjid")
|
|
217
|
+
_lock = object.__getattribute__(self, "_lock")
|
|
218
|
+
_frozen = object.__getattribute__(self, "_frozen")
|
|
219
|
+
|
|
220
|
+
if _frozen:
|
|
221
|
+
raise AttributeError("Not allowed to modify a frozen TwoWayDictionary")
|
|
222
|
+
with _lock:
|
|
223
|
+
if name in _bystrings:
|
|
224
|
+
# Remove old back-reference
|
|
225
|
+
oldobj = _bystrings[name]
|
|
226
|
+
_byobjid[id(oldobj)] = _byobjid[id(oldobj)] - frozenset({name})
|
|
227
|
+
pass
|
|
228
|
+
|
|
229
|
+
_bystrings[name] = value
|
|
230
|
+
|
|
231
|
+
nameset = frozenset({name})
|
|
232
|
+
# Check if this object already has an existing set of references
|
|
233
|
+
if id(value) in _byobjid:
|
|
234
|
+
nameset = _byobjid[id(value)] | nameset
|
|
235
|
+
pass
|
|
236
|
+
_byobjid[id(value)] = nameset
|
|
237
|
+
pass
|
|
238
|
+
pass
|
|
239
|
+
|
|
240
|
+
def _del_attr(self,name):
|
|
241
|
+
_bystrings = object.__getattribute__(self, "_bystrings")
|
|
242
|
+
_byobjid = object.__getattribute__(self, "_byobjid")
|
|
243
|
+
_lock = object.__getattribute__(self, "_lock")
|
|
244
|
+
_frozen = object.__getattribute__(self, "_frozen")
|
|
245
|
+
|
|
246
|
+
if _frozen:
|
|
247
|
+
raise AttributeError("Not allowed to modify a frozen TwoWayDictionary")
|
|
248
|
+
with _lock:
|
|
249
|
+
if name in _bystrings:
|
|
250
|
+
# Remove old back-reference
|
|
251
|
+
oldobj = _bystrings[name]
|
|
252
|
+
_byobjid[id(oldobj)] = _byobjid[id(oldobj)] - frozenset({name})
|
|
253
|
+
pass
|
|
254
|
+
del _bystrings[name]
|
|
255
|
+
pass
|
|
256
|
+
pass
|
|
257
|
+
|
|
258
|
+
def _get_attr(self, name):
|
|
259
|
+
if name.startswith("_"):
|
|
260
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
261
|
+
|
|
262
|
+
_bystrings = object.__getattribute__(self, "_bystrings")
|
|
263
|
+
return _bystrings[name]
|
|
264
|
+
|
|
265
|
+
def __getitem__(self,name):
|
|
266
|
+
|
|
267
|
+
return self._get_attr(name)
|
|
268
|
+
|
|
269
|
+
def __delitem__(self,name):
|
|
270
|
+
return self._del_attr(name)
|
|
271
|
+
|
|
272
|
+
def _freeze(self):
|
|
273
|
+
object.__setattr__(self, "_frozen", True)
|
|
274
|
+
pass
|
|
275
|
+
|
|
276
|
+
def _keys(self):
|
|
277
|
+
_bystrings = object.__getattribute__(self, "_bystrings")
|
|
278
|
+
return _bystrings.keys()
|
|
279
|
+
|
|
280
|
+
pass
|
|
281
|
+
|
|
282
|
+
class TwoWayArray(object):
|
|
283
|
+
"""An Array that is indexable by integers using
|
|
284
|
+
bracket notation. Can be
|
|
285
|
+
reverse indexed by those objects using parenthesis notation
|
|
286
|
+
to get index tuples, which will be wrapped in another layer
|
|
287
|
+
of tuple because
|
|
288
|
+
multiple index values can point to the same object."""
|
|
289
|
+
_byindex = None # numpy array, contains objects
|
|
290
|
+
_byobjid = None # id(object) dictionary, contains frozensets of strings
|
|
291
|
+
_lock = None # threading.Lock object protecting _byindex and _byobjid from changes. This lock is last in the locking order, ie you may not acquire any other lock while holding this lock.
|
|
292
|
+
_frozen = None # Boolean. Dictionary cannot be modified once frozen
|
|
293
|
+
|
|
294
|
+
def __init__(self, byindex=None,shape = None):
|
|
295
|
+
"""Pass an existing array or None as byindex"""
|
|
296
|
+
if shape is None:
|
|
297
|
+
shape = ()
|
|
298
|
+
pass
|
|
299
|
+
|
|
300
|
+
if byindex is None:
|
|
301
|
+
byindex = np.empty(shape,dtype="O")
|
|
302
|
+
pass
|
|
303
|
+
|
|
304
|
+
if isinstance(byindex, TwoWayArray):
|
|
305
|
+
byindex = byindex.byindex
|
|
306
|
+
pass
|
|
307
|
+
|
|
308
|
+
self._byindex = copy.copy(byindex)
|
|
309
|
+
self._byobjid = {}
|
|
310
|
+
nditer = np.nditer(self._byindex, flags = ("multi_index","refs_ok"))
|
|
311
|
+
for objarray in nditer:
|
|
312
|
+
obj = objarray[()]
|
|
313
|
+
if obj is not None:
|
|
314
|
+
if id(obj) in self._byobjid:
|
|
315
|
+
idx_set = self._byobjid[id(obj)]
|
|
316
|
+
pass
|
|
317
|
+
else:
|
|
318
|
+
idx_set = frozenset()
|
|
319
|
+
pass
|
|
320
|
+
|
|
321
|
+
self._byobjid[id(obj)] = idx_set | { tuple(nditer.multi_index) }
|
|
322
|
+
pass
|
|
323
|
+
pass
|
|
324
|
+
|
|
325
|
+
self._lock = threading.Lock()
|
|
326
|
+
self._frozen = False
|
|
327
|
+
|
|
328
|
+
pass
|
|
329
|
+
|
|
330
|
+
@property
|
|
331
|
+
def byindex(self):
|
|
332
|
+
"""Return safe copy of underlying array"""
|
|
333
|
+
with self._lock:
|
|
334
|
+
byindex = copy.copy(self._byindex)
|
|
335
|
+
pass
|
|
336
|
+
return byindex
|
|
337
|
+
|
|
338
|
+
def __iter__(self):
|
|
339
|
+
return np.nditer(self._byindex,flags=("multi_index","refs_ok")) # Just use iterator of underlying array
|
|
340
|
+
|
|
341
|
+
def __getitem__(self, index):
|
|
342
|
+
|
|
343
|
+
return self._byindex[index]
|
|
344
|
+
|
|
345
|
+
@property
|
|
346
|
+
def _shape(self):
|
|
347
|
+
return self._byindex.shape
|
|
348
|
+
|
|
349
|
+
def __call__(self, obj):
|
|
350
|
+
objidx = self._byobjid[id(obj)]
|
|
351
|
+
return objidx # Returns frozenset
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
def __setitem__(self, index, obj):
|
|
355
|
+
if not isinstance(index,tuple):
|
|
356
|
+
index = (index,)
|
|
357
|
+
pass
|
|
358
|
+
|
|
359
|
+
if self._frozen:
|
|
360
|
+
raise AttributeError("Not allowed to modify a frozen TwoWayArray")
|
|
361
|
+
with self._lock:
|
|
362
|
+
oldobj = self._byindex[index]
|
|
363
|
+
if oldobj is not None:
|
|
364
|
+
# Remove old back-reference
|
|
365
|
+
self._byobjid[id(oldobj)] = self._byobjid[id(oldobj)] - frozenset({index})
|
|
366
|
+
pass
|
|
367
|
+
|
|
368
|
+
self._byindex[index] = obj
|
|
369
|
+
|
|
370
|
+
indexset = frozenset({index})
|
|
371
|
+
# Check if this object already has an existing set of references
|
|
372
|
+
if id(obj) in self._byobjid:
|
|
373
|
+
indexset = self._byobjid[id(obj)] | indexset
|
|
374
|
+
pass
|
|
375
|
+
self._byobjid[id(obj)] = indexset
|
|
376
|
+
pass
|
|
377
|
+
pass
|
|
378
|
+
|
|
379
|
+
def _freeze(self):
|
|
380
|
+
self._byindex.flags.writeable = False
|
|
381
|
+
self._frozen = True
|
|
382
|
+
pass
|
|
383
|
+
pass
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
class ONDEFileGraph(object):
|
|
387
|
+
"""Represents the subgraph of interconnected ONDE objects
|
|
388
|
+
and attributes that are or will be contained within some particular ONDE file. It has an associated set of ONDEClassDefinitions that are used to interpret the attached objects."""
|
|
389
|
+
_lock = None # threading.Lock that protects access to replace the snapshot.
|
|
390
|
+
_lock_ownerthread = None # Writes protected by _lock, the threading.get_ident() of whichever thread owns the lock
|
|
391
|
+
latest_snap = None # class ONDEFileGraphSnapshot
|
|
392
|
+
class_defs = None # ONDEClassDefinitions, optional
|
|
393
|
+
|
|
394
|
+
def __init__(self, class_defs = None, latest_snap = None):
|
|
395
|
+
# self._lock = threading.Lock()
|
|
396
|
+
object.__setattr__(self, "_lock", threading.Lock())
|
|
397
|
+
|
|
398
|
+
if latest_snap is None:
|
|
399
|
+
latest_snap = ONDEFileGraphSnapshot.new()
|
|
400
|
+
|
|
401
|
+
pass
|
|
402
|
+
|
|
403
|
+
if not latest_snap._frozen:
|
|
404
|
+
latest_snap._freeze()
|
|
405
|
+
pass
|
|
406
|
+
|
|
407
|
+
object.__setattr__(self, "class_defs", class_defs)
|
|
408
|
+
|
|
409
|
+
# self.latest_snap = snapshot
|
|
410
|
+
object.__setattr__(self, "latest_snap", latest_snap)
|
|
411
|
+
|
|
412
|
+
pass
|
|
413
|
+
|
|
414
|
+
r"""
|
|
415
|
+
def __getattribute__(self, name):
|
|
416
|
+
if name.startswith("_"):
|
|
417
|
+
|
|
418
|
+
if name in {"_set_attr", "_get_attr","__dict__","__dir__","_latest_snap","_lock", "_class_defs"}:
|
|
419
|
+
return object.__getattribute__(self, name)
|
|
420
|
+
pass
|
|
421
|
+
|
|
422
|
+
_latest_snap = object.__getattribute__(self, "_latest_snap")
|
|
423
|
+
|
|
424
|
+
# call ONDEProxy
|
|
425
|
+
|
|
426
|
+
snap_proxy = ONDEProxy.new_from_snapshot(self, _latest_snap)
|
|
427
|
+
obj_proxy = ONDEProxy.new_from_proxy(snap_proxy, name)
|
|
428
|
+
|
|
429
|
+
if self._class_defs is not None:
|
|
430
|
+
return ONDEClassInstanceWrapper.new_from_proxy(obj_proxy)
|
|
431
|
+
return obj_proxy
|
|
432
|
+
|
|
433
|
+
def __setattr__(self, name, value):
|
|
434
|
+
if name.startswith("_"):
|
|
435
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
436
|
+
|
|
437
|
+
_latest_snap = object.__getattribute__(self, "_latest_snap")
|
|
438
|
+
|
|
439
|
+
snap_proxy = ONDEProxy.new_from_snapshot(self, _latest_snap)
|
|
440
|
+
|
|
441
|
+
snap_proxy._set_attr(name,value)
|
|
442
|
+
|
|
443
|
+
pass
|
|
444
|
+
|
|
445
|
+
def _get_attr(self, name):
|
|
446
|
+
if name.startswith("_"):
|
|
447
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
448
|
+
|
|
449
|
+
_latest_snap = object.__getattribute__(self, "_latest_snap")
|
|
450
|
+
|
|
451
|
+
# call ONDEProxy
|
|
452
|
+
|
|
453
|
+
snap_proxy = ONDEProxy.new_from_snapshot(self, _latest_snap)
|
|
454
|
+
obj_proxy = ONDEProxy.new_from_proxy(snap_proxy, name)
|
|
455
|
+
if self._class_defs is not None:
|
|
456
|
+
return ONDEClassInstanceWrapper.new(obj_proxy)
|
|
457
|
+
|
|
458
|
+
return obj_proxy # _latest_snap._get_attr(name)
|
|
459
|
+
|
|
460
|
+
def _set_attr(self, name, value):
|
|
461
|
+
if name.startswith("_"):
|
|
462
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
463
|
+
|
|
464
|
+
_latest_snap = object.__getattribute__(self, "_latest_snap")
|
|
465
|
+
snap_proxy._set_attr(name,value)
|
|
466
|
+
|
|
467
|
+
pass
|
|
468
|
+
|
|
469
|
+
def _get_item(self, index):
|
|
470
|
+
# Strictly shouldn't be necessary unless we shift to indexing the snapshot with numbers rather than strings
|
|
471
|
+
_latest_snap = object.__getattribute__(self, "_latest_snap")
|
|
472
|
+
|
|
473
|
+
# call ONDEProxy
|
|
474
|
+
|
|
475
|
+
snap_proxy = ONDEProxy.new_from_snapshot(self, _latest_snap)
|
|
476
|
+
obj_proxy = ONDEProxy.new_from_proxy(snap_proxy, index)
|
|
477
|
+
if self._class_defs is not None:
|
|
478
|
+
return ONDEClassInstanceWrapper.new(obj_proxy)
|
|
479
|
+
|
|
480
|
+
return obj_proxy # _latest_snap._get_attr(name)
|
|
481
|
+
|
|
482
|
+
def _set_item(self, index, value):
|
|
483
|
+
if name.startswith("_"):
|
|
484
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
485
|
+
|
|
486
|
+
_latest_snap = object.__getattribute__(self, "_latest_snap")
|
|
487
|
+
|
|
488
|
+
_latest_snap._set_item(index,value)
|
|
489
|
+
pass
|
|
490
|
+
"""
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _get_item(self, name):
|
|
494
|
+
latest_snap = object.__getattribute__(self, "latest_snap")
|
|
495
|
+
# Since The entry points are an ONDEFileGraphSnapshot, which does special flexible indexing with shorthands, we do the actual lookup to get the real index.
|
|
496
|
+
|
|
497
|
+
obj = latest_snap._get_item(name)
|
|
498
|
+
obj_keys= [ key for key in latest_snap.keys() if latest_snap._get_item(key) is obj ]
|
|
499
|
+
assert (len(obj_keys)==1)
|
|
500
|
+
name=obj_keys[0]
|
|
501
|
+
# call ONDEProxy
|
|
502
|
+
|
|
503
|
+
snap_proxy = ONDEProxy.new_from_snapshot(self, latest_snap)
|
|
504
|
+
obj_proxy = ONDEProxy.new_from_proxy(snap_proxy, name)
|
|
505
|
+
|
|
506
|
+
if self.class_defs is not None:
|
|
507
|
+
return ONDEClassInstanceWrapper.new_from_proxy(obj_proxy)
|
|
508
|
+
return obj_proxy
|
|
509
|
+
|
|
510
|
+
def __getitem__(self, name):
|
|
511
|
+
return self._get_item(name)
|
|
512
|
+
|
|
513
|
+
#def _set_item(self, name, value):
|
|
514
|
+
# latest_snap = object.__getattribute__(self, "latest_snap")
|
|
515
|
+
#
|
|
516
|
+
# snap_proxy = ONDEProxy.new_from_snapshot(self, latest_snap)
|
|
517
|
+
#
|
|
518
|
+
# snap_proxy._set_item(name,value)
|
|
519
|
+
#
|
|
520
|
+
# pass
|
|
521
|
+
|
|
522
|
+
def add(self,dataset):
|
|
523
|
+
latest_snap = object.__getattribute__(self, "latest_snap")
|
|
524
|
+
|
|
525
|
+
snap_proxy = ONDEProxy.new_from_snapshot(self, latest_snap)
|
|
526
|
+
|
|
527
|
+
snap_proxy.add(dataset)
|
|
528
|
+
|
|
529
|
+
pass
|
|
530
|
+
|
|
531
|
+
#def __setitem__(self, name, value):
|
|
532
|
+
# self._set_item(name,value)
|
|
533
|
+
# pass
|
|
534
|
+
|
|
535
|
+
def keys(self):
|
|
536
|
+
latest_snap = object.__getattribute__(self, "latest_snap")
|
|
537
|
+
return latest_snap._list_items()
|
|
538
|
+
|
|
539
|
+
def shortkeys(self):
|
|
540
|
+
latest_snap = object.__getattribute__(self, "latest_snap")
|
|
541
|
+
return latest_snap.shortkeys()
|
|
542
|
+
|
|
543
|
+
def __iter__(self):
|
|
544
|
+
return iter(self.keys())
|
|
545
|
+
|
|
546
|
+
def _repr(self, indentation, onde_class=None, extra_attrs={}):
|
|
547
|
+
extra_info = None
|
|
548
|
+
if self.class_defs is not None:
|
|
549
|
+
extra_info = f"(ONDE v{self.class_defs.version:s} class definitions)"
|
|
550
|
+
pass
|
|
551
|
+
return repr_helper(indentation, "ONDEFileGraph", _extra_info = extra_info)+"\n"+self.latest_snap._repr(indentation+2)
|
|
552
|
+
|
|
553
|
+
def __repr__(self):
|
|
554
|
+
return self._repr(0)
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
@classmethod
|
|
558
|
+
def new(cls,class_defs=None,snapshot=None):
|
|
559
|
+
#defs = None
|
|
560
|
+
#if class_def_csv_path is not None:
|
|
561
|
+
# defs = ONDEClassDefinitions.load_from_csv(class_def_csv_path)
|
|
562
|
+
# pass
|
|
563
|
+
|
|
564
|
+
return cls(class_defs = class_defs,latest_snap=snapshot)
|
|
565
|
+
|
|
566
|
+
pass
|
|
567
|
+
|
|
568
|
+
class ONDEDatasetFileGraph(ONDEFileGraph):
|
|
569
|
+
"""Represents an ONDEFileGraph that is presumed to contain at its base level a number of ONDE_DATASET objects as per the ONDE 1.0 specification."""
|
|
570
|
+
# Inherited members from ONDEFileGraph
|
|
571
|
+
# _lock = None # threading.Lock that protects access to replace the snapshot.
|
|
572
|
+
# _lock_ownerthread = None # Writes protected by _lock, the threading.get_ident() of whichever thread owns the lock
|
|
573
|
+
# latest_snap = None # class ONDEFileGraphSnapshot
|
|
574
|
+
# class_defs = None # ONDEClassDefinitions, optional
|
|
575
|
+
|
|
576
|
+
def __init__(self, class_defs = None, latest_snap = None):
|
|
577
|
+
super().__init__(self, class_defs = class_defs, latest_snap = latest_snap)
|
|
578
|
+
pass
|
|
579
|
+
# Various methods inherited from ONDEFileGraph
|
|
580
|
+
# !!!*** should provide way to add a dataset that automatically sets the index
|
|
581
|
+
pass
|
|
582
|
+
|
|
583
|
+
class ONDETransaction(object):
|
|
584
|
+
"""Represents a transaction in which the ONDEFileGraph is modified"""
|
|
585
|
+
graph = None # ONDEFileGraph object
|
|
586
|
+
scope = None
|
|
587
|
+
snap = None # This is the snapshot we're modifying
|
|
588
|
+
|
|
589
|
+
def __init__(self, graph, include_paths=None, exclude_paths=None,exclude_objects=None):
|
|
590
|
+
self.graph = graph
|
|
591
|
+
|
|
592
|
+
self.scope = ONDEOpScope(self,include_paths, exclude_paths,exclude_objects)
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
pass
|
|
596
|
+
|
|
597
|
+
def __enter__(self):
|
|
598
|
+
lockowner = object.__getattribute__(self.graph, "_lock_ownerthread")
|
|
599
|
+
if lockowner == threading.get_ident():
|
|
600
|
+
raise RuntimeError(f"Error creating a new transaction while another transaction is already open on the file graph by the same thread. This probably means that you are attempting a change on an object not accessed via the transaction. Within a transaction, always access objects via the transaction or scope objects.")
|
|
601
|
+
_lock = object.__getattribute__(self.graph, "_lock")
|
|
602
|
+
_lock.acquire()
|
|
603
|
+
|
|
604
|
+
object.__setattr__(self.graph, "_lock_ownerthread", threading.get_ident())
|
|
605
|
+
|
|
606
|
+
snap = object.__getattribute__(self.graph, "latest_snap")
|
|
607
|
+
self.snap = ONDEFileGraphSnapshot(_orig=snap) # Create mutable copy of most recent snapshot
|
|
608
|
+
|
|
609
|
+
return self.scope
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def __exit__(self, exc_type, exc, tb):
|
|
613
|
+
self.snap._freeze()
|
|
614
|
+
object.__setattr__(self.graph, "latest_snap",self.snap)
|
|
615
|
+
object.__setattr__(self.graph, "_lock_ownerthread", None)
|
|
616
|
+
_lock = object.__getattribute__(self.graph, "_lock")
|
|
617
|
+
_lock.release()
|
|
618
|
+
|
|
619
|
+
pass
|
|
620
|
+
|
|
621
|
+
pass
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
class ONDEOpScope(object):
|
|
625
|
+
"""Represents scope of an operation. This includes a list of paths originating at entry points that are included in the scope, and an overriding list of paths originating at entry points that are excluded from the scope"""
|
|
626
|
+
include_paths = None # List of ONDEPath objects, or None representing no scope specified
|
|
627
|
+
exclude_paths = None # frozenset of ONDEPath objects
|
|
628
|
+
exclude_objects = None # frozenset of objects to be excluded
|
|
629
|
+
trans = None # Transaction for this scope (warning: creates reference loop)
|
|
630
|
+
|
|
631
|
+
def __init__(self, trans,include_paths=None, exclude_paths = None, exclude_objects = None):
|
|
632
|
+
self.trans=trans
|
|
633
|
+
self.include_paths = include_paths
|
|
634
|
+
if exclude_paths is None:
|
|
635
|
+
exclude_paths = []
|
|
636
|
+
pass
|
|
637
|
+
self.exclude_paths = frozenset(exclude_paths)
|
|
638
|
+
|
|
639
|
+
if exclude_objects is None:
|
|
640
|
+
exclude_objects = []
|
|
641
|
+
pass
|
|
642
|
+
self.exclude_objects = frozenset(exclude_objects)
|
|
643
|
+
pass
|
|
644
|
+
|
|
645
|
+
def __enter__(self):
|
|
646
|
+
|
|
647
|
+
return self
|
|
648
|
+
|
|
649
|
+
def __exit__(self, exc_type, exc, tb):
|
|
650
|
+
pass
|
|
651
|
+
|
|
652
|
+
@property
|
|
653
|
+
def graph(self):
|
|
654
|
+
return ONDEProxy.new_from_scope(self)
|
|
655
|
+
|
|
656
|
+
def rescope(self,include_paths=None, exclude_paths = None, exclude_objects = None):
|
|
657
|
+
return type(self)(include_paths, exclude_paths, exclude_objects)
|
|
658
|
+
|
|
659
|
+
def __hash__(self):
|
|
660
|
+
return hash((tuple(self.include_paths), self.exclude_paths, self.exclude_objects))
|
|
661
|
+
|
|
662
|
+
# to do: the paths in the onde op scope will generally end with an
|
|
663
|
+
# attribute name or an array index and the scope starts from only that
|
|
664
|
+
# attribute of the object
|
|
665
|
+
|
|
666
|
+
pass
|
|
667
|
+
|
|
668
|
+
|
|
669
|
+
class ONDEPath(tuple):
|
|
670
|
+
"""A path, essentially a tuple of strings or indexes/tuples, starting
|
|
671
|
+
with an entry_point name and followed by attribute names
|
|
672
|
+
for ONDEObjects or ONDEReferenceArray indexes, that leads to an ONDEBase.
|
|
673
|
+
|
|
674
|
+
An ONDEPath is evaluated by calling the _follow_path method on the object it is relative to, passing the path as the parameter."""
|
|
675
|
+
|
|
676
|
+
def __new__(cls, *args, **kwargs):
|
|
677
|
+
return super().__new__(cls,*args, **kwargs)
|
|
678
|
+
|
|
679
|
+
pass
|
|
680
|
+
|
|
681
|
+
class ONDEBase(object):
|
|
682
|
+
# _graph = None # ONDEFileGraph object
|
|
683
|
+
_referencedby = None # set of ONDEBase objects that reference this object. The _referencedby member can still be changed even after an instance is frozen becuase new objects can reference it. Note that the _referencedby field is generally only updated to include new objects when those objects become frozen. Access is protected by _referencedby_lock.
|
|
684
|
+
_referencedby_lock = None # threading.Lock object that protects _referencedby
|
|
685
|
+
|
|
686
|
+
_frozen = None # True/False: has this object been finalized and therefore become immutable
|
|
687
|
+
_modification_scopes = None # A set of scopes for which the ancestor nodes in the graph have been replaced for the transaction in which this node is being updated. It is only valid for use within the context of the transaction in which the node is being created and it is cleared when the node is frozen.
|
|
688
|
+
|
|
689
|
+
_file_realizations = None # A set of ONDEFileObjects that represent realizations in hdf5 of this object. Locked by _file_realizations_lock.
|
|
690
|
+
_file_realizations_lock = None # A threading.Lock object that protects _file_realizations.
|
|
691
|
+
|
|
692
|
+
def __init__(self, _orig = None, **kwargs):
|
|
693
|
+
_referencedby = None
|
|
694
|
+
_file_realizations = None
|
|
695
|
+
#if _orig is not None:
|
|
696
|
+
# _referencedby = object.__getattribute__(_orig, "_referencedby")
|
|
697
|
+
# pass
|
|
698
|
+
if "_referencedby" in kwargs:
|
|
699
|
+
_referencedby = set(kwargs["_referencedby"])
|
|
700
|
+
del kwargs["_referencedby"]
|
|
701
|
+
pass
|
|
702
|
+
if "_file_realizations" in kwargs:
|
|
703
|
+
_file_realizations = set(kwargs["_file_realizations"])
|
|
704
|
+
del kwargs["_file_realizations"]
|
|
705
|
+
pass
|
|
706
|
+
|
|
707
|
+
_frozen = False
|
|
708
|
+
if "_frozen" in kwargs:
|
|
709
|
+
_frozen = kwargs["_frozen"]
|
|
710
|
+
del kwargs["_frozen"]
|
|
711
|
+
pass
|
|
712
|
+
if len(kwargs) > 0:
|
|
713
|
+
raise AttributeError(f"Unknown constructor parameters to {self.__class__.__name__:s}: {str(list(kwargs.keys())):s}")
|
|
714
|
+
if _referencedby is None:
|
|
715
|
+
_referencedby = set()
|
|
716
|
+
pass
|
|
717
|
+
if _file_realizations is None:
|
|
718
|
+
_file_realizations = set()
|
|
719
|
+
pass
|
|
720
|
+
|
|
721
|
+
object.__setattr__(self, "_referencedby", _referencedby)
|
|
722
|
+
object.__setattr__(self, "_referencedby_lock", threading.Lock())
|
|
723
|
+
object.__setattr__(self, "_file_realizations", _file_realizations)
|
|
724
|
+
object.__setattr__(self, "_file_realizations_lock", threading.Lock())
|
|
725
|
+
object.__setattr__(self, "_frozen", False)
|
|
726
|
+
if _frozen:
|
|
727
|
+
self._freeze() # Derived class may have additional operations
|
|
728
|
+
pass
|
|
729
|
+
else:
|
|
730
|
+
object.__setattr__(self, "_modification_scopes", set())
|
|
731
|
+
pass
|
|
732
|
+
pass
|
|
733
|
+
|
|
734
|
+
def __getattribute__(self,name):
|
|
735
|
+
if name.startswith("_"):
|
|
736
|
+
if name in {"_freeze", "_frozen", "_add_referencedby", "__class__", "__dir__", "_get_attr","_get_attr_dataset_storage", "_set_attr","_set_dataset_attr","_get_data","_set_data","_get_item","_list_items","_set_item","_follow_path","_modification_scopes","_indices_for_object","_assign_pathel","_list_edges", "_repr", "_repr_short","_hdf5_write_group","_hdf5_write_dataset","_hdf5_write_attribute","_file_realizations","_file_realizations_lock","_add_dataset"}:
|
|
737
|
+
return object.__getattribute__(self, name)
|
|
738
|
+
raise IndexError(f"ONDEBase: The attribute {name:s} has a leading underscore, which is not allowed.")
|
|
739
|
+
raise IndexError(f"ONDEBase: Unknown attribute {name:s}")
|
|
740
|
+
|
|
741
|
+
def __setattr__(self, name, value):
|
|
742
|
+
_frozen = object.__getattribute__(self, "_frozen")
|
|
743
|
+
if _frozen:
|
|
744
|
+
raise RuntimeError("Attempting to modify an object that is already frozen")
|
|
745
|
+
# object.__setattr__(self,name,value)
|
|
746
|
+
raise ValueError(f"ONDEBase is not mutable")
|
|
747
|
+
|
|
748
|
+
# Attributes are other ONDEBase objects
|
|
749
|
+
def _get_attr(self, name):
|
|
750
|
+
"""
|
|
751
|
+
Return the named conceptual attribute of an ONDE object.
|
|
752
|
+
"""
|
|
753
|
+
|
|
754
|
+
raise ValueError("ONDEBase does not have attributes")
|
|
755
|
+
|
|
756
|
+
def _get_attr_dataset_storage(self, name):
|
|
757
|
+
"""
|
|
758
|
+
Return the named conceptual attribute of an ONDE object.
|
|
759
|
+
"""
|
|
760
|
+
|
|
761
|
+
raise ValueError("ONDEBase does not have attributes")
|
|
762
|
+
|
|
763
|
+
def _has_attr(self, name):
|
|
764
|
+
"""
|
|
765
|
+
Return whether the named conceptual attribute of an ONDE object exists.
|
|
766
|
+
"""
|
|
767
|
+
|
|
768
|
+
return False
|
|
769
|
+
|
|
770
|
+
def _set_attr(self, name, value):
|
|
771
|
+
"""
|
|
772
|
+
Set the named conceptual attribute of an ONDE object.
|
|
773
|
+
"""
|
|
774
|
+
|
|
775
|
+
raise ValueError("ONDEBase does not have attributes")
|
|
776
|
+
|
|
777
|
+
def _set_dataset_attr(self, name, value):
|
|
778
|
+
"""
|
|
779
|
+
Set the named conceptual attribute of an ONDE object.
|
|
780
|
+
"""
|
|
781
|
+
|
|
782
|
+
raise ValueError("ONDEBase does not have attributes")
|
|
783
|
+
|
|
784
|
+
# Items are indexed ONDEBase that we can access
|
|
785
|
+
def _get_item(self,index):
|
|
786
|
+
"""Return the indexed data element of an ONDE object.
|
|
787
|
+
"""
|
|
788
|
+
raise ValueError("ONDEBase does not have items")
|
|
789
|
+
|
|
790
|
+
def _set_item(self,index,value):
|
|
791
|
+
"""Set the indexed data element of an ONDE object.
|
|
792
|
+
"""
|
|
793
|
+
raise ValueError("ONDEBase does not have items")
|
|
794
|
+
|
|
795
|
+
def _list_items(self,index,value):
|
|
796
|
+
"""List the data element indices of an ONDE object.
|
|
797
|
+
"""
|
|
798
|
+
raise ValueError("ONDEBase does not have items")
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
# Data are lower level objects such as arrays of numbers, integers, strings, etc.
|
|
802
|
+
def _get_data(self,name):
|
|
803
|
+
"""Return the named data element of an ONDE object.
|
|
804
|
+
"""
|
|
805
|
+
raise ValueError("ONDEBase does not have data")
|
|
806
|
+
|
|
807
|
+
def _set_data(self,name,value):
|
|
808
|
+
"""Set the named data element of an ONDE object.
|
|
809
|
+
"""
|
|
810
|
+
raise ValueError("ONDEBase does not have data")
|
|
811
|
+
|
|
812
|
+
def _hdf5_write_group(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
813
|
+
"""Write this object to a new hdf5 group
|
|
814
|
+
"""
|
|
815
|
+
raise ValueError("ONDEBase does not have content to write")
|
|
816
|
+
|
|
817
|
+
def _hdf5_write_dataset(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
818
|
+
"""Write this object to a new hdf5 dataset
|
|
819
|
+
"""
|
|
820
|
+
raise ValueError("ONDEBase does not have content to write")
|
|
821
|
+
|
|
822
|
+
def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
823
|
+
"""Write this object to a new hdf5 attribute
|
|
824
|
+
"""
|
|
825
|
+
raise ValueError("ONDEBase does not have content to write")
|
|
826
|
+
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def __copy__(self):
|
|
831
|
+
new = self.__class__(self)
|
|
832
|
+
return new
|
|
833
|
+
|
|
834
|
+
def __hash__(self):
|
|
835
|
+
return id(self) # For now we identify ourself by our pointer. In the future perhaps we might use a content hash.
|
|
836
|
+
|
|
837
|
+
def _freeze(self):
|
|
838
|
+
_frozen = object.__getattribute__(self, "_frozen")
|
|
839
|
+
if _frozen:
|
|
840
|
+
raise RuntimeError("Attempting to freeze an object that is already frozen")
|
|
841
|
+
object.__setattr__(self, "_modification_scopes", None)
|
|
842
|
+
object.__setattr__(self, "_frozen", True)
|
|
843
|
+
pass
|
|
844
|
+
|
|
845
|
+
def _add_referencedby(self, obj_that_references_us):
|
|
846
|
+
_referencedby = object.__getattribute__(self, "_referencedby")
|
|
847
|
+
_referencedby_lock = object.__getattribute__(self, "_referencedby_lock")
|
|
848
|
+
with _referencedby_lock:
|
|
849
|
+
_referencedby.add(obj_that_references_us)
|
|
850
|
+
pass
|
|
851
|
+
pass
|
|
852
|
+
|
|
853
|
+
def _follow_path(self, path):
|
|
854
|
+
if len(path) == 0:
|
|
855
|
+
return self
|
|
856
|
+
|
|
857
|
+
raise AttributeError(f"{self.__class__.__name__} is a leaf node attempting to follow path {str(path)}")
|
|
858
|
+
|
|
859
|
+
def _assign_pathel(self, pathel, value):
|
|
860
|
+
raise AttributeError(f"{self.__class__.__name__} is a leaf node and does not support element assignment of {str(pathel)} to {str(value)}")
|
|
861
|
+
|
|
862
|
+
def _list_edges(self):
|
|
863
|
+
return []
|
|
864
|
+
|
|
865
|
+
def _indices_for_object(self,obj):
|
|
866
|
+
"""identify all of the indices for self that reference object obj. Returns a frozenset."""
|
|
867
|
+
return frozenset()
|
|
868
|
+
|
|
869
|
+
def _repr_short(self, onde_class=None):
|
|
870
|
+
return type(self).__name__
|
|
871
|
+
|
|
872
|
+
def _repr(self, indentation, onde_class=None, extra_attrs={}):
|
|
873
|
+
return repr_helper(indentation, type(self).__name__, **extra_attrs)
|
|
874
|
+
|
|
875
|
+
def __repr__(self):
|
|
876
|
+
return object.__getattribute__(self, "_repr")(0)
|
|
877
|
+
|
|
878
|
+
@classmethod
|
|
879
|
+
def new(cls):
|
|
880
|
+
raise RuntimeError("ONDEBase class is not independently instantiatable")
|
|
881
|
+
|
|
882
|
+
pass
|
|
883
|
+
|
|
884
|
+
class ONDEValue(ONDEBase):
|
|
885
|
+
"""ONDEValue represents a string, integer, float,
|
|
886
|
+
or other simple value. An ONDEValue
|
|
887
|
+
cannot reference other objects, but can be referenced
|
|
888
|
+
by other objects."""
|
|
889
|
+
value = None # Immutable value
|
|
890
|
+
|
|
891
|
+
def __init__(self, _orig = None, **kwargs):
|
|
892
|
+
if _orig is not None:
|
|
893
|
+
self.value = _orig.value
|
|
894
|
+
pass
|
|
895
|
+
if "value" in kwargs:
|
|
896
|
+
value = copy.copy(kwargs["value"])
|
|
897
|
+
if isinstance(value, numbers.Integral):
|
|
898
|
+
value = int(value)
|
|
899
|
+
pass
|
|
900
|
+
elif isinstance(value, numbers.Real):
|
|
901
|
+
value = float(value)
|
|
902
|
+
pass
|
|
903
|
+
elif isinstance(value, numbers.Complex):
|
|
904
|
+
value = complex(value)
|
|
905
|
+
pass
|
|
906
|
+
elif isinstance(value, str) or isinstance(value,np.str_):
|
|
907
|
+
value = str(value)
|
|
908
|
+
pass
|
|
909
|
+
else:
|
|
910
|
+
raise ValueError(f"ONDEValue: Cannot understand value type {value.__class__.__name__:s}")
|
|
911
|
+
self.value=value
|
|
912
|
+
del kwargs["value"]
|
|
913
|
+
pass
|
|
914
|
+
super().__init__(_orig, **kwargs)
|
|
915
|
+
pass
|
|
916
|
+
|
|
917
|
+
def __getattribute__(self,name):
|
|
918
|
+
if name == "value":
|
|
919
|
+
|
|
920
|
+
_get_data = object.__getattribute__(self, "_get_data")
|
|
921
|
+
return _get_data(name)
|
|
922
|
+
|
|
923
|
+
return super().__getattribute__(name)
|
|
924
|
+
|
|
925
|
+
def __setattr__(self,name,value):
|
|
926
|
+
if name == "value":
|
|
927
|
+
self._set_data(name,value)
|
|
928
|
+
pass
|
|
929
|
+
else:
|
|
930
|
+
|
|
931
|
+
super().__setattr__(name,value)
|
|
932
|
+
pass
|
|
933
|
+
pass
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def _get_data(self, name):
|
|
937
|
+
"""
|
|
938
|
+
Return the named conceptual data of an ONDE object.
|
|
939
|
+
"""
|
|
940
|
+
|
|
941
|
+
if name == "value":
|
|
942
|
+
return object.__getattribute__(self, name)
|
|
943
|
+
|
|
944
|
+
raise ValueError("ONDEValue does not have data other than \"value\"")
|
|
945
|
+
|
|
946
|
+
def _set_data(self, name, value):
|
|
947
|
+
"""
|
|
948
|
+
Set the named conceptual data of an ONDE object.
|
|
949
|
+
"""
|
|
950
|
+
|
|
951
|
+
if name == "value":
|
|
952
|
+
return object.__setattr__(self, name, value)
|
|
953
|
+
|
|
954
|
+
raise ValueError("ONDEValue does not have data other than \"value\"")
|
|
955
|
+
|
|
956
|
+
def _repr_short(self, onde_class=None):
|
|
957
|
+
return f"ONDEValue({str(object.__getattribute__(self, 'value')):s})"
|
|
958
|
+
|
|
959
|
+
def _repr(self, indentation, onde_class=None, extra_attrs={}):
|
|
960
|
+
# return f"ONDEValue ID={id(self):x} of type {type(object.__getattribute__(self, 'value')).__name__:s} containing value: {str(object.__getattribute__(self, 'value')):s}"
|
|
961
|
+
return repr_helper(indentation, "ONDEValue",
|
|
962
|
+
ID=f"{id(self):x}",
|
|
963
|
+
type=type(object.__getattribute__(self, 'value')).__name__,
|
|
964
|
+
value=str(object.__getattribute__(self, 'value')),
|
|
965
|
+
**extra_attrs)
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
969
|
+
"""Write this object to a new hdf5 attribute
|
|
970
|
+
"""
|
|
971
|
+
parent.attrs[name] = self.value
|
|
972
|
+
pass
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
|
|
977
|
+
@classmethod
|
|
978
|
+
def new(cls, value = None, **kwargs):
|
|
979
|
+
return cls(None, value = value, **kwargs)
|
|
980
|
+
|
|
981
|
+
pass
|
|
982
|
+
|
|
983
|
+
class ONDEArray(ONDEBase):
|
|
984
|
+
"""ONDEArray represents an array. An ONDEArray
|
|
985
|
+
cannot reference other objects, but can be referenced
|
|
986
|
+
by other objects. For an array of references, see
|
|
987
|
+
ONDEReferenceArray.
|
|
988
|
+
|
|
989
|
+
An ONDEArray can be stored either as an HDF5
|
|
990
|
+
attribute or an HDF5 dataset depending on the
|
|
991
|
+
status in the parent object and whether the attribute of the parent is set to be stored as a dataset
|
|
992
|
+
"""
|
|
993
|
+
value = None # numpy array
|
|
994
|
+
#store_as_dataset = None # boolean; store as an HDF5 dataset if True, otherwise and an HDF5 attribute
|
|
995
|
+
|
|
996
|
+
def __init__(self, _orig = None, **kwargs):
|
|
997
|
+
#store_as_dataset = False
|
|
998
|
+
value = None
|
|
999
|
+
shape = None
|
|
1000
|
+
dtype = None
|
|
1001
|
+
|
|
1002
|
+
if _orig is not None:
|
|
1003
|
+
value = _orig.value
|
|
1004
|
+
#store_as_dataset = _orig.store_as_dataset
|
|
1005
|
+
pass
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
#if "store_as_dataset" in kwargs:
|
|
1009
|
+
# store_as_dataset = bool(kwargs["store_as_dataset"])
|
|
1010
|
+
# del kwargs["store_as_dataset"]
|
|
1011
|
+
# pass
|
|
1012
|
+
if "shape" in kwargs:
|
|
1013
|
+
if kwargs["shape"] is not None:
|
|
1014
|
+
shape = tuple(kwargs["shape"])
|
|
1015
|
+
pass
|
|
1016
|
+
del kwargs["shape"]
|
|
1017
|
+
pass
|
|
1018
|
+
if "dtype" in kwargs:
|
|
1019
|
+
dtype = kwargs["dtype"]
|
|
1020
|
+
del kwargs["dtype"]
|
|
1021
|
+
pass
|
|
1022
|
+
|
|
1023
|
+
if "value" in kwargs:
|
|
1024
|
+
if kwargs["value"] is not None:
|
|
1025
|
+
value = copy.copy(kwargs["value"])
|
|
1026
|
+
if isinstance(value, collections.abc.Sequence):
|
|
1027
|
+
dtype = None
|
|
1028
|
+
if len(value) > 0 and isinstance(value[0],str):
|
|
1029
|
+
# For collection of strings, use h5py string datatype for numpy
|
|
1030
|
+
dtype = h5py.string_dtype()
|
|
1031
|
+
pass
|
|
1032
|
+
value = np.array(value,dtype=dtype)
|
|
1033
|
+
pass
|
|
1034
|
+
elif isinstance(value, np.ndarray):
|
|
1035
|
+
# value.flags.writeable = False
|
|
1036
|
+
pass
|
|
1037
|
+
else:
|
|
1038
|
+
raise ValueError(f"ONDEArray: Cannot understand value type {value.__class__.__name__:s}")
|
|
1039
|
+
pass
|
|
1040
|
+
del kwargs["value"]
|
|
1041
|
+
pass
|
|
1042
|
+
if value is None:
|
|
1043
|
+
value = np.zeros(shape, dtype = dtype)
|
|
1044
|
+
pass
|
|
1045
|
+
|
|
1046
|
+
#self.store_as_dataset = store_as_dataset
|
|
1047
|
+
self.value = value
|
|
1048
|
+
super().__init__(_orig, **kwargs)
|
|
1049
|
+
pass
|
|
1050
|
+
|
|
1051
|
+
def __getattribute__(self,name):
|
|
1052
|
+
if name == "value" or name == "shape":# or name == "store_as_dataset":
|
|
1053
|
+
|
|
1054
|
+
_get_data = object.__getattribute__(self, "_get_data")
|
|
1055
|
+
return _get_data(name)
|
|
1056
|
+
|
|
1057
|
+
return super().__getattribute__(name)
|
|
1058
|
+
|
|
1059
|
+
def __setattr__(self,name,value):
|
|
1060
|
+
if name == "value": # or name == "store_as_dataset":
|
|
1061
|
+
self._set_data(name,value)
|
|
1062
|
+
pass
|
|
1063
|
+
else:
|
|
1064
|
+
|
|
1065
|
+
super().__setattr__(name,value)
|
|
1066
|
+
pass
|
|
1067
|
+
pass
|
|
1068
|
+
|
|
1069
|
+
def _get_item(self,index):
|
|
1070
|
+
return self.value[index]
|
|
1071
|
+
|
|
1072
|
+
def __getitem__(self,index):
|
|
1073
|
+
return self._get_item(index)
|
|
1074
|
+
|
|
1075
|
+
def _set_item(self,index,el_value):
|
|
1076
|
+
if self._frozen:
|
|
1077
|
+
raise RuntimeError("Attempting to modify an object that is already frozen")
|
|
1078
|
+
|
|
1079
|
+
self.value[index] = el_value
|
|
1080
|
+
pass
|
|
1081
|
+
|
|
1082
|
+
def __setitem__(self,index,el_value):
|
|
1083
|
+
self._set_item(index,el_value)
|
|
1084
|
+
pass
|
|
1085
|
+
|
|
1086
|
+
def _get_data(self, name):
|
|
1087
|
+
"""
|
|
1088
|
+
Return the named conceptual attribute of an ONDE object.
|
|
1089
|
+
"""
|
|
1090
|
+
|
|
1091
|
+
if name == "value": # or name == "store_as_dataset":
|
|
1092
|
+
return object.__getattribute__(self, name)
|
|
1093
|
+
elif name == "shape":
|
|
1094
|
+
return object.__getattribute__(self, "value").shape
|
|
1095
|
+
|
|
1096
|
+
raise ValueError("ONDEArray does not have attributes other than \"value\" and \"store_as_dataset\"")
|
|
1097
|
+
|
|
1098
|
+
def _set_data(self, name, value):
|
|
1099
|
+
"""
|
|
1100
|
+
Set the named conceptual attribute of an ONDE object.
|
|
1101
|
+
"""
|
|
1102
|
+
if self._frozen:
|
|
1103
|
+
raise RuntimeError("Attempting to modify an object that is already frozen")
|
|
1104
|
+
|
|
1105
|
+
if name == "value": # or name == "store_as_dataset":
|
|
1106
|
+
return object.__setattr__(self, name, value)
|
|
1107
|
+
|
|
1108
|
+
raise ValueError("ONDEArray does not have attributes other than \"value\"")
|
|
1109
|
+
|
|
1110
|
+
def _freeze(self):
|
|
1111
|
+
self.value.flags.writeable = False
|
|
1112
|
+
super()._freeze()
|
|
1113
|
+
pass
|
|
1114
|
+
|
|
1115
|
+
def _repr_short(self, onde_class=None):
|
|
1116
|
+
if len(self.shape)==1 and self.shape[0]<8:
|
|
1117
|
+
return f"ONDEArray({str(list(self.value))})"
|
|
1118
|
+
return f"ONDEArray(shape={str(self.shape):s},dtype={str(self.value.dtype):s})"
|
|
1119
|
+
|
|
1120
|
+
def _repr(self, indentation, onde_class=None, extra_attrs={}):
|
|
1121
|
+
return repr_helper(indentation, "ONDEArray",
|
|
1122
|
+
ID=f"{id(self):x}",
|
|
1123
|
+
type=str(object.__getattribute__(self, 'value').dtype),
|
|
1124
|
+
shape=str(object.__getattribute__(self, 'value').shape),
|
|
1125
|
+
**extra_attrs)
|
|
1126
|
+
|
|
1127
|
+
def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
1128
|
+
"""Write this object to a new hdf5 attribute
|
|
1129
|
+
"""
|
|
1130
|
+
parent.attrs[name] = self.value
|
|
1131
|
+
pass
|
|
1132
|
+
|
|
1133
|
+
def _hdf5_write_dataset(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
1134
|
+
"""Write this object to a new hdf5 dataset
|
|
1135
|
+
"""
|
|
1136
|
+
ds = parent.create_dataset(name,data = self.value)
|
|
1137
|
+
|
|
1138
|
+
pass
|
|
1139
|
+
|
|
1140
|
+
@classmethod
|
|
1141
|
+
def new(cls, value = None, **kwargs):
|
|
1142
|
+
return cls(None, value = value, **kwargs)
|
|
1143
|
+
|
|
1144
|
+
pass
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
class ONDEReferenceArray(ONDEBase):
|
|
1148
|
+
"""Represents an array of references to other ONDEObjects.
|
|
1149
|
+
|
|
1150
|
+
An ONDEReferenceArray can be stored as an HDF5
|
|
1151
|
+
attribute"""
|
|
1152
|
+
refs = None # TwoWayArray of ONDEBase references
|
|
1153
|
+
#store_as_dataset = None # True to store as an HDF5 dataset, False to store as an HDF5 attribute
|
|
1154
|
+
|
|
1155
|
+
|
|
1156
|
+
def __init__(self,_orig, **kwargs):
|
|
1157
|
+
"""Private constructor for internal use only.
|
|
1158
|
+
Use .new() classmethod or copy.copy()
|
|
1159
|
+
"""
|
|
1160
|
+
#store_as_dataset = False
|
|
1161
|
+
refs = None
|
|
1162
|
+
if _orig is not None:
|
|
1163
|
+
refs = _orig.refs
|
|
1164
|
+
#store_as_dataset = _orig.store_as_dataset
|
|
1165
|
+
pass
|
|
1166
|
+
|
|
1167
|
+
|
|
1168
|
+
#if "store_as_dataset" in kwargs:
|
|
1169
|
+
# store_as_dataset = bool(kwargs["store_as_dataset"])
|
|
1170
|
+
# del kwargs["store_as_dataset"]
|
|
1171
|
+
# pass
|
|
1172
|
+
|
|
1173
|
+
shape = ()
|
|
1174
|
+
|
|
1175
|
+
if "shape" in kwargs:
|
|
1176
|
+
if kwargs["shape"] is not None:
|
|
1177
|
+
shape = tuple(kwargs["shape"])
|
|
1178
|
+
pass
|
|
1179
|
+
del kwargs["shape"]
|
|
1180
|
+
pass
|
|
1181
|
+
|
|
1182
|
+
if "refs" in kwargs:
|
|
1183
|
+
if kwargs["refs"] is not None:
|
|
1184
|
+
refs = TwoWayArray(byindex=kwargs["refs"])
|
|
1185
|
+
shape = kwargs["refs"].shape
|
|
1186
|
+
pass
|
|
1187
|
+
del kwargs["refs"]
|
|
1188
|
+
pass
|
|
1189
|
+
|
|
1190
|
+
if refs is None:
|
|
1191
|
+
refs = TwoWayArray(shape=shape)
|
|
1192
|
+
pass
|
|
1193
|
+
|
|
1194
|
+
#self.store_as_dataset = store_as_dataset
|
|
1195
|
+
object.__setattr__(self, "refs", refs)
|
|
1196
|
+
super().__init__(_orig,**kwargs)
|
|
1197
|
+
pass
|
|
1198
|
+
|
|
1199
|
+
def __getitem__(self,index):
|
|
1200
|
+
return self._get_item(index)
|
|
1201
|
+
|
|
1202
|
+
def __setitem__(self,index,ref):
|
|
1203
|
+
self._set_item(index,ref)
|
|
1204
|
+
pass
|
|
1205
|
+
|
|
1206
|
+
def _get_data(self, name):
|
|
1207
|
+
"""
|
|
1208
|
+
Return the named conceptual data of an ONDE object.
|
|
1209
|
+
"""
|
|
1210
|
+
|
|
1211
|
+
if name == "refs": # or name == "store_as_dataset":
|
|
1212
|
+
return object.__getattribute__(self, name)
|
|
1213
|
+
|
|
1214
|
+
raise ValueError("ONDEReferenceArray does not have attributes other than \"refs\"")
|
|
1215
|
+
|
|
1216
|
+
def _set_data(self, name, value):
|
|
1217
|
+
"""
|
|
1218
|
+
Set the named conceptual data of an ONDE object.
|
|
1219
|
+
"""
|
|
1220
|
+
|
|
1221
|
+
if name == "refs": # or name == "store_as_dataset":
|
|
1222
|
+
return object.__setattr__(self, name, value)
|
|
1223
|
+
|
|
1224
|
+
raise ValueError("ONDEReferenceArray does not have attributes other than \"refs\"")
|
|
1225
|
+
|
|
1226
|
+
def _get_item(self,index):
|
|
1227
|
+
return self.refs[index]
|
|
1228
|
+
|
|
1229
|
+
def _set_item(self,index,ref):
|
|
1230
|
+
if self._frozen:
|
|
1231
|
+
raise RuntimeError("Attempting to modify an object that is already frozen")
|
|
1232
|
+
|
|
1233
|
+
if isinstance(ref, ONDEClassInstanceWrapper) or isinstance(ref, ONDEProxy):
|
|
1234
|
+
ref = ref._get_obj()
|
|
1235
|
+
pass
|
|
1236
|
+
|
|
1237
|
+
if not isinstance(ref,ONDEBase):
|
|
1238
|
+
raise ValueError(f"Attempting to assign index {str(index)} of an ONDEReferenceArray to an object of class {ref.__class__.__name__} that is not an ONDEBase reference")
|
|
1239
|
+
|
|
1240
|
+
self.refs[index] = ref
|
|
1241
|
+
pass
|
|
1242
|
+
|
|
1243
|
+
def __getattribute__(self,name):
|
|
1244
|
+
if name == "refs":# or name == "store_as_dataset":
|
|
1245
|
+
_get_data = object.__getattribute__(self, "_get_data")
|
|
1246
|
+
return _get_data(name)
|
|
1247
|
+
|
|
1248
|
+
return super().__getattribute__(name)
|
|
1249
|
+
|
|
1250
|
+
def __setattr__(self,name,value):
|
|
1251
|
+
if name == "refs": # or name == "store_as_dataset":
|
|
1252
|
+
self._set_data(name,value)
|
|
1253
|
+
pass
|
|
1254
|
+
else:
|
|
1255
|
+
|
|
1256
|
+
super().__setattr__(name,value)
|
|
1257
|
+
pass
|
|
1258
|
+
pass
|
|
1259
|
+
|
|
1260
|
+
def _freeze(self):
|
|
1261
|
+
if self._frozen:
|
|
1262
|
+
raise RuntimeError("Attempting to freeze an object that is already frozen")
|
|
1263
|
+
self.refs._freeze()
|
|
1264
|
+
|
|
1265
|
+
nditer = iter(self.refs)
|
|
1266
|
+
for arrayref in nditer:
|
|
1267
|
+
obj = arrayref[()]
|
|
1268
|
+
if obj is not None:
|
|
1269
|
+
obj._add_referencedby(self)
|
|
1270
|
+
if not obj._frozen:
|
|
1271
|
+
obj._freeze()
|
|
1272
|
+
pass
|
|
1273
|
+
pass
|
|
1274
|
+
pass
|
|
1275
|
+
super()._freeze()
|
|
1276
|
+
pass
|
|
1277
|
+
|
|
1278
|
+
def _follow_path(self, path):
|
|
1279
|
+
if len(path) == 0:
|
|
1280
|
+
return self
|
|
1281
|
+
|
|
1282
|
+
path_entry = path[0]
|
|
1283
|
+
|
|
1284
|
+
if isinstance(path_entry, numbers.Integral) or isinstance(path_entry, collections.abc.Sequence):
|
|
1285
|
+
if self.refs[path_entry] is not None:
|
|
1286
|
+
return self.refs[path_entry]._follow_path(ONDEPath(path[1:]))
|
|
1287
|
+
return None
|
|
1288
|
+
else:
|
|
1289
|
+
raise AttributeError(f"Cannot index {self.__class__.__name__} by {path_entry}")
|
|
1290
|
+
|
|
1291
|
+
pass
|
|
1292
|
+
|
|
1293
|
+
def _assign_pathel(self, pathel, value):
|
|
1294
|
+
if self._frozen:
|
|
1295
|
+
raise RuntimeError(f"Cannot assign {str(value)} to {str(pathel)} element of frozen object")
|
|
1296
|
+
self.refs[pathel]=value
|
|
1297
|
+
pass
|
|
1298
|
+
|
|
1299
|
+
def _list_edges(self):
|
|
1300
|
+
nditer = iter(self.refs) #np.nditer object
|
|
1301
|
+
sz = nditer.itersize
|
|
1302
|
+
|
|
1303
|
+
edgelist= []
|
|
1304
|
+
for cnt in range(sz):
|
|
1305
|
+
next(nditer)
|
|
1306
|
+
|
|
1307
|
+
if self.refs[nditer.multi_index] is not None:
|
|
1308
|
+
edgelist.append(nditer.multi_index)
|
|
1309
|
+
pass
|
|
1310
|
+
pass
|
|
1311
|
+
return edgelist
|
|
1312
|
+
|
|
1313
|
+
def _indices_for_object(self,obj):
|
|
1314
|
+
"""identify all of the indices for self that reference object obj. Returns a frozenset."""
|
|
1315
|
+
return self.refs(obj) # __call__ method does reverse lookup to return a set of indices
|
|
1316
|
+
|
|
1317
|
+
def _repr_short(self, onde_class=None):
|
|
1318
|
+
return f"ONDEReferenceArray(shape={str(object.__getattribute__(self, 'refs')._shape):s})"
|
|
1319
|
+
|
|
1320
|
+
def _repr(self, indentation, onde_class=None, extra_attrs={}):
|
|
1321
|
+
return repr_helper(indentation, "ONDEReferenceArray",
|
|
1322
|
+
ID=f"{id(self):x}",
|
|
1323
|
+
shape=str(object.__getattribute__(self, 'refs')._shape),
|
|
1324
|
+
**extra_attrs)
|
|
1325
|
+
|
|
1326
|
+
def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
1327
|
+
"""Write this object to a new hdf5 attribute
|
|
1328
|
+
"""
|
|
1329
|
+
ref_dtype = h5py.special_dtype(ref = h5py.Reference)
|
|
1330
|
+
|
|
1331
|
+
byindex = self.refs.byindex
|
|
1332
|
+
h5_byindex = np.empty(byindex.shape,dtype = ref_dtype)
|
|
1333
|
+
nditer = np.nditer(byindex, flags = ("multi_index","refs_ok"))
|
|
1334
|
+
|
|
1335
|
+
for objarray in nditer:
|
|
1336
|
+
obj = objarray[()]
|
|
1337
|
+
if obj is not None:
|
|
1338
|
+
# !!!*** This may need to be accelerated by caching the hdf5 object references per the various discussions on h5py indexing performance with Paul Wilcox, early 2026. https://github.com/COFREND/ONDE-format/issues/24
|
|
1339
|
+
h5_byindex[nditer.multi_index] = onde_file.fh[onde_file.file_object_dict[obj].hdf5_path].ref
|
|
1340
|
+
pass
|
|
1341
|
+
pass
|
|
1342
|
+
|
|
1343
|
+
parent.attrs[name] = h5_byindex
|
|
1344
|
+
pass
|
|
1345
|
+
|
|
1346
|
+
def _hdf5_write_dataset(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
1347
|
+
"""Write this object to a new hdf5 dataset
|
|
1348
|
+
"""
|
|
1349
|
+
ref_dtype = h5py.special_dtype(ref = h5py.Reference)
|
|
1350
|
+
|
|
1351
|
+
byindex = self.refs.byindex
|
|
1352
|
+
h5_byindex = np.zeros(byindex.shape,dtype = ref_dtype)
|
|
1353
|
+
nditer = np.nditer(byindex, flags = ("multi_index","refs_ok"))
|
|
1354
|
+
|
|
1355
|
+
for objarray in nditer:
|
|
1356
|
+
obj = objarray[()]
|
|
1357
|
+
if obj is not None:
|
|
1358
|
+
# !!!*** This may need to be accelerated by caching the hdf5 object references per the various discussions on h5py indexing performance with Paul Wilcox, early 2026. https://github.com/COFREND/ONDE-format/issues/24
|
|
1359
|
+
h5_byindex[nditer.multi_index] = onde_file.fh[onde_file.file_object_dict[obj].hdf5_path].ref
|
|
1360
|
+
pass
|
|
1361
|
+
pass
|
|
1362
|
+
ds = parent.create_dataset(name,data=h5_byindex)
|
|
1363
|
+
pass
|
|
1364
|
+
|
|
1365
|
+
@classmethod
|
|
1366
|
+
def new(cls, file_or_graph, refs = None, shape = None, **kwargs):
|
|
1367
|
+
graph = None
|
|
1368
|
+
|
|
1369
|
+
if isinstance(file_or_graph, ONDEFileGraph):
|
|
1370
|
+
graph = file_or_graph
|
|
1371
|
+
pass
|
|
1372
|
+
elif file_or_graph is not None:
|
|
1373
|
+
graph = file_or_graph.graph
|
|
1374
|
+
pass
|
|
1375
|
+
|
|
1376
|
+
newobj= cls(None, refs = refs, shape = shape, **kwargs)
|
|
1377
|
+
|
|
1378
|
+
if file_or_graph is not None:
|
|
1379
|
+
return ONDEClassInstanceWrapper(_graph=graph,_obj=newobj)
|
|
1380
|
+
return newobj
|
|
1381
|
+
|
|
1382
|
+
@classmethod
|
|
1383
|
+
def load_from_hdf5(cls,onde_file,fileobjs_by_h5path,h5_fh,np_h5ref_array):
|
|
1384
|
+
refs = np.empty(np_h5ref_array.shape,dtype = "O")
|
|
1385
|
+
nditer = np.nditer(np_h5ref_array, flags = ("multi_index","refs_ok"))
|
|
1386
|
+
|
|
1387
|
+
for refarray in nditer:
|
|
1388
|
+
ref = refarray[()]
|
|
1389
|
+
if ref is not None and not "(null)" in str(ref):
|
|
1390
|
+
refs[nditer.multi_index] = ONDEObject.load_from_hdf5(onde_file,fileobjs_by_h5path,h5_fh,h5_fh[ref])
|
|
1391
|
+
pass
|
|
1392
|
+
pass
|
|
1393
|
+
instance = cls(None,refs = refs, shape = np_h5ref_array.shape)
|
|
1394
|
+
return instance
|
|
1395
|
+
|
|
1396
|
+
pass
|
|
1397
|
+
|
|
1398
|
+
|
|
1399
|
+
class ONDEObject(ONDEBase):
|
|
1400
|
+
"""ONDEObject represents a (non-leaf) node in the graph
|
|
1401
|
+
that can point at other objects."""
|
|
1402
|
+
#ONDE_TYPE = None # List of classes starting with base class
|
|
1403
|
+
_ONDE_attrs = None # TwoWayDictionary by name of attributes that should be ONDEBase (or subclass) objects).
|
|
1404
|
+
_ONDE_dataset_attrs = None # Sub-set of ONDE_attrs that should use hdf5 dataset (as opposed to hdf5 attribute) storage.
|
|
1405
|
+
|
|
1406
|
+
def __init__(self, _orig, **kwargs):
|
|
1407
|
+
"""Private constructor for internal use only.
|
|
1408
|
+
Use .new() classmethod or copy.copy()
|
|
1409
|
+
"""
|
|
1410
|
+
|
|
1411
|
+
#ONDE_TYPE = None
|
|
1412
|
+
#if _orig is not None:
|
|
1413
|
+
# ONDE_TYPE = object.__getattribute__(_orig, "ONDE_TYPE")
|
|
1414
|
+
# pass
|
|
1415
|
+
#if "ONDE_TYPE" in kwargs:
|
|
1416
|
+
# ONDE_TYPE = kwargs["ONDE_TYPE"]
|
|
1417
|
+
# del kwargs["ONDE_TYPE"]
|
|
1418
|
+
# pass
|
|
1419
|
+
|
|
1420
|
+
_ONDE_attrs = None
|
|
1421
|
+
_ONDE_dataset_attrs = None
|
|
1422
|
+
if _orig is not None:
|
|
1423
|
+
_ONDE_attrs = object.__getattribute__(_orig, "_ONDE_attrs")
|
|
1424
|
+
_ONDE_dataset_attrs = object.__getattribute__(_orig,"_ONDE_dataset_attrs")
|
|
1425
|
+
pass
|
|
1426
|
+
if "_ONDE_attrs" in kwargs:
|
|
1427
|
+
#if _ONDE_attrs is not None:
|
|
1428
|
+
# _ONDE_attrs.update(kwargs["_ONDE_attrs"])
|
|
1429
|
+
# pass
|
|
1430
|
+
#else:
|
|
1431
|
+
_ONDE_attrs = kwargs["_ONDE_attrs"]
|
|
1432
|
+
# pass
|
|
1433
|
+
del kwargs["_ONDE_attrs"]
|
|
1434
|
+
pass
|
|
1435
|
+
|
|
1436
|
+
if "_ONDE_dataset_attrs" in kwargs:
|
|
1437
|
+
_ONDE_dataset_attrs = kwargs["_ONDE_dataset_attrs"]
|
|
1438
|
+
del kwargs["_ONDE_dataset_attrs"]
|
|
1439
|
+
pass
|
|
1440
|
+
|
|
1441
|
+
#object.__setattr__(self, "ONDE_TYPE", tuple(ONDE_TYPE))
|
|
1442
|
+
ONDE_attrs = TwoWayDictionary(_ONDE_attrs)
|
|
1443
|
+
if not "ONDE:TYPE" in ONDE_attrs:
|
|
1444
|
+
ONDE_attrs["ONDE:TYPE"] = ()
|
|
1445
|
+
pass
|
|
1446
|
+
|
|
1447
|
+
object.__setattr__(self, "_ONDE_attrs", ONDE_attrs)
|
|
1448
|
+
|
|
1449
|
+
if _ONDE_dataset_attrs is None :
|
|
1450
|
+
_ONDE_dataset_attrs = set()
|
|
1451
|
+
pass
|
|
1452
|
+
ONDE_dataset_attrs = set(_ONDE_dataset_attrs)
|
|
1453
|
+
object.__setattr__(self,"_ONDE_dataset_attrs",ONDE_dataset_attrs)
|
|
1454
|
+
|
|
1455
|
+
base_kwargs = {}
|
|
1456
|
+
for kwarg in kwargs:
|
|
1457
|
+
if kwarg.startswith("_"):
|
|
1458
|
+
base_kwargs[kwarg] = kwargs[kwarg]
|
|
1459
|
+
pass
|
|
1460
|
+
else:
|
|
1461
|
+
ONDE_attrs._set_attr(kwarg,kwargs[kwarg])
|
|
1462
|
+
pass
|
|
1463
|
+
pass
|
|
1464
|
+
super().__init__(_orig, **base_kwargs)
|
|
1465
|
+
pass
|
|
1466
|
+
|
|
1467
|
+
def __getattribute__(self, name):
|
|
1468
|
+
if name.startswith("_"):
|
|
1469
|
+
if name in {"_freeze", "_frozen", "_add_referencedby", "__class__", "__dir__", "_get_attr","_get_attr_dataset_storage", "_set_attr","_set_dataset_attr","_follow_path","_modification_scopes","_indices_for_object","_assign_pathel","_list_edges","_ONDE_attrs","_ONDE_dataset_attrs", "_has_attr","_repr","_repr_short","_hdf5_write_group","_list_attrs","_hdf5_write_dataset","_hdf5_write_attribute","_file_realizations","_file_realizations_lock","_onde_class_name","_get_item","_set_item","_list_items","_has_item"}:
|
|
1470
|
+
return object.__getattribute__(self, name)
|
|
1471
|
+
raise IndexError("ONDEObject: Attributes may not have leading underscores")
|
|
1472
|
+
|
|
1473
|
+
_get_attr = object.__getattribute__(self, "_get_attr")
|
|
1474
|
+
|
|
1475
|
+
return _get_attr(name)
|
|
1476
|
+
|
|
1477
|
+
def __setattr__(self, name, value):
|
|
1478
|
+
_frozen = object.__getattribute__(self, "_frozen")
|
|
1479
|
+
if _frozen:
|
|
1480
|
+
raise RuntimeError("Attempting to modify an object that is already frozen")
|
|
1481
|
+
|
|
1482
|
+
_set_attr = object.__getattribute__(self, "_set_attr")
|
|
1483
|
+
_set_attr(name, value)
|
|
1484
|
+
|
|
1485
|
+
pass
|
|
1486
|
+
|
|
1487
|
+
def _get_item(self,name):
|
|
1488
|
+
return self._get_attr(name)
|
|
1489
|
+
|
|
1490
|
+
def _set_item(self,name,value):
|
|
1491
|
+
return self._set_attr(name,value)
|
|
1492
|
+
|
|
1493
|
+
def __getitem__(self,name):
|
|
1494
|
+
return self._get_attr(name)
|
|
1495
|
+
|
|
1496
|
+
def __setitem__(self,name,value):
|
|
1497
|
+
self._set_attr(name,value)
|
|
1498
|
+
pass
|
|
1499
|
+
|
|
1500
|
+
def _get_attr(self, name):
|
|
1501
|
+
"""
|
|
1502
|
+
Return the named conceptual attribute of an ONDE object.
|
|
1503
|
+
"""
|
|
1504
|
+
if not isinstance(name,str):
|
|
1505
|
+
raise IndexError(f"Indexes must be strings")
|
|
1506
|
+
if name.startswith("_"):
|
|
1507
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
1508
|
+
|
|
1509
|
+
_ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
|
|
1510
|
+
|
|
1511
|
+
return _ONDE_attrs._get_attr(name)
|
|
1512
|
+
|
|
1513
|
+
def _get_attr_dataset_storage(self, name):
|
|
1514
|
+
"""
|
|
1515
|
+
Return whether the named conceptual attribute of an ONDE object uses hdf5 dataset storage.
|
|
1516
|
+
"""
|
|
1517
|
+
|
|
1518
|
+
if name.startswith("_"):
|
|
1519
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
1520
|
+
|
|
1521
|
+
_ONDE_dataset_attrs = object.__getattribute__(self, "_ONDE_dataset_attrs")
|
|
1522
|
+
|
|
1523
|
+
return name in _ONDE_dataset_attrs
|
|
1524
|
+
|
|
1525
|
+
def _has_attr(self, name):
|
|
1526
|
+
"""
|
|
1527
|
+
Return whether the named conceptual attribute of an ONDE object exists.
|
|
1528
|
+
"""
|
|
1529
|
+
_ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
|
|
1530
|
+
if name in _ONDE_attrs:
|
|
1531
|
+
return True
|
|
1532
|
+
return False
|
|
1533
|
+
def _has_item(self,name):
|
|
1534
|
+
return self._has_attr(name)
|
|
1535
|
+
|
|
1536
|
+
def __contains__(self,name):
|
|
1537
|
+
return self._has_attr(name)
|
|
1538
|
+
|
|
1539
|
+
def _set_attr(self, name, value):
|
|
1540
|
+
"""
|
|
1541
|
+
Set the named conceptual attribute of an ONDE object. Use _set_dataset_attr() instead if the attribute should have hdf5 dataset storage.
|
|
1542
|
+
"""
|
|
1543
|
+
_frozen = object.__getattribute__(self, "_frozen")
|
|
1544
|
+
if _frozen:
|
|
1545
|
+
raise RuntimeError("Attempting to modify an object that is already frozen")
|
|
1546
|
+
|
|
1547
|
+
if not isinstance(self, ONDEFileGraphSnapshot) and not ":" in name:
|
|
1548
|
+
raise ValueError(f"ONDEObject: Attempting to set field \"{name:s}\" with no colon-delimited defining class. Usually this means there is a typo in the field name, or that the field is not defined in your class definitions for some reason. If you are trying to add a custom field, be sure to use a custom namespace and accessory class or derived class. You can use bracket notation (e.g. object[\"CLASSNAME:FIELD\"]=...) to assign a field name that contains a colon manually if you do not have a formal class definition loaded")
|
|
1549
|
+
|
|
1550
|
+
if name.startswith("_"):
|
|
1551
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
1552
|
+
|
|
1553
|
+
_ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
|
|
1554
|
+
_ONDE_dataset_attrs = object.__getattribute__(self, "_ONDE_dataset_attrs")
|
|
1555
|
+
|
|
1556
|
+
if isinstance(value, ONDEClassInstanceWrapper) or isinstance(value, ONDEProxy):
|
|
1557
|
+
value = value._get_obj()
|
|
1558
|
+
pass
|
|
1559
|
+
|
|
1560
|
+
_ONDE_dataset_attrs.discard(name)
|
|
1561
|
+
return _ONDE_attrs._set_attr(name, value)
|
|
1562
|
+
|
|
1563
|
+
def _set_dataset_attr(self, name, value):
|
|
1564
|
+
"""
|
|
1565
|
+
Set the named conceptual attribute of an ONDE object with dataset storage. Use _set_attr() instead if the attribute should have hdf5 attribute storage.
|
|
1566
|
+
"""
|
|
1567
|
+
_frozen = object.__getattribute__(self, "_frozen")
|
|
1568
|
+
if _frozen:
|
|
1569
|
+
raise RuntimeError("Attempting to modify an object that is already frozen")
|
|
1570
|
+
|
|
1571
|
+
if not isinstance(self, ONDEFileGraphSnapshot) and not ":" in name:
|
|
1572
|
+
raise ValueError(f"ONDEObject: Attempting to set field \"{name:s}\" with no colon-delimited defining class. Usually this means there is a typo in the field name, or that the field is not defined in your class definitions for some reason. If you are trying to add a custom field, be sure to use a custom namespace and accessory class or derived class. You can use bracket notation (e.g. object[\"CLASSNAME:FIELD\"]=...) to assign a field name that contains a colon manually if you do not have a formal class definition loaded")
|
|
1573
|
+
|
|
1574
|
+
if name.startswith("_"):
|
|
1575
|
+
raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
|
|
1576
|
+
|
|
1577
|
+
_ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
|
|
1578
|
+
_ONDE_dataset_attrs = object.__getattribute__(self, "_ONDE_dataset_attrs")
|
|
1579
|
+
|
|
1580
|
+
if isinstance(value, ONDEClassInstanceWrapper) or isinstance(value, ONDEProxy):
|
|
1581
|
+
value = value._get_obj()
|
|
1582
|
+
pass
|
|
1583
|
+
|
|
1584
|
+
_ONDE_dataset_attrs.add(name)
|
|
1585
|
+
return _ONDE_attrs._set_attr(name, value)
|
|
1586
|
+
|
|
1587
|
+
def _list_attrs(self):
|
|
1588
|
+
_ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
|
|
1589
|
+
return [attrname for attrname in _ONDE_attrs]
|
|
1590
|
+
|
|
1591
|
+
def __dir__(self):
|
|
1592
|
+
_ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
|
|
1593
|
+
return ["_freeze","_frozen"] + [attrname for attrname in _ONDE_attrs]
|
|
1594
|
+
|
|
1595
|
+
def _freeze(self):
|
|
1596
|
+
_frozen = object.__getattribute__(self, "_frozen")
|
|
1597
|
+
if _frozen:
|
|
1598
|
+
raise RuntimeError("Attempting to freeze an object that is already frozen")
|
|
1599
|
+
_ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
|
|
1600
|
+
_ONDE_attrs._freeze()
|
|
1601
|
+
for attrname in _ONDE_attrs:
|
|
1602
|
+
obj = getattr(_ONDE_attrs, attrname)
|
|
1603
|
+
obj._add_referencedby(self)
|
|
1604
|
+
if obj is not None and not obj._frozen:
|
|
1605
|
+
obj._freeze()
|
|
1606
|
+
pass
|
|
1607
|
+
pass
|
|
1608
|
+
|
|
1609
|
+
super()._freeze()
|
|
1610
|
+
pass
|
|
1611
|
+
|
|
1612
|
+
def _follow_path(self, path):
|
|
1613
|
+
if len(path) == 0:
|
|
1614
|
+
return self
|
|
1615
|
+
|
|
1616
|
+
path_entry = path[0]
|
|
1617
|
+
|
|
1618
|
+
if isinstance(path_entry, str):
|
|
1619
|
+
#return self._get_attr(path_entry)._follow_path(ONDEPath(path[1:]))
|
|
1620
|
+
# ONDEFileGraphSnapshot (our subclass) uses items not attributes. So here we explicitly use our own _get_attr() to expand out the path entry
|
|
1621
|
+
return ONDEObject._get_attr(self,path_entry)._follow_path(ONDEPath(path[1:]))
|
|
1622
|
+
else:
|
|
1623
|
+
raise AttributeError(f"Cannot index {self.__class__.__name__} by {path_entry}")
|
|
1624
|
+
|
|
1625
|
+
pass
|
|
1626
|
+
|
|
1627
|
+
def _assign_pathel(self, pathel, value):
|
|
1628
|
+
if self._frozen:
|
|
1629
|
+
raise RuntimeError(f"Cannot assign {str(value)} to {str(pathel)} element of frozen object")
|
|
1630
|
+
# ONDEFileGraphSnapshot (our subclass) uses items not attributes. So here we explicitly use our own _set_attr() to expand out the path entry
|
|
1631
|
+
#self._set_attr(pathel, value)
|
|
1632
|
+
ONDEObject._set_attr(self,pathel, value)
|
|
1633
|
+
pass
|
|
1634
|
+
|
|
1635
|
+
def _list_edges(self):
|
|
1636
|
+
_ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
|
|
1637
|
+
edgelist= list(_ONDE_attrs._keys())
|
|
1638
|
+
return edgelist
|
|
1639
|
+
|
|
1640
|
+
def _indices_for_object(self,obj):
|
|
1641
|
+
"""identify all of the indices for self that reference object obj. Returns a frozenset."""
|
|
1642
|
+
_ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
|
|
1643
|
+
return _ONDE_attrs(obj) # __call__ method does reverse lookup to return a set of indices
|
|
1644
|
+
|
|
1645
|
+
def _repr_short(self, onde_class=None):
|
|
1646
|
+
myclass=type(self).__name__
|
|
1647
|
+
if onde_class is not None:
|
|
1648
|
+
myclass=f"{onde_class:s}({myclass:s})"
|
|
1649
|
+
pass
|
|
1650
|
+
|
|
1651
|
+
return f"{myclass:s}(ID={id(self):x})"
|
|
1652
|
+
|
|
1653
|
+
def _repr(self, indentation, onde_class=None, extra_attrs={}):
|
|
1654
|
+
myclass=type(self).__name__
|
|
1655
|
+
if onde_class is not None:
|
|
1656
|
+
myclass=f"{onde_class:s}({myclass:s})"
|
|
1657
|
+
pass
|
|
1658
|
+
|
|
1659
|
+
new_extra_attrs=collections.OrderedDict()
|
|
1660
|
+
if onde_class is None:
|
|
1661
|
+
_ONDE_attrs=object.__getattribute__(self, '_ONDE_attrs')
|
|
1662
|
+
_ONDE_dataset_attrs = object.__getattribute__(self, "_ONDE_dataset_attrs")
|
|
1663
|
+
|
|
1664
|
+
keys=list(_ONDE_attrs._keys())
|
|
1665
|
+
shortkeys=keys
|
|
1666
|
+
|
|
1667
|
+
if isinstance(self,ONDEFileGraphSnapshot):
|
|
1668
|
+
shortkeys=self.shortkeys()
|
|
1669
|
+
|
|
1670
|
+
pass
|
|
1671
|
+
|
|
1672
|
+
for keyidx in range(len(keys)):
|
|
1673
|
+
attrname=keys[keyidx]
|
|
1674
|
+
attrnameshort=shortkeys[keyidx]
|
|
1675
|
+
storage_suffix = ""
|
|
1676
|
+
if attrname in _ONDE_dataset_attrs:
|
|
1677
|
+
storage_suffix = " (hdf5 dataset storage)"
|
|
1678
|
+
pass
|
|
1679
|
+
attrval=_ONDE_attrs[attrname]
|
|
1680
|
+
if attrval is not None:
|
|
1681
|
+
new_extra_attrs[attrnameshort]=object.__getattribute__(attrval, "_repr_short")() + storage_suffix
|
|
1682
|
+
pass
|
|
1683
|
+
else:
|
|
1684
|
+
new_extra_attrs[attrnameshort]="None" + storage_suffix
|
|
1685
|
+
pass
|
|
1686
|
+
|
|
1687
|
+
pass
|
|
1688
|
+
pass
|
|
1689
|
+
new_extra_attrs.update(extra_attrs)
|
|
1690
|
+
return repr_helper(indentation, myclass,
|
|
1691
|
+
ID=f"{id(self):x}",
|
|
1692
|
+
**new_extra_attrs)
|
|
1693
|
+
|
|
1694
|
+
def _onde_class_name(self):
|
|
1695
|
+
classname = "(None)"
|
|
1696
|
+
if "ONDE:TYPE" in self._ONDE_attrs:
|
|
1697
|
+
if isinstance(self._ONDE_attrs["ONDE:TYPE"],ONDEArray):
|
|
1698
|
+
if len(self._ONDE_attrs["ONDE:TYPE"].value) > 0:
|
|
1699
|
+
classname = self._ONDE_attrs["ONDE:TYPE"].value[0]
|
|
1700
|
+
pass
|
|
1701
|
+
pass
|
|
1702
|
+
pass
|
|
1703
|
+
return classname
|
|
1704
|
+
|
|
1705
|
+
def _hdf5_write_group(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
1706
|
+
"""Write this object to a new hdf5 group
|
|
1707
|
+
"""
|
|
1708
|
+
gr = parent.create_group(name)
|
|
1709
|
+
for attrname in self._ONDE_attrs:
|
|
1710
|
+
value = self._ONDE_attrs[attrname]
|
|
1711
|
+
if attrname in self._ONDE_dataset_attrs:
|
|
1712
|
+
if isinstance(value,ONDEObject) or isinstance(value,ONDEValue):
|
|
1713
|
+
raise ValueError(f"Error writing ONDE object of class {self._onde_class_name()}: Attribute {attrname} is marked for dataset storage, but is an {type(value).__name__} not an ONDEArray or ONDEReferenceArray")
|
|
1714
|
+
value._hdf5_write_dataset(onde_file,gr,gr.name,attrname,None)
|
|
1715
|
+
pass
|
|
1716
|
+
else:
|
|
1717
|
+
value._hdf5_write_attribute(onde_file,gr,gr.name,attrname,None)
|
|
1718
|
+
pass
|
|
1719
|
+
pass
|
|
1720
|
+
|
|
1721
|
+
pass
|
|
1722
|
+
|
|
1723
|
+
def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
|
|
1724
|
+
"""Write a reference to this object to a new hdf5 attribute
|
|
1725
|
+
"""
|
|
1726
|
+
parent.attrs[name] = onde_file.fh[onde_file.file_object_dict[self].hdf5_path].ref
|
|
1727
|
+
pass
|
|
1728
|
+
|
|
1729
|
+
|
|
1730
|
+
@classmethod
|
|
1731
|
+
def new(cls, file_or_graph,onde_classname, ONDE_TYPE = None, _ONDE_attrs = None,_ONDE_dataset_attrs = None, **kwargs):
|
|
1732
|
+
"""Main constructor to call. If file_or_graph is provided, then the ONDEObject will be created but in fact the returned python object will be instead an ONDEClassInstanceWrapper with the context of that file or graph and the corresponding class definitions."""
|
|
1733
|
+
|
|
1734
|
+
graph = None
|
|
1735
|
+
|
|
1736
|
+
if isinstance(file_or_graph, ONDEFileGraph):
|
|
1737
|
+
graph = file_or_graph
|
|
1738
|
+
pass
|
|
1739
|
+
elif file_or_graph is not None:
|
|
1740
|
+
graph = file_or_graph.graph
|
|
1741
|
+
pass
|
|
1742
|
+
|
|
1743
|
+
if onde_classname is not None and ONDE_TYPE is None:
|
|
1744
|
+
class_derivation = [ onde_classname ]
|
|
1745
|
+
if graph is not None and graph.class_defs is not None:
|
|
1746
|
+
class_defs = graph.class_defs
|
|
1747
|
+
if onde_classname in class_defs.classes:
|
|
1748
|
+
onde_class = class_defs.classes[onde_classname]
|
|
1749
|
+
class_derivation = onde_class.class_derivation
|
|
1750
|
+
pass
|
|
1751
|
+
pass
|
|
1752
|
+
|
|
1753
|
+
ONDE_TYPE = ONDEArray.new(class_derivation, _frozen = True)
|
|
1754
|
+
pass
|
|
1755
|
+
|
|
1756
|
+
constructargs = {}
|
|
1757
|
+
if ONDE_TYPE is not None:
|
|
1758
|
+
constructargs["ONDE:TYPE"] = ONDE_TYPE
|
|
1759
|
+
pass
|
|
1760
|
+
if _ONDE_attrs is not None:
|
|
1761
|
+
constructargs["_ONDE_attrs"] = _ONDE_attrs
|
|
1762
|
+
pass
|
|
1763
|
+
if _ONDE_dataset_attrs is not None:
|
|
1764
|
+
constructargs["_ONDE_dataset_attrs"] = _ONDE_dataset_attrs
|
|
1765
|
+
pass
|
|
1766
|
+
|
|
1767
|
+
constructargs.update(kwargs)
|
|
1768
|
+
newobj = cls(None, **constructargs)
|
|
1769
|
+
#for attrname in kwargs:
|
|
1770
|
+
# setattr(newobj, attrname, kwargs[attrname])
|
|
1771
|
+
# pass
|
|
1772
|
+
|
|
1773
|
+
if file_or_graph is not None:
|
|
1774
|
+
return ONDEClassInstanceWrapper(_graph=graph,_obj=newobj)
|
|
1775
|
+
return newobj
|
|
1776
|
+
|
|
1777
|
+
@classmethod
|
|
1778
|
+
def load_from_hdf5(cls,onde_file,fileobjs_by_h5path,h5_fh,h5_group):
|
|
1779
|
+
_ONDE_attrs = collections.OrderedDict()
|
|
1780
|
+
_ONDE_dataset_attrs = set()
|
|
1781
|
+
|
|
1782
|
+
if h5_group.name in fileobjs_by_h5path:
|
|
1783
|
+
return fileobjs_by_h5path[h5_group.name].onde_instance
|
|
1784
|
+
|
|
1785
|
+
for attrname in h5_group.attrs:
|
|
1786
|
+
if not ":" in attrname:
|
|
1787
|
+
raise ValueError(f"Error loading fields from hdf5 group {h5_group.name:s} in file {onde_file.h5path:s}: field name {attrname:s} does not contain a colon separator between the defining class and the name.")
|
|
1788
|
+
h5_attr = h5_group.attrs[attrname]
|
|
1789
|
+
if isinstance(h5_attr,str) or isinstance(h5_attr,int) or isinstance(h5_attr,float):
|
|
1790
|
+
attr_obj = ONDEValue.new(h5_attr)
|
|
1791
|
+
_ONDE_attrs[attrname] = attr_obj
|
|
1792
|
+
pass
|
|
1793
|
+
|
|
1794
|
+
elif isinstance(h5_attr,np.ndarray):
|
|
1795
|
+
string_info = h5py.check_string_dtype(h5_attr.dtype)
|
|
1796
|
+
if string_info is not None:
|
|
1797
|
+
# array of strings
|
|
1798
|
+
pass
|
|
1799
|
+
|
|
1800
|
+
# Check if hdf5 reference
|
|
1801
|
+
ref_type = h5py.check_dtype(ref = h5_attr.dtype)
|
|
1802
|
+
if ref_type is not None:
|
|
1803
|
+
# array of references
|
|
1804
|
+
|
|
1805
|
+
attr_obj = ONDEReferenceArray.load_from_hdf5(onde_file,fileobjs_by_h5path,h5_fh,h5_attr)
|
|
1806
|
+
_ONDE_attrs[attrname] = attr_obj
|
|
1807
|
+
pass
|
|
1808
|
+
else:
|
|
1809
|
+
# Array of values
|
|
1810
|
+
attr_obj = ONDEArray.new(value = h5_attr)
|
|
1811
|
+
_ONDE_attrs[attrname] = attr_obj
|
|
1812
|
+
pass
|
|
1813
|
+
pass
|
|
1814
|
+
elif isinstance(h5_attr,h5py.Reference):
|
|
1815
|
+
attr_obj = ONDEObject.load_from_hdf5(onde_file, fileobjs_by_h5path, h5_fh, h5_fh[h5_attr])
|
|
1816
|
+
_ONDE_attrs[attrname] = attr_obj
|
|
1817
|
+
|
|
1818
|
+
pass
|
|
1819
|
+
else:
|
|
1820
|
+
raise ValueError(f"Unknown hdf5 attribute type: {type(h5_attr).__name__:s}")
|
|
1821
|
+
pass
|
|
1822
|
+
for (subname,subobj) in h5_group.items():
|
|
1823
|
+
if isinstance(subobj,h5py.Dataset):
|
|
1824
|
+
# Check if hdf5 reference
|
|
1825
|
+
ref_type = h5py.check_dtype(ref = subobj.dtype)
|
|
1826
|
+
if ref_type is not None:
|
|
1827
|
+
# array of references
|
|
1828
|
+
|
|
1829
|
+
attr_obj = ONDEReferenceArray.load_from_hdf5(onde_file,fileobjs_by_h5path,h5_fh,subobj[...])
|
|
1830
|
+
_ONDE_attrs[subname] = attr_obj
|
|
1831
|
+
_ONDE_dataset_attrs.add(subname)
|
|
1832
|
+
pass
|
|
1833
|
+
else:
|
|
1834
|
+
# Array of values
|
|
1835
|
+
attr_obj = ONDEArray.new(value = subobj[...])
|
|
1836
|
+
_ONDE_attrs[subname] = attr_obj
|
|
1837
|
+
_ONDE_dataset_attrs.add(subname)
|
|
1838
|
+
pass
|
|
1839
|
+
pass
|
|
1840
|
+
# elif isinstance(subobj,h5py.Group): # These lines will enable support for hierarchical ONDE files
|
|
1841
|
+
# attr_obj = ONDEObject.load_from_hdf5(onde_file, fileobjs_by_h5path, h5_fh, subobj)
|
|
1842
|
+
# _ONDE_attrs[subname] = attr_obj
|
|
1843
|
+
else:
|
|
1844
|
+
raise ValueError(f"Unknown h5type {type(subobj).__name__:s} at hdf5 path {h5_group.name:s}")
|
|
1845
|
+
pass
|
|
1846
|
+
instance = cls(None,_ONDE_attrs = _ONDE_attrs,_ONDE_dataset_attrs = _ONDE_dataset_attrs)
|
|
1847
|
+
fileobj = ONDEFileObject(onde_instance = instance,hdf5_path = h5_group.name)
|
|
1848
|
+
fileobjs_by_h5path[h5_group.name] = fileobj
|
|
1849
|
+
return instance
|
|
1850
|
+
pass
|
|
1851
|
+
|
|
1852
|
+
class ONDEFileGraphSnapshot(ONDEObject):
|
|
1853
|
+
""" Not allowed to be referenced by any other ONDEObject.
|
|
1854
|
+
The ONDE:TYPE field should be absent.
|
|
1855
|
+
Attributes represent entry points of the graph."""
|
|
1856
|
+
def __init__(self, _orig = None, **kwargs):
|
|
1857
|
+
super().__init__(_orig, **kwargs)
|
|
1858
|
+
# remove the ONDE:TYPE added by the ONDEObject constructor because we are not really a proper object
|
|
1859
|
+
_ONDE_attrs = object.__getattribute__(self,"_ONDE_attrs")
|
|
1860
|
+
del _ONDE_attrs["ONDE:TYPE"]
|
|
1861
|
+
pass
|
|
1862
|
+
|
|
1863
|
+
# Unlike ONDEObject, we index with _get_item, not with
|
|
1864
|
+
# _get_attr. Therefore we override the attribute methods
|
|
1865
|
+
# with the ones from ONDEBase.
|
|
1866
|
+
|
|
1867
|
+
_get_attr=ONDEBase._get_attr
|
|
1868
|
+
_has_attr=ONDEBase._has_attr
|
|
1869
|
+
_set_attr=ONDEBase._set_attr
|
|
1870
|
+
|
|
1871
|
+
# __getattribute__=ONDEBase.__getattribute__
|
|
1872
|
+
__getattribute__=object.__getattribute__
|
|
1873
|
+
__setattr__=ONDEBase.__setattr__
|
|
1874
|
+
|
|
1875
|
+
|
|
1876
|
+
|
|
1877
|
+
#def _set_item(self,index,value):
|
|
1878
|
+
# return ONDEObject._set_attr(self,index,value)
|
|
1879
|
+
|
|
1880
|
+
def _get_item(self,index):
|
|
1881
|
+
# Allow shorthand keys
|
|
1882
|
+
keys = self._list_items()
|
|
1883
|
+
if index in keys:
|
|
1884
|
+
return ONDEObject._get_attr(self,index)
|
|
1885
|
+
|
|
1886
|
+
prefixed = "2.25." + index
|
|
1887
|
+
matching = [ key for key in keys if key.startswith(prefixed) ]
|
|
1888
|
+
if len(matching) == 1:
|
|
1889
|
+
return ONDEObject._get_attr(self,matching[0])
|
|
1890
|
+
|
|
1891
|
+
raise KeyError(f"No Dataset with UUID matching given key \"{index:s}\"")
|
|
1892
|
+
def __getitem__(self,index):
|
|
1893
|
+
return self._get_item(index)
|
|
1894
|
+
#def __setitem__(self,index,value):
|
|
1895
|
+
# return self._set_item(index,value)
|
|
1896
|
+
|
|
1897
|
+
def _add_dataset(self,key,dataset):
|
|
1898
|
+
ONDEObject._set_attr(self,key,dataset)
|
|
1899
|
+
pass
|
|
1900
|
+
|
|
1901
|
+
def add(self,dataset):
|
|
1902
|
+
# Must maintain parallel logic in ONDEProxy._add()
|
|
1903
|
+
if "ONDE:TYPE" not in dataset or dataset["ONDE:TYPE"].value[0] != "ONDE_DATASET":
|
|
1904
|
+
raise ValueError(f"Attempting to add something other than a dataset to an ONDEFileGraphSnapshot")
|
|
1905
|
+
|
|
1906
|
+
if "ONDE:UUID" not in dataset:
|
|
1907
|
+
raise ValueError(f"Attempting to add a dataset without a UUID to an ONDEFileGraphSnapshot")
|
|
1908
|
+
|
|
1909
|
+
self._add_dataset(dataset["ONDE:UUID"].value,dataset)
|
|
1910
|
+
pass
|
|
1911
|
+
|
|
1912
|
+
def _list_items(self):
|
|
1913
|
+
return ONDEObject._list_attrs(self)
|
|
1914
|
+
|
|
1915
|
+
def keys(self):
|
|
1916
|
+
return self._list_items()
|
|
1917
|
+
|
|
1918
|
+
def shortkeys(self):
|
|
1919
|
+
full_keys = self._list_items()
|
|
1920
|
+
# shortable_keys is a dictionary listing all of the keys that we can use a shorthand for (i.e. those starting with 2.25.). Values are intialized to none but will be populated with the shorthand eventually
|
|
1921
|
+
shortable_keys = { key: None for key in full_keys if key.startswith("2.25.") }
|
|
1922
|
+
shortable_key_set = frozenset(shortable_keys.keys())
|
|
1923
|
+
#import pdb
|
|
1924
|
+
#pdb.set_trace()
|
|
1925
|
+
for shortable_key in shortable_key_set:
|
|
1926
|
+
for num_chars_for_unique in range(4,len(shortable_key)+1):
|
|
1927
|
+
prefix = shortable_key[:(5 + num_chars_for_unique)]
|
|
1928
|
+
num_keys_matching_prefix = len([ key for key in shortable_key_set if key[:(5 + num_chars_for_unique)] == prefix ])
|
|
1929
|
+
if prefix in full_keys:
|
|
1930
|
+
num_keys_matching_prefix += 1
|
|
1931
|
+
pass
|
|
1932
|
+
if num_keys_matching_prefix == 1:
|
|
1933
|
+
shortable_keys[shortable_key] = prefix[5:]
|
|
1934
|
+
break
|
|
1935
|
+
if num_chars_for_unique == len(shortable_key):
|
|
1936
|
+
#Didn't find any prefix
|
|
1937
|
+
shortable_keys[shortable_key] = shortable_key
|
|
1938
|
+
pass
|
|
1939
|
+
pass
|
|
1940
|
+
pass
|
|
1941
|
+
reduced_keys = [ shortable_keys[key] if key in shortable_keys else key for key in full_keys ]
|
|
1942
|
+
return reduced_keys
|
|
1943
|
+
|
|
1944
|
+
|
|
1945
|
+
def __iter__(self):
|
|
1946
|
+
return iter(self.keys())
|
|
1947
|
+
|
|
1948
|
+
#@classmethod
|
|
1949
|
+
#def new(cls, entry_points = None):
|
|
1950
|
+
# if entry_points is None:
|
|
1951
|
+
# entry_points = TwoWayDictionary()
|
|
1952
|
+
# pass
|
|
1953
|
+
# return cls(None, ONDE_TYPE = [], _ONDE_attrs = entry_points)
|
|
1954
|
+
@classmethod
|
|
1955
|
+
def new(cls, **kwargs):
|
|
1956
|
+
_frozen = False
|
|
1957
|
+
if "_frozen" in kwargs:
|
|
1958
|
+
_frozen = kwargs["_frozen"]
|
|
1959
|
+
del kwargs["_frozen"]
|
|
1960
|
+
pass
|
|
1961
|
+
newobj = cls(None)
|
|
1962
|
+
for attrname in kwargs:
|
|
1963
|
+
newobj._set_item(attrname, kwargs[attrname])
|
|
1964
|
+
pass
|
|
1965
|
+
if _frozen:
|
|
1966
|
+
newobj._freeze()
|
|
1967
|
+
pass
|
|
1968
|
+
return newobj
|
|
1969
|
+
|
|
1970
|
+
pass
|
|
1971
|
+
|
|
1972
|
+
class ONDEField(object):
|
|
1973
|
+
"""Represents a field from the .csv spec."""
|
|
1974
|
+
|
|
1975
|
+
defining_class = None # Name of the defining class
|
|
1976
|
+
class_prefix = None # Prefix on the field
|
|
1977
|
+
name = None # Field name, not including prefix or separating colon
|
|
1978
|
+
comments = None # Documentation string
|
|
1979
|
+
mandatory = None # True for mandatory attributes/datasets
|
|
1980
|
+
storage = None # "D" for dataset storage, "A" for attribute storage, "A or D" for either
|
|
1981
|
+
content_class_string = None # Referenced type
|
|
1982
|
+
dimensionality_string = None # Dimensionality field from .csv
|
|
1983
|
+
size_or_content_string = None # Value field from .csv
|
|
1984
|
+
min_val_string = None # Min field from .csv
|
|
1985
|
+
max_val_string = None # Max field from .csv
|
|
1986
|
+
|
|
1987
|
+
def __init__(self, **kwargs):
|
|
1988
|
+
for arg in kwargs:
|
|
1989
|
+
if hasattr(self, arg):
|
|
1990
|
+
setattr(self, arg, kwargs[arg])
|
|
1991
|
+
pass
|
|
1992
|
+
else:
|
|
1993
|
+
raise ValueError(f"ONDEField; unknown attribute {arg:s}")
|
|
1994
|
+
pass
|
|
1995
|
+
pass
|
|
1996
|
+
|
|
1997
|
+
@classmethod
|
|
1998
|
+
def from_csv_line(cls, row, class_derivation):
|
|
1999
|
+
(classname, name, comments, mandatory_optional, dataset_attribute, content_class, dims, size_or_content, min_val, max_val, accessory_class) = row
|
|
2000
|
+
|
|
2001
|
+
name_split = name.split(":")
|
|
2002
|
+
class_prefix = name_split[0]
|
|
2003
|
+
class_derivation = (classname,) if class_derivation is None else class_derivation
|
|
2004
|
+
|
|
2005
|
+
if len(name_split) != 2:
|
|
2006
|
+
raise ValueError(f"Field name {name:s} should have one colon")
|
|
2007
|
+
|
|
2008
|
+
if class_prefix not in class_derivation and not (class_prefix == "ONDE" and name in ["ONDE:LABEL", "ONDE:TYPE_TAGS","ONDE:UUID"]):
|
|
2009
|
+
raise ValueError(f"Mismatch between class name or super classes and prefix defining {name:s}")
|
|
2010
|
+
|
|
2011
|
+
if mandatory_optional not in ["M", "O"]:
|
|
2012
|
+
raise ValueError(f"Mandatory/Optional field is not either O or M defining {name:s}")
|
|
2013
|
+
|
|
2014
|
+
if dataset_attribute not in ["A", "D","A or D"]:
|
|
2015
|
+
raise ValueError(f"Dataset/Attribute field is not either D or A defining {name:s}")
|
|
2016
|
+
|
|
2017
|
+
name_only = name_split[1]
|
|
2018
|
+
mandatory = mandatory_optional == "M"
|
|
2019
|
+
storage = dataset_attribute
|
|
2020
|
+
|
|
2021
|
+
return cls(
|
|
2022
|
+
defining_class=classname,
|
|
2023
|
+
class_prefix=class_prefix,
|
|
2024
|
+
name=name_only,
|
|
2025
|
+
comments=comments,
|
|
2026
|
+
mandatory=mandatory,
|
|
2027
|
+
storage=storage,
|
|
2028
|
+
content_class_string=content_class,
|
|
2029
|
+
dimensionality_string=dims,
|
|
2030
|
+
size_or_content_string=size_or_content,
|
|
2031
|
+
min_val_string = min_val,
|
|
2032
|
+
max_val_string = max_val
|
|
2033
|
+
)
|
|
2034
|
+
pass
|
|
2035
|
+
|
|
2036
|
+
|
|
2037
|
+
class ONDEAccessoryClass(object):
|
|
2038
|
+
"""Represents an accessory class defined in the ONDE .csv spec."""
|
|
2039
|
+
|
|
2040
|
+
classname = None # Name of the accessory class
|
|
2041
|
+
attributes = None # Dictionary by name of ONDEField references
|
|
2042
|
+
comments = None # Comments about this class (sourced from the .csv file)
|
|
2043
|
+
|
|
2044
|
+
def __init__(self):
|
|
2045
|
+
self.attributes = collections.OrderedDict()
|
|
2046
|
+
pass
|
|
2047
|
+
pass
|
|
2048
|
+
|
|
2049
|
+
class ONDEClass(object):
|
|
2050
|
+
"""Represents a class defined in the ONDE .csv spec."""
|
|
2051
|
+
classname = None
|
|
2052
|
+
class_derivation = None # tuple of strings starting with base class and ending with this current class
|
|
2053
|
+
superclass = None # Reference the ONDEClass object for our superclass, or None
|
|
2054
|
+
type_tags = None # Dictionary by name of truth value for mandatory
|
|
2055
|
+
attributes = None # Dictionary by name of ONDEField references
|
|
2056
|
+
comments = None # Comments about thhis class (sourced from the .csv file)
|
|
2057
|
+
|
|
2058
|
+
def __init__(self):
|
|
2059
|
+
self.attributes = collections.OrderedDict()
|
|
2060
|
+
self.type_tags = collections.OrderedDict()
|
|
2061
|
+
pass
|
|
2062
|
+
pass
|
|
2063
|
+
|
|
2064
|
+
class ONDEClassInstanceWrapper(object):
|
|
2065
|
+
"""Represents an ONDEObject that is an instance of a known class. References the proxy to the underlying ONDEObject and the class definition."""
|
|
2066
|
+
|
|
2067
|
+
# only _proxy OR _obj can ever be set
|
|
2068
|
+
_proxy = None # an ONDEProxy for the ONDEObject instance
|
|
2069
|
+
_obj = None # an ONDEObject instance
|
|
2070
|
+
_graph = None # ONDEFileGraph, used only with _obj
|
|
2071
|
+
# _our_class = None # the ONDEClass instance
|
|
2072
|
+
|
|
2073
|
+
def __init__(self, **kwargs):
|
|
2074
|
+
# _dict = object.__getattribute__(self, "__dict__")
|
|
2075
|
+
|
|
2076
|
+
for arg in kwargs:
|
|
2077
|
+
if arg in type(self).__dict__:
|
|
2078
|
+
# setattr(self, arg, kwargs[arg])
|
|
2079
|
+
object.__setattr__(self, arg, kwargs[arg])
|
|
2080
|
+
|
|
2081
|
+
pass
|
|
2082
|
+
else:
|
|
2083
|
+
raise ValueError(f"ONDEClassInstanceWrapper; unknown attribute {arg:s}")
|
|
2084
|
+
pass
|
|
2085
|
+
pass
|
|
2086
|
+
|
|
2087
|
+
def _get_obj(self):
|
|
2088
|
+
if self._obj is not None:
|
|
2089
|
+
return self._obj
|
|
2090
|
+
return self._proxy._get_obj()
|
|
2091
|
+
|
|
2092
|
+
def _full_fieldname_from_attrname(self, _graph, obj, attrname):
|
|
2093
|
+
#_graph = object.__getattribute__(self,"_graph")
|
|
2094
|
+
if attrname in onde_basic_attrs[obj.__class__.__name__]:
|
|
2095
|
+
return (attrname, None)
|
|
2096
|
+
|
|
2097
|
+
if isinstance(obj,ONDEObject):
|
|
2098
|
+
classdefs = _graph.class_defs
|
|
2099
|
+
if attrname=="ONDE:TYPE":
|
|
2100
|
+
return (attrname, None)
|
|
2101
|
+
(shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set) = classdefs.get_fields_dict(obj)
|
|
2102
|
+
if attrname in combined_fields_dict:
|
|
2103
|
+
field = combined_fields_dict[attrname]
|
|
2104
|
+
full_name = field.class_prefix + ":" + field.name
|
|
2105
|
+
return (full_name, field)
|
|
2106
|
+
else:
|
|
2107
|
+
# raise NameError(f"Unknown attribute {attrname:s} on ONDEObject of type {obj._ONDE_attrs['ONDE:TYPE'].value[-1]:s}.")
|
|
2108
|
+
return (None, None)
|
|
2109
|
+
pass
|
|
2110
|
+
return (attrname, None)
|
|
2111
|
+
|
|
2112
|
+
def _get_attr_or_item(self,get_method_name,key):
|
|
2113
|
+
|
|
2114
|
+
_proxy = object.__getattribute__(self,"_proxy")
|
|
2115
|
+
_obj = object.__getattribute__(self,"_obj")
|
|
2116
|
+
_graph = object.__getattribute__(self,"_graph")
|
|
2117
|
+
|
|
2118
|
+
if _proxy is not None:
|
|
2119
|
+
_obj = _proxy._get_obj()
|
|
2120
|
+
_graph = _proxy._graph
|
|
2121
|
+
pass
|
|
2122
|
+
|
|
2123
|
+
full_name = key
|
|
2124
|
+
if get_method_name == "_get_attr":
|
|
2125
|
+
(full_name, field) = self._full_fieldname_from_attrname(_graph, _obj, key)
|
|
2126
|
+
if full_name is None:
|
|
2127
|
+
full_name = key
|
|
2128
|
+
pass
|
|
2129
|
+
|
|
2130
|
+
if not _obj._has_attr(full_name):
|
|
2131
|
+
return None
|
|
2132
|
+
|
|
2133
|
+
pass
|
|
2134
|
+
|
|
2135
|
+
if _proxy is not None:
|
|
2136
|
+
get_method = getattr(_proxy,get_method_name)
|
|
2137
|
+
value = get_method(full_name) # e.g. _proxy._get_attr(key)
|
|
2138
|
+
if isinstance(value,ONDEProxy):
|
|
2139
|
+
value_obj = value._get_obj()
|
|
2140
|
+
if isinstance(value_obj,ONDEObject) or isinstance(value_obj,ONDEReferenceArray):
|
|
2141
|
+
return self.__class__.new_from_proxy(value)
|
|
2142
|
+
return value
|
|
2143
|
+
return value
|
|
2144
|
+
else:
|
|
2145
|
+
|
|
2146
|
+
get_method = getattr(_obj,get_method_name)
|
|
2147
|
+
value = get_method(full_name) # e.g. _obj._get_attr(name)
|
|
2148
|
+
|
|
2149
|
+
if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
|
|
2150
|
+
|
|
2151
|
+
value_wrapper = self.__class__.new_from_obj(_graph,value)
|
|
2152
|
+
return value_wrapper
|
|
2153
|
+
else:
|
|
2154
|
+
|
|
2155
|
+
return value
|
|
2156
|
+
pass
|
|
2157
|
+
pass
|
|
2158
|
+
|
|
2159
|
+
|
|
2160
|
+
|
|
2161
|
+
def _has_attr_or_item(self,has_method_name,key):
|
|
2162
|
+
|
|
2163
|
+
_proxy = object.__getattribute__(self,"_proxy")
|
|
2164
|
+
_obj = object.__getattribute__(self,"_obj")
|
|
2165
|
+
_graph = object.__getattribute__(self,"_graph")
|
|
2166
|
+
|
|
2167
|
+
if _proxy is not None:
|
|
2168
|
+
_obj = _proxy._get_obj()
|
|
2169
|
+
_graph = _proxy._graph
|
|
2170
|
+
pass
|
|
2171
|
+
|
|
2172
|
+
full_name = key
|
|
2173
|
+
|
|
2174
|
+
(full_name, field) = self._full_fieldname_from_attrname(_graph, _obj, key)
|
|
2175
|
+
if full_name is None:
|
|
2176
|
+
full_name = key
|
|
2177
|
+
pass
|
|
2178
|
+
|
|
2179
|
+
if has_method_name == "_has_attr":
|
|
2180
|
+
return _obj._has_attr(full_name)
|
|
2181
|
+
|
|
2182
|
+
if has_method_name == "_has_item":
|
|
2183
|
+
return _obj._has_item(full_name)
|
|
2184
|
+
return False
|
|
2185
|
+
|
|
2186
|
+
def _get_attr(self,name):
|
|
2187
|
+
return self._get_attr_or_item("_get_attr",name)
|
|
2188
|
+
|
|
2189
|
+
def __contains__(self,name):
|
|
2190
|
+
return self._has_item(name)
|
|
2191
|
+
|
|
2192
|
+
def _get_attr_dataset_storage(self,name):
|
|
2193
|
+
obj = self._get_obj()
|
|
2194
|
+
return obj._get_attr_dataset_storage(name)
|
|
2195
|
+
|
|
2196
|
+
def _set_attr_or_item(self,set_method_name,key,value):
|
|
2197
|
+
our_proxy = object.__getattribute__(self,"_proxy")
|
|
2198
|
+
our_obj = object.__getattribute__(self,"_obj")
|
|
2199
|
+
_graph = object.__getattribute__(self,"_graph")
|
|
2200
|
+
|
|
2201
|
+
if our_proxy is not None:
|
|
2202
|
+
our_obj = our_proxy._get_obj()
|
|
2203
|
+
_graph = our_proxy._graph
|
|
2204
|
+
pass
|
|
2205
|
+
|
|
2206
|
+
full_name = key
|
|
2207
|
+
field = None
|
|
2208
|
+
if set_method_name == "_set_attr" or set_method_name == "_set_dataset_attr":
|
|
2209
|
+
(full_name, field) = self._full_fieldname_from_attrname(_graph, our_obj, key)
|
|
2210
|
+
if full_name is None:
|
|
2211
|
+
full_name = key
|
|
2212
|
+
pass
|
|
2213
|
+
pass
|
|
2214
|
+
|
|
2215
|
+
if field is not None and set_method_name == "_set_attr":
|
|
2216
|
+
# For now, if field.storage is "A or D", we always just store it as a dataset.
|
|
2217
|
+
if field.storage != "A":
|
|
2218
|
+
set_method_name = "_set_dataset_attr"
|
|
2219
|
+
pass
|
|
2220
|
+
pass
|
|
2221
|
+
|
|
2222
|
+
if isinstance(value,ONDEClassInstanceWrapper):
|
|
2223
|
+
target_proxy = object.__getattribute__(value,"_proxy")
|
|
2224
|
+
target_obj = object.__getattribute__(value,"_obj")
|
|
2225
|
+
if our_proxy is not None and target_proxy is not None:
|
|
2226
|
+
set_method = getattr(our_proxy,set_method_name)
|
|
2227
|
+
set_method(full_name,target_proxy) # self._proxy._set_attr(key,target_proxy)
|
|
2228
|
+
pass
|
|
2229
|
+
elif self._proxy is not None and target_obj is not None:
|
|
2230
|
+
set_method = getattr(self._proxy,set_method_name)
|
|
2231
|
+
set_method(full_name,target_obj)
|
|
2232
|
+
pass
|
|
2233
|
+
elif self._obj is not None and target_proxy is not None:
|
|
2234
|
+
set_method = getattr(self._obj,set_method_name)
|
|
2235
|
+
set_method(full_name,target_proxy._get_obj()) # e.g. self._obj._set_attr(key,_proxy._get_obj())
|
|
2236
|
+
pass
|
|
2237
|
+
elif self._obj is not None and target_obj is not None:
|
|
2238
|
+
set_method = getattr(self._obj,set_method_name)
|
|
2239
|
+
set_method(full_name,target_obj)
|
|
2240
|
+
pass
|
|
2241
|
+
else:
|
|
2242
|
+
assert(False) # Exactly one of _obj and _proxy should be valid
|
|
2243
|
+
pass
|
|
2244
|
+
pass
|
|
2245
|
+
elif isinstance(value,ONDEProxy):
|
|
2246
|
+
if our_proxy is not None:
|
|
2247
|
+
set_method = getattr(our_proxy,set_method_name)
|
|
2248
|
+
set_method(full_name,value)
|
|
2249
|
+
pass
|
|
2250
|
+
elif our_obj is not None:
|
|
2251
|
+
set_method = getattr(our_obj,set_method_name)
|
|
2252
|
+
set_method(full_name,value._get_obj())
|
|
2253
|
+
pass
|
|
2254
|
+
else:
|
|
2255
|
+
assert(False)
|
|
2256
|
+
pass
|
|
2257
|
+
pass
|
|
2258
|
+
elif isinstance(value,ONDEBase):
|
|
2259
|
+
if our_proxy is not None:
|
|
2260
|
+
set_method = getattr(our_proxy,set_method_name)
|
|
2261
|
+
set_method(full_name,value)
|
|
2262
|
+
pass
|
|
2263
|
+
elif our_obj is not None:
|
|
2264
|
+
set_method = getattr(our_obj,set_method_name)
|
|
2265
|
+
set_method(full_name,value)
|
|
2266
|
+
pass
|
|
2267
|
+
else:
|
|
2268
|
+
assert(False)
|
|
2269
|
+
pass
|
|
2270
|
+
pass
|
|
2271
|
+
else:
|
|
2272
|
+
#raise ValueError(f"Attribute values should be ONDEBase or ONDEProxy or ONDEClassInstanceWrapper")
|
|
2273
|
+
#build an object from python structures
|
|
2274
|
+
target_obj = onde_from_python(value, field)
|
|
2275
|
+
if our_proxy is not None:
|
|
2276
|
+
set_method = getattr(our_proxy,set_method_name)
|
|
2277
|
+
set_method(full_name,target_obj)
|
|
2278
|
+
pass
|
|
2279
|
+
elif our_obj is not None:
|
|
2280
|
+
set_method = getattr(our_obj,set_method_name)
|
|
2281
|
+
set_method(full_name,target_obj)
|
|
2282
|
+
pass
|
|
2283
|
+
else:
|
|
2284
|
+
assert(False)
|
|
2285
|
+
pass
|
|
2286
|
+
pass
|
|
2287
|
+
pass
|
|
2288
|
+
|
|
2289
|
+
|
|
2290
|
+
|
|
2291
|
+
def _set_attr(self,name,value):
|
|
2292
|
+
self._set_attr_or_item("_set_attr",name,value)
|
|
2293
|
+
pass
|
|
2294
|
+
|
|
2295
|
+
def _set_dataset_attr(self,name,value):
|
|
2296
|
+
self._set_attr_or_item("_set_dataset_attr",name,value)
|
|
2297
|
+
pass
|
|
2298
|
+
|
|
2299
|
+
def _get_item(self,index):
|
|
2300
|
+
return self._get_attr_or_item("_get_item",index)
|
|
2301
|
+
|
|
2302
|
+
def _has_item(self,index):
|
|
2303
|
+
return self._has_attr_or_item("_has_item",index)
|
|
2304
|
+
|
|
2305
|
+
def _set_item(self,index,value):
|
|
2306
|
+
self._set_attr_or_item("_set_item",index,value)
|
|
2307
|
+
pass
|
|
2308
|
+
|
|
2309
|
+
def _get_data(self,name):
|
|
2310
|
+
_proxy = object.__getattribute__(self,"_proxy")
|
|
2311
|
+
_obj = object.__getattribute__(self,"_obj")
|
|
2312
|
+
|
|
2313
|
+
if _proxy is not None:
|
|
2314
|
+
value = _proxy._get_data(self,name)
|
|
2315
|
+
|
|
2316
|
+
return value
|
|
2317
|
+
else:
|
|
2318
|
+
_graph = object.__getattribute__(self,"_graph")
|
|
2319
|
+
value = _obj._get_data(self,name)
|
|
2320
|
+
return value
|
|
2321
|
+
pass
|
|
2322
|
+
|
|
2323
|
+
def _set_data(self,name,value):
|
|
2324
|
+
if isinstance(value,ONDEClassInstanceWrapper) or isinstance(value,ONDEBase) or isinstance(value,ONDEProxy):
|
|
2325
|
+
raise ValueError(f"_set_data(): value must not be an ONDEClassInstanceWrapper, a ONDEProxy, or an ONDEBase subclass")
|
|
2326
|
+
if self._proxy is not None:
|
|
2327
|
+
self._proxy._set_data(name,value)
|
|
2328
|
+
pass
|
|
2329
|
+
elif self._obj is not None:
|
|
2330
|
+
self._obj._set_data(name,value)
|
|
2331
|
+
pass
|
|
2332
|
+
else:
|
|
2333
|
+
assert(False)
|
|
2334
|
+
pass
|
|
2335
|
+
pass
|
|
2336
|
+
|
|
2337
|
+
def _basic_attr_set(self):
|
|
2338
|
+
if self._proxy is not None:
|
|
2339
|
+
obj = self._proxy._get_obj()
|
|
2340
|
+
pass
|
|
2341
|
+
else:
|
|
2342
|
+
obj = self._obj
|
|
2343
|
+
pass
|
|
2344
|
+
|
|
2345
|
+
fields = onde_basic_attrs[obj.__class__.__name__]
|
|
2346
|
+
return frozenset(fields)
|
|
2347
|
+
|
|
2348
|
+
def _class_field_set(self):
|
|
2349
|
+
if self._proxy is not None:
|
|
2350
|
+
obj = self._proxy._get_obj()
|
|
2351
|
+
pass
|
|
2352
|
+
else:
|
|
2353
|
+
obj = self._obj
|
|
2354
|
+
pass
|
|
2355
|
+
|
|
2356
|
+
if self._proxy is not None:
|
|
2357
|
+
_graph = self._proxy._graph
|
|
2358
|
+
pass
|
|
2359
|
+
else:
|
|
2360
|
+
_graph = self._graph
|
|
2361
|
+
pass
|
|
2362
|
+
|
|
2363
|
+
classdefs = _graph.class_defs
|
|
2364
|
+
if classdefs is not None:
|
|
2365
|
+
|
|
2366
|
+
if isinstance(obj, ONDEObject):
|
|
2367
|
+
(shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(obj)
|
|
2368
|
+
return frozenset(list(full_fields_dict.keys()))
|
|
2369
|
+
pass
|
|
2370
|
+
|
|
2371
|
+
return frozenset()
|
|
2372
|
+
|
|
2373
|
+
def _shortkeys(self):
|
|
2374
|
+
obj = self._get_obj()
|
|
2375
|
+
if isinstance(obj,ONDEObject):
|
|
2376
|
+
if self._proxy is not None:
|
|
2377
|
+
_graph = self._proxy._graph
|
|
2378
|
+
pass
|
|
2379
|
+
else:
|
|
2380
|
+
_graph = self._graph
|
|
2381
|
+
pass
|
|
2382
|
+
obj_fields = self._assigned_attr_field_set()
|
|
2383
|
+
|
|
2384
|
+
classdefs = _graph.class_defs
|
|
2385
|
+
if classdefs is not None:
|
|
2386
|
+
(shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(obj)
|
|
2387
|
+
# Make concise_set mutable
|
|
2388
|
+
concise_set = set(concise_set)
|
|
2389
|
+
for obj_field in obj_fields:
|
|
2390
|
+
if obj_field in full_fields_dict and obj_field not in concise_set:
|
|
2391
|
+
# This field is in the class definitions but it is not in the concise key set. Therefore there must be a shorthand in the concise set. So we don't need to consider this field.
|
|
2392
|
+
pass
|
|
2393
|
+
else:
|
|
2394
|
+
# Make sure this field is in the concise set
|
|
2395
|
+
concise_set.add(obj_field)
|
|
2396
|
+
pass
|
|
2397
|
+
pass
|
|
2398
|
+
return list(concise_set)
|
|
2399
|
+
return list(obj_fields)
|
|
2400
|
+
return obj._shortkeys()
|
|
2401
|
+
|
|
2402
|
+
def _keys(self):
|
|
2403
|
+
obj = self._get_obj()
|
|
2404
|
+
|
|
2405
|
+
if isinstance(obj,ONDEObject):
|
|
2406
|
+
|
|
2407
|
+
full_fields = self._class_field_set()
|
|
2408
|
+
obj_fields = self._assigned_attr_field_set()
|
|
2409
|
+
return list(full_fields | obj_fields)
|
|
2410
|
+
return obj._keys()
|
|
2411
|
+
|
|
2412
|
+
def _shorthand_field_set(self):
|
|
2413
|
+
if self._proxy is not None:
|
|
2414
|
+
obj = self._proxy._get_obj()
|
|
2415
|
+
pass
|
|
2416
|
+
else:
|
|
2417
|
+
obj = self._obj
|
|
2418
|
+
pass
|
|
2419
|
+
|
|
2420
|
+
if self._proxy is not None:
|
|
2421
|
+
_graph = self._proxy._graph
|
|
2422
|
+
pass
|
|
2423
|
+
else:
|
|
2424
|
+
_graph = self._graph
|
|
2425
|
+
pass
|
|
2426
|
+
|
|
2427
|
+
classdefs = _graph.class_defs
|
|
2428
|
+
if classdefs is not None:
|
|
2429
|
+
|
|
2430
|
+
if isinstance(obj, ONDEObject):
|
|
2431
|
+
(shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(obj)
|
|
2432
|
+
return frozenset(list(shorthand_fields_dict.keys()))
|
|
2433
|
+
pass
|
|
2434
|
+
|
|
2435
|
+
return frozenset()
|
|
2436
|
+
|
|
2437
|
+
def _assigned_attr_field_set(self):
|
|
2438
|
+
"""Fields defined by actual assigned attributes of this object"""
|
|
2439
|
+
if self._proxy is not None:
|
|
2440
|
+
obj = self._proxy._get_obj()
|
|
2441
|
+
pass
|
|
2442
|
+
else:
|
|
2443
|
+
obj = self._obj
|
|
2444
|
+
pass
|
|
2445
|
+
|
|
2446
|
+
return frozenset(obj._list_attrs())
|
|
2447
|
+
|
|
2448
|
+
def _all_field_set(self):
|
|
2449
|
+
return list(self._basic_attr_set()|self._class_field_set()|self._shorthand_field_set()|self._assigned_attr_field_set()).sorted()
|
|
2450
|
+
|
|
2451
|
+
def __dir__(self):
|
|
2452
|
+
if self._proxy is not None:
|
|
2453
|
+
obj = self._proxy._get_obj()
|
|
2454
|
+
pass
|
|
2455
|
+
else:
|
|
2456
|
+
obj = self._obj
|
|
2457
|
+
pass
|
|
2458
|
+
|
|
2459
|
+
if self._proxy is not None:
|
|
2460
|
+
_graph = self._proxy._graph
|
|
2461
|
+
pass
|
|
2462
|
+
else:
|
|
2463
|
+
_graph = self._graph
|
|
2464
|
+
pass
|
|
2465
|
+
|
|
2466
|
+
classdefs = _graph.class_defs
|
|
2467
|
+
if classdefs is not None:
|
|
2468
|
+
|
|
2469
|
+
if isinstance(obj, ONDEObject):
|
|
2470
|
+
(shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(obj)
|
|
2471
|
+
return sorted(list(concise_set|self._basic_attr_set()))
|
|
2472
|
+
pass
|
|
2473
|
+
|
|
2474
|
+
return []
|
|
2475
|
+
|
|
2476
|
+
|
|
2477
|
+
|
|
2478
|
+
def __getattribute__(self,name):
|
|
2479
|
+
if name.startswith('_'):
|
|
2480
|
+
return object.__getattribute__(self,name)
|
|
2481
|
+
|
|
2482
|
+
return self._get_attr_or_item("_get_attr",name)
|
|
2483
|
+
obsolete = r"""
|
|
2484
|
+
if self._proxy is not None:
|
|
2485
|
+
obj = self._proxy._get_obj()
|
|
2486
|
+
_graph = self._proxy._graph
|
|
2487
|
+
pass
|
|
2488
|
+
else:
|
|
2489
|
+
obj = self._obj
|
|
2490
|
+
_graph = self._graph
|
|
2491
|
+
pass
|
|
2492
|
+
|
|
2493
|
+
field_dict = {}
|
|
2494
|
+
if isinstance(obj,ONDEObject):
|
|
2495
|
+
classdefs = _graph.class_defs
|
|
2496
|
+
field_dict = classdefs.get_fields_dict(obj)
|
|
2497
|
+
if name in field_dict:
|
|
2498
|
+
field = field_dict[name]
|
|
2499
|
+
full_name = field.class_prefix + ":" + field.name
|
|
2500
|
+
value = obj._ONDE_attrs[full_name]
|
|
2501
|
+
if self._proxy is not None:
|
|
2502
|
+
valueproxy = _proxy._get_attr(full_name)
|
|
2503
|
+
if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
|
|
2504
|
+
return self.__class__.new_from_proxy(valueproxy)
|
|
2505
|
+
else:
|
|
2506
|
+
return valueproxy
|
|
2507
|
+
pass
|
|
2508
|
+
else:
|
|
2509
|
+
if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
|
|
2510
|
+
return self.__class__.new_from_obj(self._graph, value)
|
|
2511
|
+
else:
|
|
2512
|
+
return value
|
|
2513
|
+
pass
|
|
2514
|
+
pass
|
|
2515
|
+
pass
|
|
2516
|
+
else:
|
|
2517
|
+
assert(isinstance(obj, ONDEReferenceArray))
|
|
2518
|
+
value = obj._get_attr(name)
|
|
2519
|
+
|
|
2520
|
+
if self._proxy is not None:
|
|
2521
|
+
valueproxy = _proxy._get_attr(name)
|
|
2522
|
+
if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
|
|
2523
|
+
return self.__class__.new_from_proxy(valueproxy)
|
|
2524
|
+
else:
|
|
2525
|
+
return valueproxy
|
|
2526
|
+
pass
|
|
2527
|
+
else:
|
|
2528
|
+
if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
|
|
2529
|
+
return self.__class__.new_from_obj(self._graph, value)
|
|
2530
|
+
else:
|
|
2531
|
+
return value
|
|
2532
|
+
pass
|
|
2533
|
+
pass
|
|
2534
|
+
assert(False)
|
|
2535
|
+
pass
|
|
2536
|
+
"""
|
|
2537
|
+
def __setitem__(self,key,value):
|
|
2538
|
+
self._set_attr_or_item("_set_item",key,value)
|
|
2539
|
+
return
|
|
2540
|
+
|
|
2541
|
+
def __getitem__(self,key):
|
|
2542
|
+
return self._get_attr_or_item("_get_item",key)
|
|
2543
|
+
|
|
2544
|
+
|
|
2545
|
+
def __setattr__(self,name,value):
|
|
2546
|
+
if name.startswith('_'):
|
|
2547
|
+
raise ValueError('Cannot assign attribute with leading underscore')
|
|
2548
|
+
self._set_attr_or_item("_set_attr",name,value)
|
|
2549
|
+
return
|
|
2550
|
+
obsolete = r"""
|
|
2551
|
+
if self._proxy is not None:
|
|
2552
|
+
if isinstance(value, ONDEClassInstanceWrapper):
|
|
2553
|
+
_proxy = value._proxy
|
|
2554
|
+
_obj = value._obj
|
|
2555
|
+
if _proxy is not None:
|
|
2556
|
+
setattr(self._proxy, _proxy)
|
|
2557
|
+
pass
|
|
2558
|
+
elif _obj is not None:
|
|
2559
|
+
setattr(self._proxy, _obj)
|
|
2560
|
+
pass
|
|
2561
|
+
else:
|
|
2562
|
+
assert(False)
|
|
2563
|
+
pass
|
|
2564
|
+
pass
|
|
2565
|
+
else:
|
|
2566
|
+
setattr(self._proxy, value)
|
|
2567
|
+
pass
|
|
2568
|
+
pass
|
|
2569
|
+
elif self._obj is not None:
|
|
2570
|
+
if isinstance(value, ONDEClassInstanceWrapper):
|
|
2571
|
+
_proxy = value._proxy
|
|
2572
|
+
_obj = value._obj
|
|
2573
|
+
if _proxy is not None:
|
|
2574
|
+
setattr(self._obj, _proxy._get_obj())
|
|
2575
|
+
pass
|
|
2576
|
+
elif _obj is not None:
|
|
2577
|
+
setattr(self._obj, _obj)
|
|
2578
|
+
pass
|
|
2579
|
+
else:
|
|
2580
|
+
assert(False)
|
|
2581
|
+
pass
|
|
2582
|
+
pass
|
|
2583
|
+
else:
|
|
2584
|
+
if isinstance(value, ONDEProxy):
|
|
2585
|
+
setattr(self._obj, value._get_obj())
|
|
2586
|
+
pass
|
|
2587
|
+
else:
|
|
2588
|
+
setattr(self._obj, value)
|
|
2589
|
+
pass
|
|
2590
|
+
pass
|
|
2591
|
+
pass
|
|
2592
|
+
else:
|
|
2593
|
+
assert(False)
|
|
2594
|
+
pass
|
|
2595
|
+
pass
|
|
2596
|
+
|
|
2597
|
+
"""
|
|
2598
|
+
|
|
2599
|
+
def _repr_short(self, onde_class = None):
|
|
2600
|
+
our_proxy = object.__getattribute__(self,"_proxy")
|
|
2601
|
+
our_obj = object.__getattribute__(self,"_obj")
|
|
2602
|
+
_graph = object.__getattribute__(self,"_graph")
|
|
2603
|
+
|
|
2604
|
+
if our_proxy is not None:
|
|
2605
|
+
our_obj = our_proxy._get_obj()
|
|
2606
|
+
_graph = our_proxy._graph
|
|
2607
|
+
pass
|
|
2608
|
+
|
|
2609
|
+
return "InstanceWrapper of " + our_obj._repr_short()
|
|
2610
|
+
|
|
2611
|
+
def _repr(self, indentation, onde_class=None, extra_attrs={}):
|
|
2612
|
+
our_proxy = object.__getattribute__(self,"_proxy")
|
|
2613
|
+
our_obj = object.__getattribute__(self,"_obj")
|
|
2614
|
+
_graph = object.__getattribute__(self,"_graph")
|
|
2615
|
+
|
|
2616
|
+
if our_proxy is not None:
|
|
2617
|
+
our_obj = our_proxy._get_obj()
|
|
2618
|
+
_graph = our_proxy._graph
|
|
2619
|
+
pass
|
|
2620
|
+
|
|
2621
|
+
classdefs = _graph.class_defs
|
|
2622
|
+
if classdefs is not None:
|
|
2623
|
+
|
|
2624
|
+
if isinstance(our_obj, ONDEObject):
|
|
2625
|
+
(shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(our_obj)
|
|
2626
|
+
attrdict=collections.OrderedDict()
|
|
2627
|
+
for attrname in sorted(list(concise_set)):
|
|
2628
|
+
attrval = self._get_attr(attrname)
|
|
2629
|
+
dataset_storage = self._get_attr_dataset_storage(attrname)
|
|
2630
|
+
storage_suffix = ""
|
|
2631
|
+
if dataset_storage:
|
|
2632
|
+
storage_suffix = " (hdf5 dataset storage)"
|
|
2633
|
+
pass
|
|
2634
|
+
|
|
2635
|
+
if attrval is not None:
|
|
2636
|
+
attrdict[attrname] = attrval._repr_short() + storage_suffix
|
|
2637
|
+
pass
|
|
2638
|
+
else:
|
|
2639
|
+
attrdict[attrname] = "None"
|
|
2640
|
+
pass
|
|
2641
|
+
pass
|
|
2642
|
+
return our_obj._repr(indentation,onde_class = f"InstanceWrapper({our_obj._ONDE_attrs['ONDE:TYPE'].value[-1]:s})", extra_attrs = attrdict)
|
|
2643
|
+
pass
|
|
2644
|
+
return our_obj._repr(indentation)
|
|
2645
|
+
|
|
2646
|
+
def __repr__(self):
|
|
2647
|
+
return self._repr(0)
|
|
2648
|
+
|
|
2649
|
+
@classmethod
|
|
2650
|
+
def new_from_proxy(cls, proxy):
|
|
2651
|
+
obj = proxy._get_obj()
|
|
2652
|
+
if not isinstance(obj,ONDEObject) and not isinstance(obj,ONDEReferenceArray):
|
|
2653
|
+
raise ValueError(f"ONDEClassInstanceWrapper only wraps non-leaf nodes, not {obj.__class__.__name__}")
|
|
2654
|
+
return cls(_proxy=proxy)
|
|
2655
|
+
|
|
2656
|
+
@classmethod
|
|
2657
|
+
def new_obj(cls, graph, onde_classname, **kwargs):
|
|
2658
|
+
if graph.class_defs is None:
|
|
2659
|
+
raise ValueError("Graph does not have class definitions loaded")
|
|
2660
|
+
|
|
2661
|
+
onde_class = graph.class_defs.classes[onde_classname]
|
|
2662
|
+
obj = ONDEObject.new(None,None,ONDE_TYPE = ONDEArray.new(onde_class.class_derivation, _frozen = True), **kwargs)
|
|
2663
|
+
# proxy = ONDEProxy(_path=None)
|
|
2664
|
+
|
|
2665
|
+
return cls(_graph=graph,_obj=obj)
|
|
2666
|
+
|
|
2667
|
+
@classmethod
|
|
2668
|
+
def new_from_obj(cls,graph,obj):
|
|
2669
|
+
if not isinstance(obj,ONDEBase):
|
|
2670
|
+
raise ValueError(f"Given object is of type {obj.__class__.__name__:s} and is not an ONDEBase instance")
|
|
2671
|
+
|
|
2672
|
+
# onde_classname = object.__getattribute__(_orig, "ONDE:TYPE")
|
|
2673
|
+
if graph.class_defs is None:
|
|
2674
|
+
raise ValueError("Graph does not have class definitions loaded")
|
|
2675
|
+
return cls(_graph=graph,_obj=obj)
|
|
2676
|
+
|
|
2677
|
+
pass
|
|
2678
|
+
|
|
2679
|
+
class ONDEClassDefinitions(object):
|
|
2680
|
+
"""Represents the set of class definitions from the ONDE .csv file."""
|
|
2681
|
+
#NOTE: Immutable once constructed, except for field cache
|
|
2682
|
+
classes = None # Dictionary by name of ONDEClass
|
|
2683
|
+
acc_classes = None # Dictionary by name of accessory classes
|
|
2684
|
+
file_type = None # From the size_or_content of the blank ONDE:TYPE entry at the top of the csv file
|
|
2685
|
+
version = None # VERSION specification from the .csv file
|
|
2686
|
+
|
|
2687
|
+
|
|
2688
|
+
|
|
2689
|
+
field_cache = None #dictionary by (tuple_of_class_derivation, frozen_set_of_accessory_classes) of a dictionary by field names and field name abbreviations of ONDEField objects
|
|
2690
|
+
|
|
2691
|
+
def __init__(self):
|
|
2692
|
+
self.classes = collections.OrderedDict()
|
|
2693
|
+
self.acc_classes = collections.OrderedDict()
|
|
2694
|
+
self.field_cache = {}
|
|
2695
|
+
pass
|
|
2696
|
+
|
|
2697
|
+
def get_fields_dict(self,obj):
|
|
2698
|
+
"""returns three dictionaries by field name of ONDEField: the shorthand dictionary, the full dictionary, the combined dictionary, and the concise set. Automatically omits any shorthands that conflict with actual attributes. The concise set includes the shortest name for each field plus any fields which don't correspond to ONDEField objects"""
|
|
2699
|
+
onde_type_value = obj._ONDE_attrs["ONDE:TYPE"]
|
|
2700
|
+
assert(isinstance(onde_type_value,ONDEArray))
|
|
2701
|
+
|
|
2702
|
+
class_derivation = tuple([str(classname) for classname in onde_type_value.value])
|
|
2703
|
+
|
|
2704
|
+
if "ONDE:TYPE_TAGS" in obj._ONDE_attrs:
|
|
2705
|
+
acc_class_value = obj._ONDE_attrs["ONDE:TYPE_TAGS"]
|
|
2706
|
+
assert(isinstance(acc_class_value,ONDEArray))
|
|
2707
|
+
type_tags = frozenset([str(type_tag) for typ_tag in acc_class_value.value])
|
|
2708
|
+
pass
|
|
2709
|
+
else:
|
|
2710
|
+
type_tags = frozenset()
|
|
2711
|
+
pass
|
|
2712
|
+
|
|
2713
|
+
field_cache_key = (class_derivation, type_tags, frozenset(obj._ONDE_attrs._keys()))
|
|
2714
|
+
|
|
2715
|
+
if field_cache_key in self.field_cache:
|
|
2716
|
+
return self.field_cache[field_cache_key]
|
|
2717
|
+
|
|
2718
|
+
shorthand_fields_dict = collections.OrderedDict()
|
|
2719
|
+
full_fields_dict = collections.OrderedDict()
|
|
2720
|
+
combined_fields_dict = collections.OrderedDict()
|
|
2721
|
+
derivation_index=0
|
|
2722
|
+
mandatory_acc_classes = set()
|
|
2723
|
+
shorthand_blacklist = set()
|
|
2724
|
+
|
|
2725
|
+
def add_field(fieldname,field,extra_keys):
|
|
2726
|
+
full_fields_dict[fieldname]=field
|
|
2727
|
+
combined_fields_dict[fieldname]=field
|
|
2728
|
+
for key in extra_keys:
|
|
2729
|
+
if key in shorthand_blacklist:
|
|
2730
|
+
continue
|
|
2731
|
+
if key in obj._ONDE_attrs:
|
|
2732
|
+
# Don't allow overriding an actual attribute of this object with a shorthand
|
|
2733
|
+
continue
|
|
2734
|
+
if key in shorthand_fields_dict:
|
|
2735
|
+
# key already present
|
|
2736
|
+
if shorthand_fields_dict[key].class_prefix==full_fields_dict[fieldname].class_prefix and shorthand_fields_dict[key].name==full_fields_dict[fieldname].name:
|
|
2737
|
+
# Override of a shorthand for the same thing already present. This is allowable.
|
|
2738
|
+
shorthand_fields_dict[key]=full_fields_dict[fieldname]
|
|
2739
|
+
combined_fields_dict[key]=full_fields_dict[fieldname]
|
|
2740
|
+
pass
|
|
2741
|
+
else:
|
|
2742
|
+
# Conflicting shorthands. Remove pre-existing entry and blacklist
|
|
2743
|
+
|
|
2744
|
+
del shorthand_fields_dict[key]
|
|
2745
|
+
del combined_fields_dict[key]
|
|
2746
|
+
shorthand_blacklist.add(key)
|
|
2747
|
+
pass
|
|
2748
|
+
pass
|
|
2749
|
+
else:
|
|
2750
|
+
shorthand_fields_dict[key]=full_fields_dict[fieldname]
|
|
2751
|
+
combined_fields_dict[key]=full_fields_dict[fieldname]
|
|
2752
|
+
pass
|
|
2753
|
+
pass
|
|
2754
|
+
pass
|
|
2755
|
+
|
|
2756
|
+
for classname in ("ONDE",)+class_derivation:
|
|
2757
|
+
# from base class to most derived
|
|
2758
|
+
derivation_index+=1
|
|
2759
|
+
|
|
2760
|
+
if not classname in self.classes:
|
|
2761
|
+
continue
|
|
2762
|
+
|
|
2763
|
+
if class_derivation[:(derivation_index-1)] != self.classes[classname].class_derivation:
|
|
2764
|
+
print(f"Object class derivation mismatch: {class_derivation[:(derivation_index-1)]} from object definition versus {self.classes[classname].class_derivation} from class definition.",file=sys.stderr)
|
|
2765
|
+
break
|
|
2766
|
+
|
|
2767
|
+
|
|
2768
|
+
# Gather fields from this class
|
|
2769
|
+
for fieldname in self.classes[classname].attributes:
|
|
2770
|
+
extra_keys=set()
|
|
2771
|
+
if ':' in fieldname:
|
|
2772
|
+
#Shorthand for field name by admitting colon
|
|
2773
|
+
extra_keys.add(fieldname[(fieldname.index(':')+1):])
|
|
2774
|
+
pass
|
|
2775
|
+
|
|
2776
|
+
add_field(fieldname,self.classes[classname].attributes[fieldname],extra_keys)
|
|
2777
|
+
pass
|
|
2778
|
+
|
|
2779
|
+
# Gather mandatory accessory classes
|
|
2780
|
+
for acc_class_name in self.classes[classname].type_tags:
|
|
2781
|
+
if self.classes[classname].type_tags[acc_class_name]:
|
|
2782
|
+
# This accessory class is mandatory
|
|
2783
|
+
mandatory_acc_classes.add(acc_class_name)
|
|
2784
|
+
pass
|
|
2785
|
+
pass
|
|
2786
|
+
pass
|
|
2787
|
+
|
|
2788
|
+
acc_classes = type_tags | mandatory_acc_classes # Set union
|
|
2789
|
+
|
|
2790
|
+
for acc_class_name in acc_classes:
|
|
2791
|
+
|
|
2792
|
+
# Gather fields from accessory classes
|
|
2793
|
+
acc_class = self.acc_classes[acc_class_name]
|
|
2794
|
+
|
|
2795
|
+
# Gather fields from this accessory class
|
|
2796
|
+
for fieldname in acc_class.attributes:
|
|
2797
|
+
extra_keys=set()
|
|
2798
|
+
if ':' in fieldname:
|
|
2799
|
+
#Shorthand for field name by admitting colon
|
|
2800
|
+
extra_keys.add(fieldname[(fieldname.index(':')+1):])
|
|
2801
|
+
pass
|
|
2802
|
+
|
|
2803
|
+
add_field(fieldname,acc_class.attributes[fieldname],extra_keys)
|
|
2804
|
+
pass
|
|
2805
|
+
pass
|
|
2806
|
+
# Construct concise set
|
|
2807
|
+
reverse_combined = {}
|
|
2808
|
+
for fieldname in combined_fields_dict:
|
|
2809
|
+
if id(combined_fields_dict[fieldname]) in reverse_combined:
|
|
2810
|
+
reverse_combined[id(combined_fields_dict[fieldname])].append(fieldname)
|
|
2811
|
+
pass
|
|
2812
|
+
else:
|
|
2813
|
+
reverse_combined[id(combined_fields_dict[fieldname])] = [fieldname]
|
|
2814
|
+
pass
|
|
2815
|
+
pass
|
|
2816
|
+
|
|
2817
|
+
concise_set=set()
|
|
2818
|
+
for objid in reverse_combined:
|
|
2819
|
+
shortest = sorted(reverse_combined[objid], key=len)[0]
|
|
2820
|
+
concise_set.add(shortest)
|
|
2821
|
+
pass
|
|
2822
|
+
|
|
2823
|
+
for attrname in obj._ONDE_attrs:
|
|
2824
|
+
if attrname not in full_fields_dict:
|
|
2825
|
+
concise_set.add(attrname)
|
|
2826
|
+
pass
|
|
2827
|
+
pass
|
|
2828
|
+
|
|
2829
|
+
concise_set_frozen=frozenset(concise_set)
|
|
2830
|
+
|
|
2831
|
+
|
|
2832
|
+
# import pdb
|
|
2833
|
+
# pdb.set_trace()
|
|
2834
|
+
|
|
2835
|
+
self.field_cache[field_cache_key] = (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set_frozen)
|
|
2836
|
+
return (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set_frozen)
|
|
2837
|
+
|
|
2838
|
+
|
|
2839
|
+
@classmethod
|
|
2840
|
+
def load_from_csv(cls, filename,extra_class_defs = None):
|
|
2841
|
+
class_defs = ONDEClassDefinitions()
|
|
2842
|
+
|
|
2843
|
+
if extra_class_defs is not None:
|
|
2844
|
+
raise ValueError("Extra class definitions not yet supported")
|
|
2845
|
+
|
|
2846
|
+
with open(filename,mode="r",encoding="utf-8-sig") as csvfh:
|
|
2847
|
+
reader = csv.reader(csvfh, delimiter=";")
|
|
2848
|
+
|
|
2849
|
+
for row in reader:
|
|
2850
|
+
if len(row) < 8:
|
|
2851
|
+
row += [""]*(8-len(row))
|
|
2852
|
+
pass
|
|
2853
|
+
|
|
2854
|
+
stripped_row = [col.strip() for col in row]
|
|
2855
|
+
(classname, name, comments, mandatory_optional, dataset_attribute, content_class, dims, size_or_content, min_val, max_val, accessory_class) = stripped_row
|
|
2856
|
+
|
|
2857
|
+
if classname == "Class":
|
|
2858
|
+
# header csv line
|
|
2859
|
+
continue
|
|
2860
|
+
|
|
2861
|
+
if name == "ONDE:VERSION":
|
|
2862
|
+
class_defs.version=size_or_content
|
|
2863
|
+
continue
|
|
2864
|
+
|
|
2865
|
+
if name == "ONDE:FILETYPE":
|
|
2866
|
+
# ignore filetype definition in .csv
|
|
2867
|
+
continue
|
|
2868
|
+
|
|
2869
|
+
if accessory_class == "True":
|
|
2870
|
+
accessory_class = True
|
|
2871
|
+
pass
|
|
2872
|
+
elif accessory_class == "False" or accessory_class == "":
|
|
2873
|
+
accessory_class = False
|
|
2874
|
+
pass
|
|
2875
|
+
else:
|
|
2876
|
+
raise ValueError(f"unknown accessory_class value in .csv: {accessory_class:s}")
|
|
2877
|
+
|
|
2878
|
+
if name == "ONDE:TYPE" and not accessory_class:
|
|
2879
|
+
# class definition
|
|
2880
|
+
if classname in class_defs.classes:
|
|
2881
|
+
raise ValueError(f"Class {classname:s} multiply defined in {filename:s}")
|
|
2882
|
+
newclass = ONDEClass()
|
|
2883
|
+
newclass.classname = classname
|
|
2884
|
+
newclass.comments = comments
|
|
2885
|
+
# size_or_content should be a list of base classes terminated with this class, written roughly like a python list of strings
|
|
2886
|
+
baseclasses = ast.literal_eval(size_or_content)
|
|
2887
|
+
assert(type(baseclasses) is list)
|
|
2888
|
+
if classname == "":
|
|
2889
|
+
# Blank ONDE:TYPE entry specifying the file type
|
|
2890
|
+
assert(len(baseclasses) == 1)
|
|
2891
|
+
assert(type(baseclasses[0]) is str)
|
|
2892
|
+
class_defs.file_type = baseclasses[0]
|
|
2893
|
+
superclasses = []
|
|
2894
|
+
superclass_def = None
|
|
2895
|
+
pass
|
|
2896
|
+
else:
|
|
2897
|
+
assert(baseclasses[-1] == classname)
|
|
2898
|
+
|
|
2899
|
+
# check superclasses
|
|
2900
|
+
superclasses = baseclasses[:-1]
|
|
2901
|
+
|
|
2902
|
+
if len(superclasses) > 0:
|
|
2903
|
+
immediate_superclass = superclasses[-1]
|
|
2904
|
+
if immediate_superclass not in class_defs.classes:
|
|
2905
|
+
raise ValueError(f"Superclass {immediate_superclass:s} of {classname:s} in {filename:s} is unknown")
|
|
2906
|
+
superclass_def = class_defs.classes[immediate_superclass]
|
|
2907
|
+
# Superclass definition list of class names should match our superclasses
|
|
2908
|
+
if tuple(superclasses) != superclass_def.class_derivation:
|
|
2909
|
+
raise ValueError(f"Superclass {immediate_superclass:s} derviation {str(superclass_def.class_derivation):s} does not match superclass list {str(tuple(superclasses)):s} from class {classname:s} in csv file {filename:s}")
|
|
2910
|
+
pass
|
|
2911
|
+
else:
|
|
2912
|
+
superclass_def = None
|
|
2913
|
+
pass
|
|
2914
|
+
pass
|
|
2915
|
+
|
|
2916
|
+
newclass.class_derivation = tuple(superclasses) + (classname,)
|
|
2917
|
+
newclass.superclass = superclass_def # ONDEClassObject
|
|
2918
|
+
class_defs.classes[classname] = newclass
|
|
2919
|
+
pass
|
|
2920
|
+
|
|
2921
|
+
elif name == "ONDE:TYPE" and accessory_class:
|
|
2922
|
+
# accessory class definition
|
|
2923
|
+
|
|
2924
|
+
if classname in class_defs.acc_classes:
|
|
2925
|
+
raise ValueError(f"Accessory class {classname:s} multiply defined in {filename:s}")
|
|
2926
|
+
|
|
2927
|
+
newclass = ONDEAccessoryClass()
|
|
2928
|
+
newclass.classname = classname
|
|
2929
|
+
newclass.comments = comments
|
|
2930
|
+
# size_or_content should be a list of base classes terminated with this class, written roughly like a python list of strings
|
|
2931
|
+
acc_classes = ast.literal_eval(size_or_content)
|
|
2932
|
+
|
|
2933
|
+
assert(type(acc_classes) is list)
|
|
2934
|
+
assert(len(acc_classes) == 1)
|
|
2935
|
+
assert(acc_classes[0] == classname)
|
|
2936
|
+
|
|
2937
|
+
class_defs.acc_classes[classname] = newclass
|
|
2938
|
+
|
|
2939
|
+
pass
|
|
2940
|
+
|
|
2941
|
+
elif name == "ONDE:TYPE_TAGS":
|
|
2942
|
+
# specification for a class having a particular accessory class
|
|
2943
|
+
|
|
2944
|
+
type_tags = ast.literal_eval(size_or_content)
|
|
2945
|
+
|
|
2946
|
+
for type_tag in type_tags:
|
|
2947
|
+
class_defs.classes[classname].type_tags[type_tag] = mandatory_optional == "M"
|
|
2948
|
+
|
|
2949
|
+
pass
|
|
2950
|
+
|
|
2951
|
+
pass
|
|
2952
|
+
|
|
2953
|
+
else:
|
|
2954
|
+
# populating class with a field
|
|
2955
|
+
|
|
2956
|
+
class_derivation = None
|
|
2957
|
+
class_or_acc = None
|
|
2958
|
+
|
|
2959
|
+
if classname in class_defs.classes:
|
|
2960
|
+
class_or_acc = class_defs.classes[classname]
|
|
2961
|
+
class_derivation = class_or_acc.class_derivation
|
|
2962
|
+
pass
|
|
2963
|
+
elif classname in class_defs.acc_classes:
|
|
2964
|
+
class_or_acc = class_defs.acc_classes[classname]
|
|
2965
|
+
pass
|
|
2966
|
+
else:
|
|
2967
|
+
raise ValueError(f"Class {classname:s} not found; not a known Class or AccessoryClass while defining attribute {name:s}")
|
|
2968
|
+
|
|
2969
|
+
if name in class_or_acc.attributes:
|
|
2970
|
+
raise ValueError(f"Attribute {name} already in Class or AccessoryClass")
|
|
2971
|
+
|
|
2972
|
+
|
|
2973
|
+
class_or_acc.attributes[name] = ONDEField.from_csv_line(stripped_row, class_derivation)
|
|
2974
|
+
|
|
2975
|
+
pass
|
|
2976
|
+
pass
|
|
2977
|
+
pass
|
|
2978
|
+
|
|
2979
|
+
return class_defs
|
|
2980
|
+
pass
|
|
2981
|
+
|
|
2982
|
+
|
|
2983
|
+
@dataclass
|
|
2984
|
+
class ScopeNode(object):
|
|
2985
|
+
referrers: Optional[set[ONDEBase]]=field(default_factory=set) # Set of referring objects.
|
|
2986
|
+
edges: Optional[set[Any]]=field(default_factory=set) # Set of edge indexes. If it is None, then all edges are in scope. Usually either each edge index is either a string for an ONDEObject or a tuple of integers for an ONDEReferenceArray.
|
|
2987
|
+
pass
|
|
2988
|
+
|
|
2989
|
+
def graph_replace_node__walk(starting_path, starting_obj,referring_obj, scope, scope_nodes, trans):
|
|
2990
|
+
"""Trans is a transaction that has a mutable graph snapshot.
|
|
2991
|
+
|
|
2992
|
+
scope_nodes is a mutable dictionary, which may be already partially prepopulated, indexed by node, of ScopeNode instances.
|
|
2993
|
+
|
|
2994
|
+
scope is an ONDEOpScope.
|
|
2995
|
+
starting_obj is the object corresponding to starting_path. It is not assumed to already be inserted into scope_nodes. It is assumed that starting_obj has already been checked against the exclude_objs of scope.
|
|
2996
|
+
starting_path is an ONDEPath indicating the full path of starting_obj. It is assumed that starting_path has been already checked against the include_paths of scope.
|
|
2997
|
+
|
|
2998
|
+
Walk the graph starting at the given starting_path, ignoring explicitly excluded paths and nodes from the scope, while accumulating nodes into scope_nodes, keeping track of which edges are fair game (represented by a set corresponding to the node in scope_nodes), or where all edges are fair game (represented by None instead of the set)
|
|
2999
|
+
"""
|
|
3000
|
+
|
|
3001
|
+
snap = trans.snap
|
|
3002
|
+
|
|
3003
|
+
# Register starting_obj into scope_nodes
|
|
3004
|
+
if starting_obj in scope_nodes:
|
|
3005
|
+
new=False
|
|
3006
|
+
scope_node = scope_nodes[starting_obj]
|
|
3007
|
+
pass
|
|
3008
|
+
else:
|
|
3009
|
+
new=True
|
|
3010
|
+
scope_node = ScopeNode()
|
|
3011
|
+
scope_nodes[starting_obj] = scope_node
|
|
3012
|
+
pass
|
|
3013
|
+
|
|
3014
|
+
#if starting_scope_set is not None:
|
|
3015
|
+
# If we are traversing this object, then all edges are in scope, so we replace any set with None
|
|
3016
|
+
|
|
3017
|
+
scope_node.edges = None
|
|
3018
|
+
scope_node.referrers.add(referring_obj)
|
|
3019
|
+
|
|
3020
|
+
if new:
|
|
3021
|
+
edges = starting_obj._list_edges()
|
|
3022
|
+
|
|
3023
|
+
for edge in edges:
|
|
3024
|
+
current_path = ONDEPath(starting_path + (edge,))
|
|
3025
|
+
current_obj = starting_obj._follow_path(ONDEPath((edge,)))
|
|
3026
|
+
|
|
3027
|
+
if current_path in scope.exclude_paths:
|
|
3028
|
+
continue
|
|
3029
|
+
|
|
3030
|
+
if current_obj in scope.exclude_objects:
|
|
3031
|
+
continue
|
|
3032
|
+
|
|
3033
|
+
graph_replace_node__walk(current_path, current_obj, starting_obj,scope, scope_nodes, trans)
|
|
3034
|
+
|
|
3035
|
+
pass
|
|
3036
|
+
pass
|
|
3037
|
+
pass
|
|
3038
|
+
|
|
3039
|
+
|
|
3040
|
+
@dataclass
|
|
3041
|
+
class ReplacedNode(object):
|
|
3042
|
+
replacement: Optional[ONDEBase]=None
|
|
3043
|
+
refersto: Optional[set[ONDEBase]]=field(default_factory=set) # Set of objects this node refers to that we might want to replace.
|
|
3044
|
+
|
|
3045
|
+
pass
|
|
3046
|
+
|
|
3047
|
+
|
|
3048
|
+
def graph_replace_node__reversewalk(current_node,refersto,scope_nodes,changed_nodes):
|
|
3049
|
+
if current_node not in changed_nodes:
|
|
3050
|
+
new = True
|
|
3051
|
+
changed_nodes[current_node] = ReplacedNode()
|
|
3052
|
+
pass
|
|
3053
|
+
else:
|
|
3054
|
+
new = False
|
|
3055
|
+
pass
|
|
3056
|
+
|
|
3057
|
+
changed_nodes[current_node].refersto.update(refersto)
|
|
3058
|
+
|
|
3059
|
+
if new:
|
|
3060
|
+
referrers = scope_nodes[current_node].referrers
|
|
3061
|
+
|
|
3062
|
+
for referrer in referrers:
|
|
3063
|
+
if referrer in changed_nodes:
|
|
3064
|
+
continue
|
|
3065
|
+
|
|
3066
|
+
graph_replace_node__reversewalk(referrer,set([current_node]),scope_nodes,changed_nodes)
|
|
3067
|
+
pass
|
|
3068
|
+
pass
|
|
3069
|
+
pass
|
|
3070
|
+
|
|
3071
|
+
def graph_replace_node(trans, scope, path, orig_node, replacement_node):
|
|
3072
|
+
# to do: proposed algorithm:
|
|
3073
|
+
#
|
|
3074
|
+
# step 1: follow each starting location path to its end; then, continue to walk the graph, ignoring explicitly excluded (edges or paths?) while accumulating all nodes into a dict (indexed by pre-existing nodes) of scope_nodes and keeping track of which edges are "fair game" for each node (i.e. all edges, if we were walking the graph, or edges called out explicitly in a starting location path)
|
|
3075
|
+
snap = trans.snap
|
|
3076
|
+
scope_nodes = collections.OrderedDict()
|
|
3077
|
+
scope_nodes[snap] = ScopeNode() # Give the snapshot itself a blank ScopeNode
|
|
3078
|
+
|
|
3079
|
+
for starting_path in scope.include_paths:
|
|
3080
|
+
if starting_path in scope.exclude_paths:
|
|
3081
|
+
continue
|
|
3082
|
+
|
|
3083
|
+
|
|
3084
|
+
current_parent = snap
|
|
3085
|
+
current_path = ONDEPath(())
|
|
3086
|
+
pending_path = starting_path
|
|
3087
|
+
current_scope_set = scope_nodes[snap].edges
|
|
3088
|
+
|
|
3089
|
+
# Follow the starting path, element by element, because we need to accumulate all referrers into the scope dictionary.
|
|
3090
|
+
while len(current_path) < len(starting_path)-1:
|
|
3091
|
+
current_scope_set.add(pending_path[0])
|
|
3092
|
+
current_path = ONDEPath(current_path + (pending_path[0],))
|
|
3093
|
+
|
|
3094
|
+
current_parent_parent=current_parent
|
|
3095
|
+
current_parent = current_parent._follow_path(ONDEPath((pending_path[0],)))
|
|
3096
|
+
#print("current_parent=",str(current_path))
|
|
3097
|
+
pending_path = ONDEPath(pending_path[1:])
|
|
3098
|
+
if current_parent in scope_nodes:
|
|
3099
|
+
current_scope_node = scope_nodes[current_parent]
|
|
3100
|
+
current_scope_set = current_scope_node.edges
|
|
3101
|
+
pass
|
|
3102
|
+
else:
|
|
3103
|
+
current_scope_node = ScopeNode()
|
|
3104
|
+
current_scope_set = current_scope_node.edges
|
|
3105
|
+
scope_nodes[current_parent] = current_scope_node
|
|
3106
|
+
pass
|
|
3107
|
+
current_scope_node.referrers.add(current_parent_parent)
|
|
3108
|
+
pass
|
|
3109
|
+
|
|
3110
|
+
if current_scope_set is not None:
|
|
3111
|
+
current_scope_set.add(pending_path[0])
|
|
3112
|
+
pass
|
|
3113
|
+
|
|
3114
|
+
|
|
3115
|
+
|
|
3116
|
+
assert(starting_path[-1] == pending_path[0])
|
|
3117
|
+
starting_parent = current_parent
|
|
3118
|
+
starting_obj = starting_parent._follow_path(ONDEPath((starting_path[-1],)))
|
|
3119
|
+
if starting_obj in scope.exclude_objects:
|
|
3120
|
+
continue
|
|
3121
|
+
|
|
3122
|
+
graph_replace_node__walk(starting_path, starting_obj, starting_parent, scope, scope_nodes, trans)
|
|
3123
|
+
pass
|
|
3124
|
+
|
|
3125
|
+
# print("scope_nodes=",scope_nodes)
|
|
3126
|
+
|
|
3127
|
+
|
|
3128
|
+
#
|
|
3129
|
+
# step 2: identify the node to be changed within the set (if it's not included, the new node is not referenced)
|
|
3130
|
+
#
|
|
3131
|
+
if orig_node not in scope_nodes:
|
|
3132
|
+
return
|
|
3133
|
+
# step 3: reverse walk the set of nodes, starting at the node to be changed, identifying these nodes into a new (and probably smaller) dict called changed_nodes, the keys of which are a set of nodes through which the change will propagate while the values are ReplacedNode objects with the replacement set to None
|
|
3134
|
+
#
|
|
3135
|
+
changed_nodes = collections.OrderedDict()
|
|
3136
|
+
|
|
3137
|
+
current_node = orig_node
|
|
3138
|
+
|
|
3139
|
+
graph_replace_node__reversewalk(current_node,set([]),scope_nodes,changed_nodes)
|
|
3140
|
+
|
|
3141
|
+
|
|
3142
|
+
#print("changed_nodes=",changed_nodes)
|
|
3143
|
+
|
|
3144
|
+
# step 4: iterate through the second set creating a replacement for each where we update changed_nodes, populating each entry's value with the replacement stored in a ReplacedNode object
|
|
3145
|
+
#
|
|
3146
|
+
changed_node_indexes = list(changed_nodes.keys())
|
|
3147
|
+
|
|
3148
|
+
for changed_node in changed_node_indexes:
|
|
3149
|
+
if changed_node is not orig_node:
|
|
3150
|
+
# check to see if the changed node is frozen; if has modification scopes been cleared?
|
|
3151
|
+
if changed_node._modification_scopes is None:
|
|
3152
|
+
replacement = changed_node.__class__(_orig = changed_node)
|
|
3153
|
+
pass
|
|
3154
|
+
else:
|
|
3155
|
+
replacement = changed_node
|
|
3156
|
+
pass
|
|
3157
|
+
|
|
3158
|
+
replacement._modification_scopes.add(scope)
|
|
3159
|
+
|
|
3160
|
+
pass
|
|
3161
|
+
else:
|
|
3162
|
+
replacement = replacement_node
|
|
3163
|
+
pass
|
|
3164
|
+
changed_nodes[changed_node].replacement = replacement
|
|
3165
|
+
pass
|
|
3166
|
+
|
|
3167
|
+
|
|
3168
|
+
|
|
3169
|
+
#print("changed_nodes=",changed_nodes)
|
|
3170
|
+
|
|
3171
|
+
|
|
3172
|
+
|
|
3173
|
+
# step 5: iterate through the replacements, identifying every "fair game" reference to changed_nodes, and re-pointing that to the replacements
|
|
3174
|
+
#
|
|
3175
|
+
|
|
3176
|
+
for replaced_node in changed_nodes:
|
|
3177
|
+
replacement = changed_nodes[replaced_node].replacement
|
|
3178
|
+
scope_node = scope_nodes[replaced_node]
|
|
3179
|
+
fair_game_edges = scope_node.edges
|
|
3180
|
+
refersto = changed_nodes[replaced_node].refersto
|
|
3181
|
+
# refersto is a set of ONDEBase that includes all of the outgoing-referenced-objects referred to by replacement that may need to be repointed at a newly created copy that is findable by changed_nodes
|
|
3182
|
+
# Only those edges listed in fair_game_edges (or all edges if fair_game_edges is None) need to be swapped out.
|
|
3183
|
+
for dest in refersto:
|
|
3184
|
+
dest_indices = replaced_node._indices_for_object(dest)
|
|
3185
|
+
if fair_game_edges is not None:
|
|
3186
|
+
dest_indices = dest_indices.intersection(fair_game_edges)
|
|
3187
|
+
pass
|
|
3188
|
+
for dest_index in dest_indices:
|
|
3189
|
+
replacement._assign_pathel(dest_index, changed_nodes[dest].replacement)
|
|
3190
|
+
pass
|
|
3191
|
+
pass
|
|
3192
|
+
pass
|
|
3193
|
+
|
|
3194
|
+
|
|
3195
|
+
# step 6: the result is potential replacement for the entry point for each scope starting location
|
|
3196
|
+
|
|
3197
|
+
# implementation plan:
|
|
3198
|
+
#
|
|
3199
|
+
# we need a set of node, with each node having a set of "fair game" edges, stored globally for this operation as a dict, indexed by nodes with the values being either None (all possible edges) or a set of edges identifiers
|
|
3200
|
+
#
|
|
3201
|
+
# we also need a set of references of all scope nodes that point at any given scope node of intereset
|
|
3202
|
+
#
|
|
3203
|
+
# for the task above, we'll need a ScopeNode class that has a set of "fair game" edges (or None indicating that all edges are fair game) and it will need a set of referring nodes; as we assemble the scope_nodes dictionary, any time we find a node that we have seen before, we add to the referring node set of the ScopeNode, rather than creating a new ScopeNode
|
|
3204
|
+
|
|
3205
|
+
pass
|
|
3206
|
+
|
|
3207
|
+
class ONDEProxy(object):
|
|
3208
|
+
"""Mutable proxy reference to a graph entry that remembers context."""
|
|
3209
|
+
_graph = None # ONDEFileGraph object we started with
|
|
3210
|
+
_path = None # ONDEPath of the object we are proxying
|
|
3211
|
+
# _obj = None # The actual object we are proxying
|
|
3212
|
+
# _obj_snap = None # Snapshot from which we obtained _obj
|
|
3213
|
+
_trans = None # Transaction may be none not inside a transaction
|
|
3214
|
+
_scope = None # A scope separate from the one in _trans (if None, use the scope from _trans)
|
|
3215
|
+
_snap = None # Snapshot; only used if there is no transaction
|
|
3216
|
+
|
|
3217
|
+
def __init__(self, **kwargs):
|
|
3218
|
+
# _dict = object.__getattribute__(self, "__dict__")
|
|
3219
|
+
|
|
3220
|
+
for arg in kwargs:
|
|
3221
|
+
if arg in type(self).__dict__:
|
|
3222
|
+
# setattr(self, arg, kwargs[arg])
|
|
3223
|
+
object.__setattr__(self, arg, kwargs[arg])
|
|
3224
|
+
|
|
3225
|
+
pass
|
|
3226
|
+
else:
|
|
3227
|
+
raise ValueError(f"ONDEProxy; unknown attribute {arg:s}")
|
|
3228
|
+
pass
|
|
3229
|
+
pass
|
|
3230
|
+
|
|
3231
|
+
@classmethod
|
|
3232
|
+
def new_from_proxy(cls, parent, subpath):
|
|
3233
|
+
_graph = object.__getattribute__(parent, "_graph")
|
|
3234
|
+
|
|
3235
|
+
# parent_obj = object.__getattribute__(parent, "_obj")
|
|
3236
|
+
# _obj = parent_obj._get_attr(attr_name)
|
|
3237
|
+
# _obj_snap = object.__getattribute__(parent, "_obj_snap")
|
|
3238
|
+
|
|
3239
|
+
_parent_path = object.__getattribute__(parent, "_path")
|
|
3240
|
+
path = ONDEPath(_parent_path + (subpath,))
|
|
3241
|
+
|
|
3242
|
+
_trans = object.__getattribute__(parent, "_trans")
|
|
3243
|
+
|
|
3244
|
+
return cls(_graph=_graph, _path=path, _trans=_trans)
|
|
3245
|
+
|
|
3246
|
+
@classmethod
|
|
3247
|
+
def new_from_snapshot(cls, graph, snapshot):
|
|
3248
|
+
path = ONDEPath()
|
|
3249
|
+
return cls(_graph=graph, _path=path, _snap=snapshot)
|
|
3250
|
+
|
|
3251
|
+
@classmethod
|
|
3252
|
+
def new_from_scope(cls, scope):
|
|
3253
|
+
_trans = scope.trans
|
|
3254
|
+
_graph = _trans.graph
|
|
3255
|
+
_path = ONDEPath()
|
|
3256
|
+
|
|
3257
|
+
return cls(_graph=_graph, _path=_path, _trans=_trans,_scope=scope)
|
|
3258
|
+
|
|
3259
|
+
def _get_obj(self):
|
|
3260
|
+
trans = object.__getattribute__(self, "_trans")
|
|
3261
|
+
|
|
3262
|
+
if trans is not None:
|
|
3263
|
+
snap = trans.snap
|
|
3264
|
+
pass
|
|
3265
|
+
|
|
3266
|
+
else:
|
|
3267
|
+
graph = object.__getattribute__(self, "_graph")
|
|
3268
|
+
snap = graph.latest_snap
|
|
3269
|
+
pass
|
|
3270
|
+
|
|
3271
|
+
_path = object.__getattribute__(self, "_path")
|
|
3272
|
+
return snap._follow_path(_path)
|
|
3273
|
+
|
|
3274
|
+
def __getattribute__(self, name):
|
|
3275
|
+
if name.startswith("_"):
|
|
3276
|
+
if name in { "_set_attr","_set_dataset_attr", "_get_attr","_get_attr_dataset_storage","_get_item","_set_item","_get_data","_set_data","_set_attr_or_item_or_data","_get_obj","_follow_path","__class__","__dict__","_graph","_repr","_repr_short","_add_dataset"}:
|
|
3277
|
+
return object.__getattribute__(self, name)
|
|
3278
|
+
elif name in {"_freeze", "_frozen",}:
|
|
3279
|
+
obj = self._get_obj()
|
|
3280
|
+
return getattr(obj,name)
|
|
3281
|
+
raise ValueError(f"ONDEProxy: invalid underscore attribute {name}")
|
|
3282
|
+
obj = self._get_obj()
|
|
3283
|
+
if isinstance(obj,ONDEFileGraphSnapshot):
|
|
3284
|
+
# only attribute of a snapshot is .add(),
|
|
3285
|
+
# which we redirect to ourselves as ._add()
|
|
3286
|
+
if name == "add":
|
|
3287
|
+
return object.__getattribute__(self, "_add")
|
|
3288
|
+
elif name=="keys":
|
|
3289
|
+
return object.__getattribute__(obj, "keys")
|
|
3290
|
+
raise IndexError(f"No attribute {name:s} of a proxy for ONDEFileGraphSnapshot")
|
|
3291
|
+
|
|
3292
|
+
if isinstance(obj,ONDEObject):
|
|
3293
|
+
if isinstance(obj,ONDEFileGraphSnapshot) and name == "add":
|
|
3294
|
+
name = "_add"
|
|
3295
|
+
pass
|
|
3296
|
+
|
|
3297
|
+
_get_attr = object.__getattribute__(self, "_get_attr")
|
|
3298
|
+
return _get_attr(name)
|
|
3299
|
+
|
|
3300
|
+
#elif isinstance(obj,ONDEReferenceArray):
|
|
3301
|
+
# self._set_item(name,value)
|
|
3302
|
+
# pass
|
|
3303
|
+
else:
|
|
3304
|
+
assert(isinstance(obj,ONDEBase))
|
|
3305
|
+
_get_data = object.__getattribute__(self, "_get_data")
|
|
3306
|
+
return _get_data(name)
|
|
3307
|
+
|
|
3308
|
+
pass
|
|
3309
|
+
|
|
3310
|
+
def __setattr__(self,name,value):
|
|
3311
|
+
_set_attr = object.__getattribute__(self, "_set_attr")
|
|
3312
|
+
obj = self._get_obj()
|
|
3313
|
+
if isinstance(obj,ONDEObject):
|
|
3314
|
+
self._set_attr(name,value)
|
|
3315
|
+
pass
|
|
3316
|
+
#elif isinstance(obj,ONDEReferenceArray):
|
|
3317
|
+
# self._set_item(name,value)
|
|
3318
|
+
# pass
|
|
3319
|
+
else:
|
|
3320
|
+
assert(isinstance(obj,ONDEBase))
|
|
3321
|
+
self._set_data(name,value)
|
|
3322
|
+
pass
|
|
3323
|
+
pass
|
|
3324
|
+
|
|
3325
|
+
def _get_attr(self, name):
|
|
3326
|
+
obj = self._get_obj()
|
|
3327
|
+
attr_obj = obj._get_attr(name)
|
|
3328
|
+
|
|
3329
|
+
assert(isinstance(attr_obj, ONDEBase))
|
|
3330
|
+
return self.__class__.new_from_proxy(self, name)
|
|
3331
|
+
|
|
3332
|
+
#return attr_obj
|
|
3333
|
+
|
|
3334
|
+
def _get_attr_dataset_storage(self, name):
|
|
3335
|
+
obj = self._get_obj()
|
|
3336
|
+
return obj._get_attr_dataset_storage(name)
|
|
3337
|
+
|
|
3338
|
+
|
|
3339
|
+
def _has_attr(self, name):
|
|
3340
|
+
obj = self._get_obj()
|
|
3341
|
+
return obj._has_attr(name)
|
|
3342
|
+
|
|
3343
|
+
|
|
3344
|
+
def _set_attr(self, name, value):
|
|
3345
|
+
self._set_attr_or_item_or_data('_set_attr', name, value)
|
|
3346
|
+
pass
|
|
3347
|
+
|
|
3348
|
+
def _add(self,dataset):
|
|
3349
|
+
"""add() method for proxying ONDEFileGraphSnapshot"""
|
|
3350
|
+
# Must maintain parallel logic in ONDEFileGraphSnapshot.add()
|
|
3351
|
+
obj=dataset
|
|
3352
|
+
if not isinstance(obj,ONDEBase):
|
|
3353
|
+
obj = dataset._get_obj()
|
|
3354
|
+
pass
|
|
3355
|
+
if "ONDE:TYPE" not in obj or obj["ONDE:TYPE"].value[0] != "ONDE_DATASET":
|
|
3356
|
+
raise ValueError(f"Attempting to add something other than a dataset to an ONDEFileGraphSnapshot")
|
|
3357
|
+
|
|
3358
|
+
if "ONDE:UUID" not in dataset:
|
|
3359
|
+
raise ValueError(f"Attempting to add a dataset without a UUID to an ONDEFileGraphSnapshot")
|
|
3360
|
+
self._set_attr_or_item_or_data("_add_dataset",dataset["ONDE:UUID"].value,dataset)
|
|
3361
|
+
pass
|
|
3362
|
+
|
|
3363
|
+
def _set_dataset_attr(self, name, value):
|
|
3364
|
+
self._set_attr_or_item_or_data('_set_dataset_attr', name, value)
|
|
3365
|
+
pass
|
|
3366
|
+
|
|
3367
|
+
|
|
3368
|
+
def _get_item(self, key):
|
|
3369
|
+
obj = self._get_obj()
|
|
3370
|
+
item_obj = obj._get_item(key)
|
|
3371
|
+
|
|
3372
|
+
if isinstance(item_obj, ONDEBase):
|
|
3373
|
+
|
|
3374
|
+
# if obj is an ONDEFileGraphSnapshot, then we have flexible indexing with shorthands but for the proxy we need the real key
|
|
3375
|
+
if isinstance(obj, ONDEFileGraphSnapshot):
|
|
3376
|
+
item_obj_keys = [ k for k in obj.keys() if obj._get_item(key) is item_obj ]
|
|
3377
|
+
assert (len(item_obj_keys)==1)
|
|
3378
|
+
key = item_obj_keys[0]
|
|
3379
|
+
pass
|
|
3380
|
+
|
|
3381
|
+
return self.__class__.new_from_proxy(self, key)
|
|
3382
|
+
|
|
3383
|
+
return item_obj
|
|
3384
|
+
|
|
3385
|
+
def _set_item(self, index, value):
|
|
3386
|
+
self._set_attr_or_item_or_data('_set_item', index, value)
|
|
3387
|
+
pass
|
|
3388
|
+
|
|
3389
|
+
def __getitem__(self,index):
|
|
3390
|
+
return self._get_item(index)
|
|
3391
|
+
|
|
3392
|
+
def __setitem__(self,index,value):
|
|
3393
|
+
return self._set_item(index,value)
|
|
3394
|
+
|
|
3395
|
+
def _get_data(self, name):
|
|
3396
|
+
obj = self._get_obj()
|
|
3397
|
+
data_obj = obj._get_data(name)
|
|
3398
|
+
|
|
3399
|
+
return data_obj
|
|
3400
|
+
|
|
3401
|
+
def _set_data(self, name, value):
|
|
3402
|
+
self._set_attr_or_item_or_data('_set_data', name, value)
|
|
3403
|
+
pass
|
|
3404
|
+
|
|
3405
|
+
def _add_dataset(self,key,dataset):
|
|
3406
|
+
assert(key==dataset["ONDE:UUID"].value)
|
|
3407
|
+
self._set_attr_or_item_or_data("_add_dataset",key,dataset)
|
|
3408
|
+
pass
|
|
3409
|
+
|
|
3410
|
+
def _set_attr_or_item_or_data(self, set_method_name, key, value):
|
|
3411
|
+
''' Use the named set method (_set_attr,_set_dataset_attr, _set_item, or _set_data) to assign the element specified by key to the given value'''
|
|
3412
|
+
|
|
3413
|
+
_trans = object.__getattribute__(self, "_trans")
|
|
3414
|
+
#import pdb
|
|
3415
|
+
#pdb.set_trace()
|
|
3416
|
+
|
|
3417
|
+
if _trans is None:
|
|
3418
|
+
_graph = object.__getattribute__(self, "_graph")
|
|
3419
|
+
|
|
3420
|
+
transaction = ONDETransaction(_graph)
|
|
3421
|
+
|
|
3422
|
+
with transaction as scope:
|
|
3423
|
+
proxy=scope.graph
|
|
3424
|
+
# proxy now has a potentially updated snapshot that we
|
|
3425
|
+
|
|
3426
|
+
# should use for the transaction. Need to use our path to
|
|
3427
|
+
# recreate our object and then call _set_attr on that.
|
|
3428
|
+
path = object.__getattribute__(self,"_path")
|
|
3429
|
+
newproxy = proxy._follow_path(path)
|
|
3430
|
+
set_method = getattr(newproxy, set_method_name) # extract the bound method e.g. newproxy._set_attr
|
|
3431
|
+
set_method(key, value)
|
|
3432
|
+
pass
|
|
3433
|
+
|
|
3434
|
+
pass
|
|
3435
|
+
|
|
3436
|
+
else:
|
|
3437
|
+
# create replacement node that has the desired change
|
|
3438
|
+
# obj = object.__getattribute__(self, "_obj")
|
|
3439
|
+
|
|
3440
|
+
# create a new object of obj's class, passing the original that we want to copy as its first constructor parameter; the ONDE classes are built to handle this
|
|
3441
|
+
_get_obj = object.__getattribute__(self, "_get_obj")
|
|
3442
|
+
obj = _get_obj()
|
|
3443
|
+
scope = object.__getattribute__(self, "_scope")
|
|
3444
|
+
path = object.__getattribute__(self, "_path")
|
|
3445
|
+
|
|
3446
|
+
if scope is None:
|
|
3447
|
+
scope = _trans.scope
|
|
3448
|
+
pass
|
|
3449
|
+
|
|
3450
|
+
if scope.include_paths is None:
|
|
3451
|
+
# Null scope: use most constrained scope for each op
|
|
3452
|
+
scope = ONDEOpScope(_trans,[path])
|
|
3453
|
+
pass
|
|
3454
|
+
|
|
3455
|
+
# if obj._frozen:
|
|
3456
|
+
if obj._modification_scopes is None:
|
|
3457
|
+
replacement = obj.__class__(_orig=obj)
|
|
3458
|
+
# object.__setattr__(replacement, "_modification_scopes", set([scope]))
|
|
3459
|
+
replacement._modification_scopes.add(scope)
|
|
3460
|
+
new = True
|
|
3461
|
+
pass
|
|
3462
|
+
else:
|
|
3463
|
+
replacement = obj
|
|
3464
|
+
new = False
|
|
3465
|
+
pass
|
|
3466
|
+
|
|
3467
|
+
set_method = getattr(replacement, set_method_name) # e.g. replacement._set_attr
|
|
3468
|
+
set_method(key, value)
|
|
3469
|
+
#replacement._freeze() Freezing happens at the end of the transaction
|
|
3470
|
+
#if new: # We could avoid the replacement for preexisting modifications if we knew that the scope of the previous modification matched our scope.
|
|
3471
|
+
if new or scope not in obj._modification_scopes:
|
|
3472
|
+
replacement._modification_scopes.add(scope)
|
|
3473
|
+
if len(path) > 0:
|
|
3474
|
+
graph_replace_node(_trans, scope, path, obj, replacement)
|
|
3475
|
+
pass
|
|
3476
|
+
else:
|
|
3477
|
+
assert(obj is _trans.snap and replacement is _trans.snap)
|
|
3478
|
+
pass
|
|
3479
|
+
|
|
3480
|
+
pass
|
|
3481
|
+
|
|
3482
|
+
pass
|
|
3483
|
+
|
|
3484
|
+
pass
|
|
3485
|
+
|
|
3486
|
+
def _follow_path(self,path):
|
|
3487
|
+
obj = self._get_obj()
|
|
3488
|
+
dest=obj._follow_path(path)
|
|
3489
|
+
|
|
3490
|
+
|
|
3491
|
+
_graph = object.__getattribute__(self, "_graph")
|
|
3492
|
+
|
|
3493
|
+
_our_path = object.__getattribute__(self, "_path")
|
|
3494
|
+
full_path = ONDEPath(_our_path + path)
|
|
3495
|
+
|
|
3496
|
+
_trans = object.__getattribute__(self, "_trans")
|
|
3497
|
+
|
|
3498
|
+
return self.__class__(_graph=_graph, _path=full_path, _trans=_trans)
|
|
3499
|
+
|
|
3500
|
+
def _repr_short(self, onde_class=None):
|
|
3501
|
+
return object.__getattribute__(self._get_obj(), "_repr_short")(onde_class)
|
|
3502
|
+
|
|
3503
|
+
def _repr(self, indentation, onde_class=None, extra_attrs={}):
|
|
3504
|
+
return repr_helper(indentation, f"Writeable proxy of {object.__getattribute__(self._get_obj(), '_repr')(indentation+2, onde_class=onde_class, extra_attrs=extra_attrs):s}")
|
|
3505
|
+
|
|
3506
|
+
def __repr__(self):
|
|
3507
|
+
return f"Writeable proxy of {repr(self._get_obj()):s}"
|
|
3508
|
+
pass
|
|
3509
|
+
|
|
3510
|
+
|
|
3511
|
+
def _traverse_snapshot_fileneeded_objs(class_defs,cur_obj,add_cur_to_needed,needed_objs):
|
|
3512
|
+
"""Traverse the graph of ONDEBase objects depth-first starting at cur_obj, accumulating ONDEObjects that will need to be written into the file into the ordered dictionary needed_objs. It is an ordered dictionary because we will use its ordering for the object creation so that objects are created before they are used. The values stored into the ordered dictionary are all None."""
|
|
3513
|
+
edges = cur_obj._list_edges()
|
|
3514
|
+
|
|
3515
|
+
for edge in edges:
|
|
3516
|
+
sub_obj = cur_obj._follow_path((edge,))
|
|
3517
|
+
if sub_obj in needed_objs:
|
|
3518
|
+
continue # sub_obj has already been processed
|
|
3519
|
+
sub_is_needed = type(sub_obj) is ONDEObject
|
|
3520
|
+
_traverse_snapshot_fileneeded_objs(class_defs,sub_obj,sub_is_needed,needed_objs)
|
|
3521
|
+
pass
|
|
3522
|
+
|
|
3523
|
+
if add_cur_to_needed:
|
|
3524
|
+
needed_objs[cur_obj] = None
|
|
3525
|
+
pass
|
|
3526
|
+
pass
|
|
3527
|
+
|
|
3528
|
+
|
|
3529
|
+
def _search_h5_for_onde_datasets(h5_fh,h5_group,dataset_h5_groups,h5_groups_seen):
|
|
3530
|
+
""" dataset_h5_groups is a dictionary by hdf5 path of hdf5 group objects that contain an ONDE:TYPE attribute starting with ONDE_DATASET
|
|
3531
|
+
|
|
3532
|
+
|
|
3533
|
+
h5_groups_seen is a set of hdf5 group paths seen during the traversal
|
|
3534
|
+
"""
|
|
3535
|
+
|
|
3536
|
+
if "ONDE:TYPE" in h5_group.attrs and h5_group.attrs["ONDE:TYPE"][0] == "ONDE_DATASET":
|
|
3537
|
+
dataset_h5_groups[h5_group.name] = h5_group
|
|
3538
|
+
pass
|
|
3539
|
+
|
|
3540
|
+
h5_groups_seen.add(h5_group.name)
|
|
3541
|
+
|
|
3542
|
+
for (name,item) in h5_group.items():
|
|
3543
|
+
if isinstance(item,h5py.Group) and item.name not in h5_groups_seen:
|
|
3544
|
+
_search_h5_for_onde_datasets(h5_fh,item,dataset_h5_groups,h5_groups_seen)
|
|
3545
|
+
pass
|
|
3546
|
+
pass
|
|
3547
|
+
pass
|
|
3548
|
+
|
|
3549
|
+
class ONDEFile(object):
|
|
3550
|
+
"""Represents an HDF5 file that may contain ONDE objects. Unlike most other data structures it is not thread safe (only one thread at a time should be manipulating an ONDEFile). However it is safe for multiple threads to simultaneously access and even modify (by the usual ONDEProxy methods) objects from a single file, so long as only a single thread attempts to write any changes to disk."""
|
|
3551
|
+
file_object_dict = None # Dictionary by frozen ONDEBase object of ONDEFileObject.
|
|
3552
|
+
class_defs = None # ONDEClassDefinitions object
|
|
3553
|
+
h5path = None # file path used
|
|
3554
|
+
mode = None # mode used to open the file
|
|
3555
|
+
fh = None # h5py file handle.
|
|
3556
|
+
graph = None # ONDEFileGraph object (or subclass)
|
|
3557
|
+
db_maxidx = None # integer representing the highest index used in the PyONDE_DB hdf5 group
|
|
3558
|
+
version = None
|
|
3559
|
+
filetype = None
|
|
3560
|
+
|
|
3561
|
+
def __init__(self, class_defs = None, h5path = None, mode = None, fh = None,version = None, filetype = None):
|
|
3562
|
+
self.class_defs = class_defs
|
|
3563
|
+
self.h5path = h5path
|
|
3564
|
+
self.mode = mode
|
|
3565
|
+
self.fh = fh
|
|
3566
|
+
self.version = version
|
|
3567
|
+
self.filetype = filetype
|
|
3568
|
+
|
|
3569
|
+
self.file_object_dict = {}
|
|
3570
|
+
self.graph = ONDEFileGraph(class_defs = self.class_defs)
|
|
3571
|
+
|
|
3572
|
+
if self.mode not in {"r","r+","w","w-","x","a"}:
|
|
3573
|
+
raise ValueError(f"Unknown mode: {self.mode:s}")
|
|
3574
|
+
|
|
3575
|
+
if self.mode == "r+" or self.mode == "a" or self.mode == "r":
|
|
3576
|
+
|
|
3577
|
+
if version is not None:
|
|
3578
|
+
raise ValueError("Attempting to set ONDE_VERSION on existing file.")
|
|
3579
|
+
if filetype is not None:
|
|
3580
|
+
raise ValueError("Attempting to set ONDE_FILETYPE on existing file.")
|
|
3581
|
+
self.load() # Attempt to read in current contents.
|
|
3582
|
+
pass
|
|
3583
|
+
|
|
3584
|
+
pass
|
|
3585
|
+
|
|
3586
|
+
def __getitem__(self,index):
|
|
3587
|
+
return self.graph._get_item(index)
|
|
3588
|
+
|
|
3589
|
+
def __setitem__(self,index,value):
|
|
3590
|
+
return self.graph._set_item(index,value)
|
|
3591
|
+
|
|
3592
|
+
def keys(self):
|
|
3593
|
+
return self.graph.keys()
|
|
3594
|
+
|
|
3595
|
+
def __iter__(self):
|
|
3596
|
+
return iter(self.keys())
|
|
3597
|
+
|
|
3598
|
+
def load(self):
|
|
3599
|
+
self.version = self.fh.attrs["ONDE_VERSION"]
|
|
3600
|
+
self.filetype = set(self.fh.attrs["ONDE_FILETYPE"])
|
|
3601
|
+
|
|
3602
|
+
# check for PyONDE_DB and db_maxidx
|
|
3603
|
+
self.db_maxidx = 0
|
|
3604
|
+
if "PyONDE_DB" in self.fh:
|
|
3605
|
+
db = self.fh["PyONDE_DB"]
|
|
3606
|
+
for db_key in db.keys():
|
|
3607
|
+
if db_key.isdigit():
|
|
3608
|
+
db_keynum = int(db_key)
|
|
3609
|
+
if db_keynum > self.db_maxidx:
|
|
3610
|
+
self.db_maxidx = db_keynum
|
|
3611
|
+
pass
|
|
3612
|
+
pass
|
|
3613
|
+
pass
|
|
3614
|
+
pass
|
|
3615
|
+
#import pdb
|
|
3616
|
+
#pdb.set_trace()
|
|
3617
|
+
# Search through group structure for datasets.
|
|
3618
|
+
dataset_h5_groups = collections.OrderedDict() # dictionary by hdf5 path of hdf5 group objects that contain an ONDE:TYPE attribute starting with ONDE_DATASET
|
|
3619
|
+
h5_groups_seen = set() # set of hdf5 group paths seen during the traversal
|
|
3620
|
+
_search_h5_for_onde_datasets(self.fh,self.fh,dataset_h5_groups,h5_groups_seen)
|
|
3621
|
+
|
|
3622
|
+
fileobjs_by_h5path = {}
|
|
3623
|
+
|
|
3624
|
+
dataset_instances_by_h5path = collections.OrderedDict()
|
|
3625
|
+
for group_path in dataset_h5_groups:
|
|
3626
|
+
dataset_instances_by_h5path[group_path] = ONDEObject.load_from_hdf5(self,fileobjs_by_h5path,self.fh,dataset_h5_groups[group_path])
|
|
3627
|
+
pass
|
|
3628
|
+
|
|
3629
|
+
# replace existing contents of self.group as an atomic transaction
|
|
3630
|
+
with ONDETransaction(self.graph) as tr:
|
|
3631
|
+
for key in list(tr.graph.keys()):
|
|
3632
|
+
del tr.graph[key]
|
|
3633
|
+
pass
|
|
3634
|
+
|
|
3635
|
+
for h5path in dataset_instances_by_h5path:
|
|
3636
|
+
|
|
3637
|
+
instance = dataset_instances_by_h5path[h5path]
|
|
3638
|
+
if not "ONDE:UUID" in instance:
|
|
3639
|
+
raise ValueError(f"Error loading dataset from file {self.h5path:s} at hdf5 path {h5path:s}: No ONDE:UUID field specified")
|
|
3640
|
+
|
|
3641
|
+
# unique_id = instance["ONDE:UUID"].value
|
|
3642
|
+
# tr.graph[str(unique_id)] = instance
|
|
3643
|
+
tr.graph.add(instance)
|
|
3644
|
+
pass
|
|
3645
|
+
pass
|
|
3646
|
+
pass
|
|
3647
|
+
|
|
3648
|
+
|
|
3649
|
+
|
|
3650
|
+
def flush(self):
|
|
3651
|
+
snap = self.graph.latest_snap
|
|
3652
|
+
|
|
3653
|
+
# Ensure ONDE_FILETYPE and ONDE_VERSION are present
|
|
3654
|
+
if "ONDE_VERSION" not in self.fh.attrs:
|
|
3655
|
+
if self.version is not None:
|
|
3656
|
+
self.fh.attrs["ONDE_VERSION"] = self.version
|
|
3657
|
+
pass
|
|
3658
|
+
else:
|
|
3659
|
+
self.fh.attrs["ONDE_VERSION"] = self.class_defs.version
|
|
3660
|
+
pass
|
|
3661
|
+
pass
|
|
3662
|
+
|
|
3663
|
+
if "ONDE_FILETYPE" not in self.fh.attrs:
|
|
3664
|
+
if self.filetype is not None:
|
|
3665
|
+
self.fh.attrs["ONDE_FILETYPE"] = list(self.filetype)
|
|
3666
|
+
pass
|
|
3667
|
+
else:
|
|
3668
|
+
self.fh.attrs["ONDE_FILETYPE"] = ["ONDE_DATASET"]
|
|
3669
|
+
pass
|
|
3670
|
+
pass
|
|
3671
|
+
|
|
3672
|
+
# Make sure PyONDE_DB HDF5 group exists.
|
|
3673
|
+
if not "PyONDE_DB" in self.fh:
|
|
3674
|
+
self.fh.create_group("PyONDE_DB")
|
|
3675
|
+
self.db_maxidx = 0
|
|
3676
|
+
pass
|
|
3677
|
+
|
|
3678
|
+
db=self.fh["PyONDE_DB"]
|
|
3679
|
+
db_path = db.name
|
|
3680
|
+
|
|
3681
|
+
present_objs = set(self.file_object_dict.keys()) # set of ONDEBase objects already present in the file
|
|
3682
|
+
needed_objs = collections.OrderedDict()
|
|
3683
|
+
_traverse_snapshot_fileneeded_objs(self.class_defs,snap,False,needed_objs) # this function will fill needed_objs with an ordered set (implemented as keys of an ordered dictionary) of ONDEBase objects that are needed in the file. The ordering is depth-first so that leaf nodes get included prior to branch nodes that reference them. The False parameter indicates that the snap itself does NOT get included.
|
|
3684
|
+
|
|
3685
|
+
needed_objs_set = set(needed_objs.keys())
|
|
3686
|
+
# We need to remove objects that are listed in present_objs but not needed_objs
|
|
3687
|
+
to_remove = present_objs.difference(needed_objs_set)
|
|
3688
|
+
|
|
3689
|
+
# We need to add objects that are listed in needed_objs but not present_objs
|
|
3690
|
+
to_add = needed_objs_set.difference(present_objs)
|
|
3691
|
+
|
|
3692
|
+
# Perform the removal
|
|
3693
|
+
for obj in to_remove:
|
|
3694
|
+
fileobj = self.file_object_dict[obj]
|
|
3695
|
+
|
|
3696
|
+
#if fileobj.hdf5_attrname is not None:
|
|
3697
|
+
# # Object is stored as an attribute ... unnecessary becuase attributes will be written as part of their parent object.
|
|
3698
|
+
# attr_parent = self.fh[fileobj.hdf5_path]
|
|
3699
|
+
# del attr_parent[fileobj.hdf5_attrname]
|
|
3700
|
+
# pass
|
|
3701
|
+
#else:
|
|
3702
|
+
hdf5_obj=self.fh[fileobj.hdf5_path]
|
|
3703
|
+
hdf5_parent = hdf5_obj.parent
|
|
3704
|
+
del hdf5_parent[posixpath.basename(hdf5_obj.name)]
|
|
3705
|
+
|
|
3706
|
+
|
|
3707
|
+
fileobj.close()
|
|
3708
|
+
del self.file_object_dict[obj]
|
|
3709
|
+
pass
|
|
3710
|
+
|
|
3711
|
+
|
|
3712
|
+
# add in the needed objects. Loop ordering is because needed_objs is actually an OrderedDict.
|
|
3713
|
+
for needed_obj in needed_objs:
|
|
3714
|
+
if not needed_obj in to_add:
|
|
3715
|
+
continue
|
|
3716
|
+
|
|
3717
|
+
obj_name = f"{self.db_maxidx + 1:05d}"
|
|
3718
|
+
self.db_maxidx += 1
|
|
3719
|
+
|
|
3720
|
+
fileobj = ONDEFileObject(onde_instance = needed_obj,hdf5_path = posixpath.join(db_path,obj_name))
|
|
3721
|
+
|
|
3722
|
+
needed_obj._hdf5_write_group(onde_file = self,parent = db,parent_path = db_path,name = obj_name,onde_fileobj = fileobj)
|
|
3723
|
+
|
|
3724
|
+
self.file_object_dict[needed_obj] = fileobj
|
|
3725
|
+
|
|
3726
|
+
pass
|
|
3727
|
+
|
|
3728
|
+
|
|
3729
|
+
self.fh.flush()
|
|
3730
|
+
pass
|
|
3731
|
+
|
|
3732
|
+
|
|
3733
|
+
|
|
3734
|
+
def close(self):
|
|
3735
|
+
|
|
3736
|
+
if self.mode in {"r+","w","w-","x","a"}:
|
|
3737
|
+
self.flush()
|
|
3738
|
+
pass
|
|
3739
|
+
|
|
3740
|
+
self.fh.close()
|
|
3741
|
+
for key in self.file_object_dict:
|
|
3742
|
+
self.file_object_dict[key].close()
|
|
3743
|
+
pass
|
|
3744
|
+
self.file_object_dict = None
|
|
3745
|
+
self.fh = None
|
|
3746
|
+
pass
|
|
3747
|
+
|
|
3748
|
+
def __enter__(self):
|
|
3749
|
+
return self
|
|
3750
|
+
|
|
3751
|
+
def __exit__(self, exc_type, exc_value, tb):
|
|
3752
|
+
self.close()
|
|
3753
|
+
return False
|
|
3754
|
+
|
|
3755
|
+
def _repr(self, indentation, onde_class=None, extra_attrs={}):
|
|
3756
|
+
return repr_helper(indentation, "ONDEFile", _extra_info = f"open on \"{self.h5path:s}\"")+"\n"+self.graph._repr(indentation+2)
|
|
3757
|
+
|
|
3758
|
+
def __repr__(self):
|
|
3759
|
+
return self._repr(0)
|
|
3760
|
+
|
|
3761
|
+
@classmethod
|
|
3762
|
+
def new(cls, h5path, mode, class_defs_path=None, extra_class_defs = None, onde_version = None):
|
|
3763
|
+
fh = h5py.File(h5path, mode)
|
|
3764
|
+
|
|
3765
|
+
class_defs = None
|
|
3766
|
+
if class_defs_path is None and onde_version is not None:
|
|
3767
|
+
class_defs_path = os.path.join(os.path.split(sys.modules[cls.__module__].__file__)[0],"onde_versions",f"ONDE_fields_v{onde_version:s}.csv")
|
|
3768
|
+
pass
|
|
3769
|
+
if class_defs_path is not None:
|
|
3770
|
+
class_defs = ONDEClassDefinitions.load_from_csv(class_defs_path,extra_class_defs = extra_class_defs)
|
|
3771
|
+
pass
|
|
3772
|
+
|
|
3773
|
+
return cls(class_defs = class_defs, h5path = h5path, mode = mode, fh = fh)
|
|
3774
|
+
|
|
3775
|
+
pass
|
|
3776
|
+
|
|
3777
|
+
class ONDEDatasetFile(ONDEFile):
|
|
3778
|
+
"""Represents an ONDE 1.0 file that is expected to contain one or more ONDE_DATASET instances."""
|
|
3779
|
+
pass
|
|
3780
|
+
|
|
3781
|
+
class ONDEFileObject(object):
|
|
3782
|
+
"""Represents a frozen ONDEBase subclass instance that is represented in an HDF5 file."""
|
|
3783
|
+
onde_instance = None # Reference to an ONDEBase subclass instance.
|
|
3784
|
+
hdf5_path = None # Path of the given object in the file.
|
|
3785
|
+
#hdf5_attrname = None # If this FileObject is stored as an attribute, then this is the attribute name. Unnecessary because attributes will be written as part of their parent object.
|
|
3786
|
+
# hdf5_obj = None # Actual h5py object.
|
|
3787
|
+
|
|
3788
|
+
def __init__(self, onde_instance = None, hdf5_path = None): #, hdf5_obj = None):
|
|
3789
|
+
self.onde_instance = onde_instance
|
|
3790
|
+
self.hdf5_path = hdf5_path
|
|
3791
|
+
#self.hdf5_obj = hdf5_obj
|
|
3792
|
+
|
|
3793
|
+
with self.onde_instance._file_realizations_lock:
|
|
3794
|
+
self.onde_instance._file_realizations.add(self)
|
|
3795
|
+
pass
|
|
3796
|
+
|
|
3797
|
+
pass
|
|
3798
|
+
|
|
3799
|
+
def close(self):
|
|
3800
|
+
self.hdf5_path = None
|
|
3801
|
+
#self.hdf5_obj = None
|
|
3802
|
+
|
|
3803
|
+
with self.onde_instance._file_realizations_lock:
|
|
3804
|
+
self.onde_instance._file_realizations.remove(self)
|
|
3805
|
+
pass
|
|
3806
|
+
self.onde_instance = None
|
|
3807
|
+
pass
|
|
3808
|
+
pass
|
|
3809
|
+
|
|
3810
|
+
|
|
3811
|
+
|
|
3812
|
+
# rough api of transactions
|
|
3813
|
+
# with ONDETransaction(graph, include_scope, exclude_scope) as transaction:
|
|
3814
|
+
# transaction.obj.attr = "hello"
|
|
3815
|
+
|
|
3816
|
+
|
|
3817
|
+
#class ONDEOperation(object):
|
|
3818
|
+
# """Represents an operation that can be part of a transaction"""
|
|
3819
|
+
# scope = None # ONDEOpScope reference or None representing default (local) scope
|
|
3820
|
+
# pass
|
|
3821
|
+
|
|
3822
|
+
|
|
3823
|
+
#class ONDEAssignValue(ONDEOperation):
|
|
3824
|
+
# """Represents assignment of the value within an ONDEValue object"""
|
|
3825
|
+
# attr_name = None # Usually "value"
|
|
3826
|
+
# attr_value = None # String, int, float, immutable array, etc.
|
|
3827
|
+
# pass
|