PyONDE 0.1.2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
PyONDE/onde.py ADDED
@@ -0,0 +1,3827 @@
1
+ # Road map:
2
+ # *Need to implement _del_attr and _del_item on various classes.
3
+
4
+ import h5py
5
+ import sys
6
+ import os
7
+ import os.path
8
+ import posixpath
9
+ import threading
10
+ import ast
11
+ import csv
12
+ import copy
13
+ import numbers
14
+ import collections
15
+ import collections.abc
16
+ from dataclasses import dataclass, field
17
+ from typing import Optional, Any
18
+ import numpy as np
19
+
20
+ onde_basic_attrs = {
21
+ "ONDEValue":(
22
+ "_freeze",
23
+ "_frozen",
24
+ "_get_attr",
25
+ "_get_data",
26
+ "_set_attr",
27
+ "_set_data",
28
+ "_has_attr",
29
+ "value"
30
+ ),
31
+ "ONDEArray":(
32
+ "_freeze",
33
+ "_frozen",
34
+ "_get_attr",
35
+ "_get_data",
36
+ "_set_attr",
37
+ "_set_data",
38
+ "_has_attr",
39
+ "value"
40
+ ),
41
+ "ONDEReferenceArray":(
42
+ "_freeze",
43
+ "_frozen",
44
+ "_get_attr",
45
+ "_get_data",
46
+ "_get_item",
47
+ "_set_attr",
48
+ "_set_data",
49
+ "_set_item",
50
+ "_has_attr",
51
+ "refs"
52
+ ),
53
+ "ONDEObject":(
54
+ "_freeze",
55
+ "_frozen",
56
+ "_get_attr",
57
+ "_set_attr",
58
+ "_has_attr",
59
+ "_list_attrs"
60
+ ),
61
+ "ONDEFileGraphSnapshot":(
62
+ "_freeze",
63
+ "_frozen",
64
+ "_get_attr",
65
+ "_set_attr",
66
+ "_has_attr",
67
+ "_list_attrs"
68
+ )
69
+ }
70
+
71
+ def repr_helper(indentation, classname, _extra_info = None, **attrdict):
72
+ lines = []
73
+ classline = " "*indentation + classname
74
+ if _extra_info is not None:
75
+ classline += " "+_extra_info
76
+ pass
77
+ lines.append(classline)
78
+ for attrname in attrdict:
79
+ lines.append(" "*(indentation+2) + f"{attrname:>18s}: {attrdict[attrname]:s}")
80
+ pass
81
+ return "\n".join(lines)
82
+
83
+ def onde_from_python(value,field):
84
+ """field is an ONDEField or None. So far it is not used because we don't have code to parse the type specification"""
85
+ if isinstance(value,numbers.Integral) or isinstance(value, numbers.Real) or isinstance(value,numbers.Complex) or isinstance(value,str) or isinstance(value,np.str_):
86
+ return ONDEValue.new(value)
87
+
88
+ if isinstance(value,collections.abc.Sequence):
89
+ subvalues = []
90
+ all_string = True
91
+ all_numeric = True
92
+ any_complex = False
93
+ all_object = True
94
+ for subvalue in value:
95
+ if isinstance(value,str) or isinstance(value,np.str_):
96
+ all_numeric = False
97
+ all_object = False
98
+ pass
99
+ elif isinstance(value,numbers.Real):
100
+ all_string = False
101
+ all_object = False
102
+ pass
103
+ elif isinstance(value,numbers.Complex):
104
+ all_string = False
105
+ any_complex = True
106
+ all_object = False
107
+ pass
108
+ elif isinstance(value,collections.abc.Sequence):
109
+ all_string = False
110
+ all_numeric = False
111
+ value = onde_from_python(value)
112
+ pass
113
+ else:
114
+ raise ValueError(f"Could not convert object of type {value.__class__.__name__:s} into an ONDE object")
115
+
116
+ subvalues.append(value)
117
+ pass
118
+
119
+ if all_string and not(all_numeric) and not(all_object):
120
+ dtype = np.StringDType()
121
+ return ONDEArray.new(value = np.array(subvalues,dtype = dtype))
122
+ elif all_numeric and not(all_string) and not(all_object):
123
+ if any_complex:
124
+ dtype = "D"
125
+ pass
126
+ else:
127
+ dtype = "d"
128
+ pass
129
+ return ONDEArray.new(value = np.array(subvalues,dtype = dtype))
130
+
131
+ elif all_object and not(all_string) and not(all_numeric):
132
+ return ONDEReferenceArray.new(refs = np.array(subvalues,dtype = "O"))
133
+ else:
134
+ raise ValueError(f"Could not identify unique type for converting python object {str(value):s} into an ONDE object")
135
+ pass
136
+
137
+
138
+ class TwoWayDictionary(object):
139
+ """A dictionary that is indexable by strings using
140
+ getattr() or dot notation to get objects. Can be
141
+ reverse indexed by those objects using bracket notation
142
+ to get strings, which will return a tuple because
143
+ multiple index values can point to the same object."""
144
+ _bystrings = None # String index dictionary, contains objects
145
+ _byobjid = None # id(object) dictionary, contains frozensets of strings
146
+ _lock = None # threading.Lock object protecting _bystrings and _byobjid from changes. This lock is last in the locking order, ie you may not acquire any other lock while holding this lock.
147
+ _frozen = None # Boolean. Dictionary cannot be modified once frozen
148
+
149
+ def __init__(self, bystrings=None):
150
+ """Pass an existing dictionary or None as bystrings"""
151
+ if bystrings is None:
152
+ bystrings = {}
153
+ pass
154
+
155
+ if isinstance(bystrings, TwoWayDictionary):
156
+ bystrings = object.__getattribute__(bystrings, "_bystrings")
157
+ pass
158
+
159
+ object.__setattr__(self, "_bystrings", dict(bystrings))
160
+ byobjid = dict()
161
+ for s in bystrings:
162
+ obj = bystrings[s]
163
+
164
+ if obj is not None:
165
+ if id(obj) in byobjid:
166
+ idx_set = byobjid[id(obj)]
167
+ pass
168
+ else:
169
+ idx_set = frozenset()
170
+ pass
171
+
172
+ byobjid[id(obj)] = idx_set | { s }
173
+ pass
174
+ pass
175
+ object.__setattr__(self, "_byobjid",byobjid)
176
+ object.__setattr__(self, "_lock", threading.Lock())
177
+ object.__setattr__(self, "_frozen", False)
178
+ pass
179
+
180
+ def __iter__(self):
181
+ _bystrings = object.__getattribute__(self, "_bystrings")
182
+ return _bystrings.__iter__() # Just use iterator of underlying dictionary
183
+
184
+ def __getattribute__(self, name):
185
+ if name.startswith("_"):
186
+ if name in {"_freeze", "_frozen", "_set_attr", "_get_attr", "_keys","_del_attr"}:
187
+ return object.__getattribute__(self, name)
188
+ raise IndexError("TwoWayDictionary: Indexes are not allowed to have leading underscores")
189
+
190
+ _get_attr = object.__getattribute__(self, "_get_attr")
191
+
192
+ return _get_attr(name)
193
+
194
+ def __call__(self, obj):
195
+ _byobjid = object.__getattribute__(self, "_byobjid")
196
+ objidx = _byobjid[id(obj)]
197
+ return objidx # Returns frozenset
198
+
199
+
200
+ def __setattr__(self, name, value):
201
+ _set_attr = object.__getattribute__(self, "_set_attr")
202
+ _set_attr(name, value)
203
+
204
+ pass
205
+
206
+ def __setitem__(self,name,value):
207
+ _set_attr = object.__getattribute__(self, "_set_attr")
208
+ _set_attr(name, value)
209
+
210
+ pass
211
+
212
+ def _set_attr(self, name, value):
213
+ if name.startswith("_"):
214
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
215
+ _bystrings = object.__getattribute__(self, "_bystrings")
216
+ _byobjid = object.__getattribute__(self, "_byobjid")
217
+ _lock = object.__getattribute__(self, "_lock")
218
+ _frozen = object.__getattribute__(self, "_frozen")
219
+
220
+ if _frozen:
221
+ raise AttributeError("Not allowed to modify a frozen TwoWayDictionary")
222
+ with _lock:
223
+ if name in _bystrings:
224
+ # Remove old back-reference
225
+ oldobj = _bystrings[name]
226
+ _byobjid[id(oldobj)] = _byobjid[id(oldobj)] - frozenset({name})
227
+ pass
228
+
229
+ _bystrings[name] = value
230
+
231
+ nameset = frozenset({name})
232
+ # Check if this object already has an existing set of references
233
+ if id(value) in _byobjid:
234
+ nameset = _byobjid[id(value)] | nameset
235
+ pass
236
+ _byobjid[id(value)] = nameset
237
+ pass
238
+ pass
239
+
240
+ def _del_attr(self,name):
241
+ _bystrings = object.__getattribute__(self, "_bystrings")
242
+ _byobjid = object.__getattribute__(self, "_byobjid")
243
+ _lock = object.__getattribute__(self, "_lock")
244
+ _frozen = object.__getattribute__(self, "_frozen")
245
+
246
+ if _frozen:
247
+ raise AttributeError("Not allowed to modify a frozen TwoWayDictionary")
248
+ with _lock:
249
+ if name in _bystrings:
250
+ # Remove old back-reference
251
+ oldobj = _bystrings[name]
252
+ _byobjid[id(oldobj)] = _byobjid[id(oldobj)] - frozenset({name})
253
+ pass
254
+ del _bystrings[name]
255
+ pass
256
+ pass
257
+
258
+ def _get_attr(self, name):
259
+ if name.startswith("_"):
260
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
261
+
262
+ _bystrings = object.__getattribute__(self, "_bystrings")
263
+ return _bystrings[name]
264
+
265
+ def __getitem__(self,name):
266
+
267
+ return self._get_attr(name)
268
+
269
+ def __delitem__(self,name):
270
+ return self._del_attr(name)
271
+
272
+ def _freeze(self):
273
+ object.__setattr__(self, "_frozen", True)
274
+ pass
275
+
276
+ def _keys(self):
277
+ _bystrings = object.__getattribute__(self, "_bystrings")
278
+ return _bystrings.keys()
279
+
280
+ pass
281
+
282
+ class TwoWayArray(object):
283
+ """An Array that is indexable by integers using
284
+ bracket notation. Can be
285
+ reverse indexed by those objects using parenthesis notation
286
+ to get index tuples, which will be wrapped in another layer
287
+ of tuple because
288
+ multiple index values can point to the same object."""
289
+ _byindex = None # numpy array, contains objects
290
+ _byobjid = None # id(object) dictionary, contains frozensets of strings
291
+ _lock = None # threading.Lock object protecting _byindex and _byobjid from changes. This lock is last in the locking order, ie you may not acquire any other lock while holding this lock.
292
+ _frozen = None # Boolean. Dictionary cannot be modified once frozen
293
+
294
+ def __init__(self, byindex=None,shape = None):
295
+ """Pass an existing array or None as byindex"""
296
+ if shape is None:
297
+ shape = ()
298
+ pass
299
+
300
+ if byindex is None:
301
+ byindex = np.empty(shape,dtype="O")
302
+ pass
303
+
304
+ if isinstance(byindex, TwoWayArray):
305
+ byindex = byindex.byindex
306
+ pass
307
+
308
+ self._byindex = copy.copy(byindex)
309
+ self._byobjid = {}
310
+ nditer = np.nditer(self._byindex, flags = ("multi_index","refs_ok"))
311
+ for objarray in nditer:
312
+ obj = objarray[()]
313
+ if obj is not None:
314
+ if id(obj) in self._byobjid:
315
+ idx_set = self._byobjid[id(obj)]
316
+ pass
317
+ else:
318
+ idx_set = frozenset()
319
+ pass
320
+
321
+ self._byobjid[id(obj)] = idx_set | { tuple(nditer.multi_index) }
322
+ pass
323
+ pass
324
+
325
+ self._lock = threading.Lock()
326
+ self._frozen = False
327
+
328
+ pass
329
+
330
+ @property
331
+ def byindex(self):
332
+ """Return safe copy of underlying array"""
333
+ with self._lock:
334
+ byindex = copy.copy(self._byindex)
335
+ pass
336
+ return byindex
337
+
338
+ def __iter__(self):
339
+ return np.nditer(self._byindex,flags=("multi_index","refs_ok")) # Just use iterator of underlying array
340
+
341
+ def __getitem__(self, index):
342
+
343
+ return self._byindex[index]
344
+
345
+ @property
346
+ def _shape(self):
347
+ return self._byindex.shape
348
+
349
+ def __call__(self, obj):
350
+ objidx = self._byobjid[id(obj)]
351
+ return objidx # Returns frozenset
352
+
353
+
354
+ def __setitem__(self, index, obj):
355
+ if not isinstance(index,tuple):
356
+ index = (index,)
357
+ pass
358
+
359
+ if self._frozen:
360
+ raise AttributeError("Not allowed to modify a frozen TwoWayArray")
361
+ with self._lock:
362
+ oldobj = self._byindex[index]
363
+ if oldobj is not None:
364
+ # Remove old back-reference
365
+ self._byobjid[id(oldobj)] = self._byobjid[id(oldobj)] - frozenset({index})
366
+ pass
367
+
368
+ self._byindex[index] = obj
369
+
370
+ indexset = frozenset({index})
371
+ # Check if this object already has an existing set of references
372
+ if id(obj) in self._byobjid:
373
+ indexset = self._byobjid[id(obj)] | indexset
374
+ pass
375
+ self._byobjid[id(obj)] = indexset
376
+ pass
377
+ pass
378
+
379
+ def _freeze(self):
380
+ self._byindex.flags.writeable = False
381
+ self._frozen = True
382
+ pass
383
+ pass
384
+
385
+
386
+ class ONDEFileGraph(object):
387
+ """Represents the subgraph of interconnected ONDE objects
388
+ and attributes that are or will be contained within some particular ONDE file. It has an associated set of ONDEClassDefinitions that are used to interpret the attached objects."""
389
+ _lock = None # threading.Lock that protects access to replace the snapshot.
390
+ _lock_ownerthread = None # Writes protected by _lock, the threading.get_ident() of whichever thread owns the lock
391
+ latest_snap = None # class ONDEFileGraphSnapshot
392
+ class_defs = None # ONDEClassDefinitions, optional
393
+
394
+ def __init__(self, class_defs = None, latest_snap = None):
395
+ # self._lock = threading.Lock()
396
+ object.__setattr__(self, "_lock", threading.Lock())
397
+
398
+ if latest_snap is None:
399
+ latest_snap = ONDEFileGraphSnapshot.new()
400
+
401
+ pass
402
+
403
+ if not latest_snap._frozen:
404
+ latest_snap._freeze()
405
+ pass
406
+
407
+ object.__setattr__(self, "class_defs", class_defs)
408
+
409
+ # self.latest_snap = snapshot
410
+ object.__setattr__(self, "latest_snap", latest_snap)
411
+
412
+ pass
413
+
414
+ r"""
415
+ def __getattribute__(self, name):
416
+ if name.startswith("_"):
417
+
418
+ if name in {"_set_attr", "_get_attr","__dict__","__dir__","_latest_snap","_lock", "_class_defs"}:
419
+ return object.__getattribute__(self, name)
420
+ pass
421
+
422
+ _latest_snap = object.__getattribute__(self, "_latest_snap")
423
+
424
+ # call ONDEProxy
425
+
426
+ snap_proxy = ONDEProxy.new_from_snapshot(self, _latest_snap)
427
+ obj_proxy = ONDEProxy.new_from_proxy(snap_proxy, name)
428
+
429
+ if self._class_defs is not None:
430
+ return ONDEClassInstanceWrapper.new_from_proxy(obj_proxy)
431
+ return obj_proxy
432
+
433
+ def __setattr__(self, name, value):
434
+ if name.startswith("_"):
435
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
436
+
437
+ _latest_snap = object.__getattribute__(self, "_latest_snap")
438
+
439
+ snap_proxy = ONDEProxy.new_from_snapshot(self, _latest_snap)
440
+
441
+ snap_proxy._set_attr(name,value)
442
+
443
+ pass
444
+
445
+ def _get_attr(self, name):
446
+ if name.startswith("_"):
447
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
448
+
449
+ _latest_snap = object.__getattribute__(self, "_latest_snap")
450
+
451
+ # call ONDEProxy
452
+
453
+ snap_proxy = ONDEProxy.new_from_snapshot(self, _latest_snap)
454
+ obj_proxy = ONDEProxy.new_from_proxy(snap_proxy, name)
455
+ if self._class_defs is not None:
456
+ return ONDEClassInstanceWrapper.new(obj_proxy)
457
+
458
+ return obj_proxy # _latest_snap._get_attr(name)
459
+
460
+ def _set_attr(self, name, value):
461
+ if name.startswith("_"):
462
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
463
+
464
+ _latest_snap = object.__getattribute__(self, "_latest_snap")
465
+ snap_proxy._set_attr(name,value)
466
+
467
+ pass
468
+
469
+ def _get_item(self, index):
470
+ # Strictly shouldn't be necessary unless we shift to indexing the snapshot with numbers rather than strings
471
+ _latest_snap = object.__getattribute__(self, "_latest_snap")
472
+
473
+ # call ONDEProxy
474
+
475
+ snap_proxy = ONDEProxy.new_from_snapshot(self, _latest_snap)
476
+ obj_proxy = ONDEProxy.new_from_proxy(snap_proxy, index)
477
+ if self._class_defs is not None:
478
+ return ONDEClassInstanceWrapper.new(obj_proxy)
479
+
480
+ return obj_proxy # _latest_snap._get_attr(name)
481
+
482
+ def _set_item(self, index, value):
483
+ if name.startswith("_"):
484
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
485
+
486
+ _latest_snap = object.__getattribute__(self, "_latest_snap")
487
+
488
+ _latest_snap._set_item(index,value)
489
+ pass
490
+ """
491
+
492
+
493
+ def _get_item(self, name):
494
+ latest_snap = object.__getattribute__(self, "latest_snap")
495
+ # Since The entry points are an ONDEFileGraphSnapshot, which does special flexible indexing with shorthands, we do the actual lookup to get the real index.
496
+
497
+ obj = latest_snap._get_item(name)
498
+ obj_keys= [ key for key in latest_snap.keys() if latest_snap._get_item(key) is obj ]
499
+ assert (len(obj_keys)==1)
500
+ name=obj_keys[0]
501
+ # call ONDEProxy
502
+
503
+ snap_proxy = ONDEProxy.new_from_snapshot(self, latest_snap)
504
+ obj_proxy = ONDEProxy.new_from_proxy(snap_proxy, name)
505
+
506
+ if self.class_defs is not None:
507
+ return ONDEClassInstanceWrapper.new_from_proxy(obj_proxy)
508
+ return obj_proxy
509
+
510
+ def __getitem__(self, name):
511
+ return self._get_item(name)
512
+
513
+ #def _set_item(self, name, value):
514
+ # latest_snap = object.__getattribute__(self, "latest_snap")
515
+ #
516
+ # snap_proxy = ONDEProxy.new_from_snapshot(self, latest_snap)
517
+ #
518
+ # snap_proxy._set_item(name,value)
519
+ #
520
+ # pass
521
+
522
+ def add(self,dataset):
523
+ latest_snap = object.__getattribute__(self, "latest_snap")
524
+
525
+ snap_proxy = ONDEProxy.new_from_snapshot(self, latest_snap)
526
+
527
+ snap_proxy.add(dataset)
528
+
529
+ pass
530
+
531
+ #def __setitem__(self, name, value):
532
+ # self._set_item(name,value)
533
+ # pass
534
+
535
+ def keys(self):
536
+ latest_snap = object.__getattribute__(self, "latest_snap")
537
+ return latest_snap._list_items()
538
+
539
+ def shortkeys(self):
540
+ latest_snap = object.__getattribute__(self, "latest_snap")
541
+ return latest_snap.shortkeys()
542
+
543
+ def __iter__(self):
544
+ return iter(self.keys())
545
+
546
+ def _repr(self, indentation, onde_class=None, extra_attrs={}):
547
+ extra_info = None
548
+ if self.class_defs is not None:
549
+ extra_info = f"(ONDE v{self.class_defs.version:s} class definitions)"
550
+ pass
551
+ return repr_helper(indentation, "ONDEFileGraph", _extra_info = extra_info)+"\n"+self.latest_snap._repr(indentation+2)
552
+
553
+ def __repr__(self):
554
+ return self._repr(0)
555
+
556
+
557
+ @classmethod
558
+ def new(cls,class_defs=None,snapshot=None):
559
+ #defs = None
560
+ #if class_def_csv_path is not None:
561
+ # defs = ONDEClassDefinitions.load_from_csv(class_def_csv_path)
562
+ # pass
563
+
564
+ return cls(class_defs = class_defs,latest_snap=snapshot)
565
+
566
+ pass
567
+
568
+ class ONDEDatasetFileGraph(ONDEFileGraph):
569
+ """Represents an ONDEFileGraph that is presumed to contain at its base level a number of ONDE_DATASET objects as per the ONDE 1.0 specification."""
570
+ # Inherited members from ONDEFileGraph
571
+ # _lock = None # threading.Lock that protects access to replace the snapshot.
572
+ # _lock_ownerthread = None # Writes protected by _lock, the threading.get_ident() of whichever thread owns the lock
573
+ # latest_snap = None # class ONDEFileGraphSnapshot
574
+ # class_defs = None # ONDEClassDefinitions, optional
575
+
576
+ def __init__(self, class_defs = None, latest_snap = None):
577
+ super().__init__(self, class_defs = class_defs, latest_snap = latest_snap)
578
+ pass
579
+ # Various methods inherited from ONDEFileGraph
580
+ # !!!*** should provide way to add a dataset that automatically sets the index
581
+ pass
582
+
583
+ class ONDETransaction(object):
584
+ """Represents a transaction in which the ONDEFileGraph is modified"""
585
+ graph = None # ONDEFileGraph object
586
+ scope = None
587
+ snap = None # This is the snapshot we're modifying
588
+
589
+ def __init__(self, graph, include_paths=None, exclude_paths=None,exclude_objects=None):
590
+ self.graph = graph
591
+
592
+ self.scope = ONDEOpScope(self,include_paths, exclude_paths,exclude_objects)
593
+
594
+
595
+ pass
596
+
597
+ def __enter__(self):
598
+ lockowner = object.__getattribute__(self.graph, "_lock_ownerthread")
599
+ if lockowner == threading.get_ident():
600
+ raise RuntimeError(f"Error creating a new transaction while another transaction is already open on the file graph by the same thread. This probably means that you are attempting a change on an object not accessed via the transaction. Within a transaction, always access objects via the transaction or scope objects.")
601
+ _lock = object.__getattribute__(self.graph, "_lock")
602
+ _lock.acquire()
603
+
604
+ object.__setattr__(self.graph, "_lock_ownerthread", threading.get_ident())
605
+
606
+ snap = object.__getattribute__(self.graph, "latest_snap")
607
+ self.snap = ONDEFileGraphSnapshot(_orig=snap) # Create mutable copy of most recent snapshot
608
+
609
+ return self.scope
610
+
611
+
612
+ def __exit__(self, exc_type, exc, tb):
613
+ self.snap._freeze()
614
+ object.__setattr__(self.graph, "latest_snap",self.snap)
615
+ object.__setattr__(self.graph, "_lock_ownerthread", None)
616
+ _lock = object.__getattribute__(self.graph, "_lock")
617
+ _lock.release()
618
+
619
+ pass
620
+
621
+ pass
622
+
623
+
624
+ class ONDEOpScope(object):
625
+ """Represents scope of an operation. This includes a list of paths originating at entry points that are included in the scope, and an overriding list of paths originating at entry points that are excluded from the scope"""
626
+ include_paths = None # List of ONDEPath objects, or None representing no scope specified
627
+ exclude_paths = None # frozenset of ONDEPath objects
628
+ exclude_objects = None # frozenset of objects to be excluded
629
+ trans = None # Transaction for this scope (warning: creates reference loop)
630
+
631
+ def __init__(self, trans,include_paths=None, exclude_paths = None, exclude_objects = None):
632
+ self.trans=trans
633
+ self.include_paths = include_paths
634
+ if exclude_paths is None:
635
+ exclude_paths = []
636
+ pass
637
+ self.exclude_paths = frozenset(exclude_paths)
638
+
639
+ if exclude_objects is None:
640
+ exclude_objects = []
641
+ pass
642
+ self.exclude_objects = frozenset(exclude_objects)
643
+ pass
644
+
645
+ def __enter__(self):
646
+
647
+ return self
648
+
649
+ def __exit__(self, exc_type, exc, tb):
650
+ pass
651
+
652
+ @property
653
+ def graph(self):
654
+ return ONDEProxy.new_from_scope(self)
655
+
656
+ def rescope(self,include_paths=None, exclude_paths = None, exclude_objects = None):
657
+ return type(self)(include_paths, exclude_paths, exclude_objects)
658
+
659
+ def __hash__(self):
660
+ return hash((tuple(self.include_paths), self.exclude_paths, self.exclude_objects))
661
+
662
+ # to do: the paths in the onde op scope will generally end with an
663
+ # attribute name or an array index and the scope starts from only that
664
+ # attribute of the object
665
+
666
+ pass
667
+
668
+
669
+ class ONDEPath(tuple):
670
+ """A path, essentially a tuple of strings or indexes/tuples, starting
671
+ with an entry_point name and followed by attribute names
672
+ for ONDEObjects or ONDEReferenceArray indexes, that leads to an ONDEBase.
673
+
674
+ An ONDEPath is evaluated by calling the _follow_path method on the object it is relative to, passing the path as the parameter."""
675
+
676
+ def __new__(cls, *args, **kwargs):
677
+ return super().__new__(cls,*args, **kwargs)
678
+
679
+ pass
680
+
681
+ class ONDEBase(object):
682
+ # _graph = None # ONDEFileGraph object
683
+ _referencedby = None # set of ONDEBase objects that reference this object. The _referencedby member can still be changed even after an instance is frozen becuase new objects can reference it. Note that the _referencedby field is generally only updated to include new objects when those objects become frozen. Access is protected by _referencedby_lock.
684
+ _referencedby_lock = None # threading.Lock object that protects _referencedby
685
+
686
+ _frozen = None # True/False: has this object been finalized and therefore become immutable
687
+ _modification_scopes = None # A set of scopes for which the ancestor nodes in the graph have been replaced for the transaction in which this node is being updated. It is only valid for use within the context of the transaction in which the node is being created and it is cleared when the node is frozen.
688
+
689
+ _file_realizations = None # A set of ONDEFileObjects that represent realizations in hdf5 of this object. Locked by _file_realizations_lock.
690
+ _file_realizations_lock = None # A threading.Lock object that protects _file_realizations.
691
+
692
+ def __init__(self, _orig = None, **kwargs):
693
+ _referencedby = None
694
+ _file_realizations = None
695
+ #if _orig is not None:
696
+ # _referencedby = object.__getattribute__(_orig, "_referencedby")
697
+ # pass
698
+ if "_referencedby" in kwargs:
699
+ _referencedby = set(kwargs["_referencedby"])
700
+ del kwargs["_referencedby"]
701
+ pass
702
+ if "_file_realizations" in kwargs:
703
+ _file_realizations = set(kwargs["_file_realizations"])
704
+ del kwargs["_file_realizations"]
705
+ pass
706
+
707
+ _frozen = False
708
+ if "_frozen" in kwargs:
709
+ _frozen = kwargs["_frozen"]
710
+ del kwargs["_frozen"]
711
+ pass
712
+ if len(kwargs) > 0:
713
+ raise AttributeError(f"Unknown constructor parameters to {self.__class__.__name__:s}: {str(list(kwargs.keys())):s}")
714
+ if _referencedby is None:
715
+ _referencedby = set()
716
+ pass
717
+ if _file_realizations is None:
718
+ _file_realizations = set()
719
+ pass
720
+
721
+ object.__setattr__(self, "_referencedby", _referencedby)
722
+ object.__setattr__(self, "_referencedby_lock", threading.Lock())
723
+ object.__setattr__(self, "_file_realizations", _file_realizations)
724
+ object.__setattr__(self, "_file_realizations_lock", threading.Lock())
725
+ object.__setattr__(self, "_frozen", False)
726
+ if _frozen:
727
+ self._freeze() # Derived class may have additional operations
728
+ pass
729
+ else:
730
+ object.__setattr__(self, "_modification_scopes", set())
731
+ pass
732
+ pass
733
+
734
+ def __getattribute__(self,name):
735
+ if name.startswith("_"):
736
+ if name in {"_freeze", "_frozen", "_add_referencedby", "__class__", "__dir__", "_get_attr","_get_attr_dataset_storage", "_set_attr","_set_dataset_attr","_get_data","_set_data","_get_item","_list_items","_set_item","_follow_path","_modification_scopes","_indices_for_object","_assign_pathel","_list_edges", "_repr", "_repr_short","_hdf5_write_group","_hdf5_write_dataset","_hdf5_write_attribute","_file_realizations","_file_realizations_lock","_add_dataset"}:
737
+ return object.__getattribute__(self, name)
738
+ raise IndexError(f"ONDEBase: The attribute {name:s} has a leading underscore, which is not allowed.")
739
+ raise IndexError(f"ONDEBase: Unknown attribute {name:s}")
740
+
741
+ def __setattr__(self, name, value):
742
+ _frozen = object.__getattribute__(self, "_frozen")
743
+ if _frozen:
744
+ raise RuntimeError("Attempting to modify an object that is already frozen")
745
+ # object.__setattr__(self,name,value)
746
+ raise ValueError(f"ONDEBase is not mutable")
747
+
748
+ # Attributes are other ONDEBase objects
749
+ def _get_attr(self, name):
750
+ """
751
+ Return the named conceptual attribute of an ONDE object.
752
+ """
753
+
754
+ raise ValueError("ONDEBase does not have attributes")
755
+
756
+ def _get_attr_dataset_storage(self, name):
757
+ """
758
+ Return the named conceptual attribute of an ONDE object.
759
+ """
760
+
761
+ raise ValueError("ONDEBase does not have attributes")
762
+
763
+ def _has_attr(self, name):
764
+ """
765
+ Return whether the named conceptual attribute of an ONDE object exists.
766
+ """
767
+
768
+ return False
769
+
770
+ def _set_attr(self, name, value):
771
+ """
772
+ Set the named conceptual attribute of an ONDE object.
773
+ """
774
+
775
+ raise ValueError("ONDEBase does not have attributes")
776
+
777
+ def _set_dataset_attr(self, name, value):
778
+ """
779
+ Set the named conceptual attribute of an ONDE object.
780
+ """
781
+
782
+ raise ValueError("ONDEBase does not have attributes")
783
+
784
+ # Items are indexed ONDEBase that we can access
785
+ def _get_item(self,index):
786
+ """Return the indexed data element of an ONDE object.
787
+ """
788
+ raise ValueError("ONDEBase does not have items")
789
+
790
+ def _set_item(self,index,value):
791
+ """Set the indexed data element of an ONDE object.
792
+ """
793
+ raise ValueError("ONDEBase does not have items")
794
+
795
+ def _list_items(self,index,value):
796
+ """List the data element indices of an ONDE object.
797
+ """
798
+ raise ValueError("ONDEBase does not have items")
799
+
800
+
801
+ # Data are lower level objects such as arrays of numbers, integers, strings, etc.
802
+ def _get_data(self,name):
803
+ """Return the named data element of an ONDE object.
804
+ """
805
+ raise ValueError("ONDEBase does not have data")
806
+
807
+ def _set_data(self,name,value):
808
+ """Set the named data element of an ONDE object.
809
+ """
810
+ raise ValueError("ONDEBase does not have data")
811
+
812
+ def _hdf5_write_group(self,onde_file,parent,parent_path,name,onde_fileobj):
813
+ """Write this object to a new hdf5 group
814
+ """
815
+ raise ValueError("ONDEBase does not have content to write")
816
+
817
+ def _hdf5_write_dataset(self,onde_file,parent,parent_path,name,onde_fileobj):
818
+ """Write this object to a new hdf5 dataset
819
+ """
820
+ raise ValueError("ONDEBase does not have content to write")
821
+
822
+ def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
823
+ """Write this object to a new hdf5 attribute
824
+ """
825
+ raise ValueError("ONDEBase does not have content to write")
826
+
827
+
828
+
829
+
830
+ def __copy__(self):
831
+ new = self.__class__(self)
832
+ return new
833
+
834
+ def __hash__(self):
835
+ return id(self) # For now we identify ourself by our pointer. In the future perhaps we might use a content hash.
836
+
837
+ def _freeze(self):
838
+ _frozen = object.__getattribute__(self, "_frozen")
839
+ if _frozen:
840
+ raise RuntimeError("Attempting to freeze an object that is already frozen")
841
+ object.__setattr__(self, "_modification_scopes", None)
842
+ object.__setattr__(self, "_frozen", True)
843
+ pass
844
+
845
+ def _add_referencedby(self, obj_that_references_us):
846
+ _referencedby = object.__getattribute__(self, "_referencedby")
847
+ _referencedby_lock = object.__getattribute__(self, "_referencedby_lock")
848
+ with _referencedby_lock:
849
+ _referencedby.add(obj_that_references_us)
850
+ pass
851
+ pass
852
+
853
+ def _follow_path(self, path):
854
+ if len(path) == 0:
855
+ return self
856
+
857
+ raise AttributeError(f"{self.__class__.__name__} is a leaf node attempting to follow path {str(path)}")
858
+
859
+ def _assign_pathel(self, pathel, value):
860
+ raise AttributeError(f"{self.__class__.__name__} is a leaf node and does not support element assignment of {str(pathel)} to {str(value)}")
861
+
862
+ def _list_edges(self):
863
+ return []
864
+
865
+ def _indices_for_object(self,obj):
866
+ """identify all of the indices for self that reference object obj. Returns a frozenset."""
867
+ return frozenset()
868
+
869
+ def _repr_short(self, onde_class=None):
870
+ return type(self).__name__
871
+
872
+ def _repr(self, indentation, onde_class=None, extra_attrs={}):
873
+ return repr_helper(indentation, type(self).__name__, **extra_attrs)
874
+
875
+ def __repr__(self):
876
+ return object.__getattribute__(self, "_repr")(0)
877
+
878
+ @classmethod
879
+ def new(cls):
880
+ raise RuntimeError("ONDEBase class is not independently instantiatable")
881
+
882
+ pass
883
+
884
+ class ONDEValue(ONDEBase):
885
+ """ONDEValue represents a string, integer, float,
886
+ or other simple value. An ONDEValue
887
+ cannot reference other objects, but can be referenced
888
+ by other objects."""
889
+ value = None # Immutable value
890
+
891
+ def __init__(self, _orig = None, **kwargs):
892
+ if _orig is not None:
893
+ self.value = _orig.value
894
+ pass
895
+ if "value" in kwargs:
896
+ value = copy.copy(kwargs["value"])
897
+ if isinstance(value, numbers.Integral):
898
+ value = int(value)
899
+ pass
900
+ elif isinstance(value, numbers.Real):
901
+ value = float(value)
902
+ pass
903
+ elif isinstance(value, numbers.Complex):
904
+ value = complex(value)
905
+ pass
906
+ elif isinstance(value, str) or isinstance(value,np.str_):
907
+ value = str(value)
908
+ pass
909
+ else:
910
+ raise ValueError(f"ONDEValue: Cannot understand value type {value.__class__.__name__:s}")
911
+ self.value=value
912
+ del kwargs["value"]
913
+ pass
914
+ super().__init__(_orig, **kwargs)
915
+ pass
916
+
917
+ def __getattribute__(self,name):
918
+ if name == "value":
919
+
920
+ _get_data = object.__getattribute__(self, "_get_data")
921
+ return _get_data(name)
922
+
923
+ return super().__getattribute__(name)
924
+
925
+ def __setattr__(self,name,value):
926
+ if name == "value":
927
+ self._set_data(name,value)
928
+ pass
929
+ else:
930
+
931
+ super().__setattr__(name,value)
932
+ pass
933
+ pass
934
+
935
+
936
+ def _get_data(self, name):
937
+ """
938
+ Return the named conceptual data of an ONDE object.
939
+ """
940
+
941
+ if name == "value":
942
+ return object.__getattribute__(self, name)
943
+
944
+ raise ValueError("ONDEValue does not have data other than \"value\"")
945
+
946
+ def _set_data(self, name, value):
947
+ """
948
+ Set the named conceptual data of an ONDE object.
949
+ """
950
+
951
+ if name == "value":
952
+ return object.__setattr__(self, name, value)
953
+
954
+ raise ValueError("ONDEValue does not have data other than \"value\"")
955
+
956
+ def _repr_short(self, onde_class=None):
957
+ return f"ONDEValue({str(object.__getattribute__(self, 'value')):s})"
958
+
959
+ def _repr(self, indentation, onde_class=None, extra_attrs={}):
960
+ # return f"ONDEValue ID={id(self):x} of type {type(object.__getattribute__(self, 'value')).__name__:s} containing value: {str(object.__getattribute__(self, 'value')):s}"
961
+ return repr_helper(indentation, "ONDEValue",
962
+ ID=f"{id(self):x}",
963
+ type=type(object.__getattribute__(self, 'value')).__name__,
964
+ value=str(object.__getattribute__(self, 'value')),
965
+ **extra_attrs)
966
+
967
+
968
+ def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
969
+ """Write this object to a new hdf5 attribute
970
+ """
971
+ parent.attrs[name] = self.value
972
+ pass
973
+
974
+
975
+
976
+
977
+ @classmethod
978
+ def new(cls, value = None, **kwargs):
979
+ return cls(None, value = value, **kwargs)
980
+
981
+ pass
982
+
983
+ class ONDEArray(ONDEBase):
984
+ """ONDEArray represents an array. An ONDEArray
985
+ cannot reference other objects, but can be referenced
986
+ by other objects. For an array of references, see
987
+ ONDEReferenceArray.
988
+
989
+ An ONDEArray can be stored either as an HDF5
990
+ attribute or an HDF5 dataset depending on the
991
+ status in the parent object and whether the attribute of the parent is set to be stored as a dataset
992
+ """
993
+ value = None # numpy array
994
+ #store_as_dataset = None # boolean; store as an HDF5 dataset if True, otherwise and an HDF5 attribute
995
+
996
+ def __init__(self, _orig = None, **kwargs):
997
+ #store_as_dataset = False
998
+ value = None
999
+ shape = None
1000
+ dtype = None
1001
+
1002
+ if _orig is not None:
1003
+ value = _orig.value
1004
+ #store_as_dataset = _orig.store_as_dataset
1005
+ pass
1006
+
1007
+
1008
+ #if "store_as_dataset" in kwargs:
1009
+ # store_as_dataset = bool(kwargs["store_as_dataset"])
1010
+ # del kwargs["store_as_dataset"]
1011
+ # pass
1012
+ if "shape" in kwargs:
1013
+ if kwargs["shape"] is not None:
1014
+ shape = tuple(kwargs["shape"])
1015
+ pass
1016
+ del kwargs["shape"]
1017
+ pass
1018
+ if "dtype" in kwargs:
1019
+ dtype = kwargs["dtype"]
1020
+ del kwargs["dtype"]
1021
+ pass
1022
+
1023
+ if "value" in kwargs:
1024
+ if kwargs["value"] is not None:
1025
+ value = copy.copy(kwargs["value"])
1026
+ if isinstance(value, collections.abc.Sequence):
1027
+ dtype = None
1028
+ if len(value) > 0 and isinstance(value[0],str):
1029
+ # For collection of strings, use h5py string datatype for numpy
1030
+ dtype = h5py.string_dtype()
1031
+ pass
1032
+ value = np.array(value,dtype=dtype)
1033
+ pass
1034
+ elif isinstance(value, np.ndarray):
1035
+ # value.flags.writeable = False
1036
+ pass
1037
+ else:
1038
+ raise ValueError(f"ONDEArray: Cannot understand value type {value.__class__.__name__:s}")
1039
+ pass
1040
+ del kwargs["value"]
1041
+ pass
1042
+ if value is None:
1043
+ value = np.zeros(shape, dtype = dtype)
1044
+ pass
1045
+
1046
+ #self.store_as_dataset = store_as_dataset
1047
+ self.value = value
1048
+ super().__init__(_orig, **kwargs)
1049
+ pass
1050
+
1051
+ def __getattribute__(self,name):
1052
+ if name == "value" or name == "shape":# or name == "store_as_dataset":
1053
+
1054
+ _get_data = object.__getattribute__(self, "_get_data")
1055
+ return _get_data(name)
1056
+
1057
+ return super().__getattribute__(name)
1058
+
1059
+ def __setattr__(self,name,value):
1060
+ if name == "value": # or name == "store_as_dataset":
1061
+ self._set_data(name,value)
1062
+ pass
1063
+ else:
1064
+
1065
+ super().__setattr__(name,value)
1066
+ pass
1067
+ pass
1068
+
1069
+ def _get_item(self,index):
1070
+ return self.value[index]
1071
+
1072
+ def __getitem__(self,index):
1073
+ return self._get_item(index)
1074
+
1075
+ def _set_item(self,index,el_value):
1076
+ if self._frozen:
1077
+ raise RuntimeError("Attempting to modify an object that is already frozen")
1078
+
1079
+ self.value[index] = el_value
1080
+ pass
1081
+
1082
+ def __setitem__(self,index,el_value):
1083
+ self._set_item(index,el_value)
1084
+ pass
1085
+
1086
+ def _get_data(self, name):
1087
+ """
1088
+ Return the named conceptual attribute of an ONDE object.
1089
+ """
1090
+
1091
+ if name == "value": # or name == "store_as_dataset":
1092
+ return object.__getattribute__(self, name)
1093
+ elif name == "shape":
1094
+ return object.__getattribute__(self, "value").shape
1095
+
1096
+ raise ValueError("ONDEArray does not have attributes other than \"value\" and \"store_as_dataset\"")
1097
+
1098
+ def _set_data(self, name, value):
1099
+ """
1100
+ Set the named conceptual attribute of an ONDE object.
1101
+ """
1102
+ if self._frozen:
1103
+ raise RuntimeError("Attempting to modify an object that is already frozen")
1104
+
1105
+ if name == "value": # or name == "store_as_dataset":
1106
+ return object.__setattr__(self, name, value)
1107
+
1108
+ raise ValueError("ONDEArray does not have attributes other than \"value\"")
1109
+
1110
+ def _freeze(self):
1111
+ self.value.flags.writeable = False
1112
+ super()._freeze()
1113
+ pass
1114
+
1115
+ def _repr_short(self, onde_class=None):
1116
+ if len(self.shape)==1 and self.shape[0]<8:
1117
+ return f"ONDEArray({str(list(self.value))})"
1118
+ return f"ONDEArray(shape={str(self.shape):s},dtype={str(self.value.dtype):s})"
1119
+
1120
+ def _repr(self, indentation, onde_class=None, extra_attrs={}):
1121
+ return repr_helper(indentation, "ONDEArray",
1122
+ ID=f"{id(self):x}",
1123
+ type=str(object.__getattribute__(self, 'value').dtype),
1124
+ shape=str(object.__getattribute__(self, 'value').shape),
1125
+ **extra_attrs)
1126
+
1127
+ def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
1128
+ """Write this object to a new hdf5 attribute
1129
+ """
1130
+ parent.attrs[name] = self.value
1131
+ pass
1132
+
1133
+ def _hdf5_write_dataset(self,onde_file,parent,parent_path,name,onde_fileobj):
1134
+ """Write this object to a new hdf5 dataset
1135
+ """
1136
+ ds = parent.create_dataset(name,data = self.value)
1137
+
1138
+ pass
1139
+
1140
+ @classmethod
1141
+ def new(cls, value = None, **kwargs):
1142
+ return cls(None, value = value, **kwargs)
1143
+
1144
+ pass
1145
+
1146
+
1147
+ class ONDEReferenceArray(ONDEBase):
1148
+ """Represents an array of references to other ONDEObjects.
1149
+
1150
+ An ONDEReferenceArray can be stored as an HDF5
1151
+ attribute"""
1152
+ refs = None # TwoWayArray of ONDEBase references
1153
+ #store_as_dataset = None # True to store as an HDF5 dataset, False to store as an HDF5 attribute
1154
+
1155
+
1156
+ def __init__(self,_orig, **kwargs):
1157
+ """Private constructor for internal use only.
1158
+ Use .new() classmethod or copy.copy()
1159
+ """
1160
+ #store_as_dataset = False
1161
+ refs = None
1162
+ if _orig is not None:
1163
+ refs = _orig.refs
1164
+ #store_as_dataset = _orig.store_as_dataset
1165
+ pass
1166
+
1167
+
1168
+ #if "store_as_dataset" in kwargs:
1169
+ # store_as_dataset = bool(kwargs["store_as_dataset"])
1170
+ # del kwargs["store_as_dataset"]
1171
+ # pass
1172
+
1173
+ shape = ()
1174
+
1175
+ if "shape" in kwargs:
1176
+ if kwargs["shape"] is not None:
1177
+ shape = tuple(kwargs["shape"])
1178
+ pass
1179
+ del kwargs["shape"]
1180
+ pass
1181
+
1182
+ if "refs" in kwargs:
1183
+ if kwargs["refs"] is not None:
1184
+ refs = TwoWayArray(byindex=kwargs["refs"])
1185
+ shape = kwargs["refs"].shape
1186
+ pass
1187
+ del kwargs["refs"]
1188
+ pass
1189
+
1190
+ if refs is None:
1191
+ refs = TwoWayArray(shape=shape)
1192
+ pass
1193
+
1194
+ #self.store_as_dataset = store_as_dataset
1195
+ object.__setattr__(self, "refs", refs)
1196
+ super().__init__(_orig,**kwargs)
1197
+ pass
1198
+
1199
+ def __getitem__(self,index):
1200
+ return self._get_item(index)
1201
+
1202
+ def __setitem__(self,index,ref):
1203
+ self._set_item(index,ref)
1204
+ pass
1205
+
1206
+ def _get_data(self, name):
1207
+ """
1208
+ Return the named conceptual data of an ONDE object.
1209
+ """
1210
+
1211
+ if name == "refs": # or name == "store_as_dataset":
1212
+ return object.__getattribute__(self, name)
1213
+
1214
+ raise ValueError("ONDEReferenceArray does not have attributes other than \"refs\"")
1215
+
1216
+ def _set_data(self, name, value):
1217
+ """
1218
+ Set the named conceptual data of an ONDE object.
1219
+ """
1220
+
1221
+ if name == "refs": # or name == "store_as_dataset":
1222
+ return object.__setattr__(self, name, value)
1223
+
1224
+ raise ValueError("ONDEReferenceArray does not have attributes other than \"refs\"")
1225
+
1226
+ def _get_item(self,index):
1227
+ return self.refs[index]
1228
+
1229
+ def _set_item(self,index,ref):
1230
+ if self._frozen:
1231
+ raise RuntimeError("Attempting to modify an object that is already frozen")
1232
+
1233
+ if isinstance(ref, ONDEClassInstanceWrapper) or isinstance(ref, ONDEProxy):
1234
+ ref = ref._get_obj()
1235
+ pass
1236
+
1237
+ if not isinstance(ref,ONDEBase):
1238
+ raise ValueError(f"Attempting to assign index {str(index)} of an ONDEReferenceArray to an object of class {ref.__class__.__name__} that is not an ONDEBase reference")
1239
+
1240
+ self.refs[index] = ref
1241
+ pass
1242
+
1243
+ def __getattribute__(self,name):
1244
+ if name == "refs":# or name == "store_as_dataset":
1245
+ _get_data = object.__getattribute__(self, "_get_data")
1246
+ return _get_data(name)
1247
+
1248
+ return super().__getattribute__(name)
1249
+
1250
+ def __setattr__(self,name,value):
1251
+ if name == "refs": # or name == "store_as_dataset":
1252
+ self._set_data(name,value)
1253
+ pass
1254
+ else:
1255
+
1256
+ super().__setattr__(name,value)
1257
+ pass
1258
+ pass
1259
+
1260
+ def _freeze(self):
1261
+ if self._frozen:
1262
+ raise RuntimeError("Attempting to freeze an object that is already frozen")
1263
+ self.refs._freeze()
1264
+
1265
+ nditer = iter(self.refs)
1266
+ for arrayref in nditer:
1267
+ obj = arrayref[()]
1268
+ if obj is not None:
1269
+ obj._add_referencedby(self)
1270
+ if not obj._frozen:
1271
+ obj._freeze()
1272
+ pass
1273
+ pass
1274
+ pass
1275
+ super()._freeze()
1276
+ pass
1277
+
1278
+ def _follow_path(self, path):
1279
+ if len(path) == 0:
1280
+ return self
1281
+
1282
+ path_entry = path[0]
1283
+
1284
+ if isinstance(path_entry, numbers.Integral) or isinstance(path_entry, collections.abc.Sequence):
1285
+ if self.refs[path_entry] is not None:
1286
+ return self.refs[path_entry]._follow_path(ONDEPath(path[1:]))
1287
+ return None
1288
+ else:
1289
+ raise AttributeError(f"Cannot index {self.__class__.__name__} by {path_entry}")
1290
+
1291
+ pass
1292
+
1293
+ def _assign_pathel(self, pathel, value):
1294
+ if self._frozen:
1295
+ raise RuntimeError(f"Cannot assign {str(value)} to {str(pathel)} element of frozen object")
1296
+ self.refs[pathel]=value
1297
+ pass
1298
+
1299
+ def _list_edges(self):
1300
+ nditer = iter(self.refs) #np.nditer object
1301
+ sz = nditer.itersize
1302
+
1303
+ edgelist= []
1304
+ for cnt in range(sz):
1305
+ next(nditer)
1306
+
1307
+ if self.refs[nditer.multi_index] is not None:
1308
+ edgelist.append(nditer.multi_index)
1309
+ pass
1310
+ pass
1311
+ return edgelist
1312
+
1313
+ def _indices_for_object(self,obj):
1314
+ """identify all of the indices for self that reference object obj. Returns a frozenset."""
1315
+ return self.refs(obj) # __call__ method does reverse lookup to return a set of indices
1316
+
1317
+ def _repr_short(self, onde_class=None):
1318
+ return f"ONDEReferenceArray(shape={str(object.__getattribute__(self, 'refs')._shape):s})"
1319
+
1320
+ def _repr(self, indentation, onde_class=None, extra_attrs={}):
1321
+ return repr_helper(indentation, "ONDEReferenceArray",
1322
+ ID=f"{id(self):x}",
1323
+ shape=str(object.__getattribute__(self, 'refs')._shape),
1324
+ **extra_attrs)
1325
+
1326
+ def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
1327
+ """Write this object to a new hdf5 attribute
1328
+ """
1329
+ ref_dtype = h5py.special_dtype(ref = h5py.Reference)
1330
+
1331
+ byindex = self.refs.byindex
1332
+ h5_byindex = np.empty(byindex.shape,dtype = ref_dtype)
1333
+ nditer = np.nditer(byindex, flags = ("multi_index","refs_ok"))
1334
+
1335
+ for objarray in nditer:
1336
+ obj = objarray[()]
1337
+ if obj is not None:
1338
+ # !!!*** This may need to be accelerated by caching the hdf5 object references per the various discussions on h5py indexing performance with Paul Wilcox, early 2026. https://github.com/COFREND/ONDE-format/issues/24
1339
+ h5_byindex[nditer.multi_index] = onde_file.fh[onde_file.file_object_dict[obj].hdf5_path].ref
1340
+ pass
1341
+ pass
1342
+
1343
+ parent.attrs[name] = h5_byindex
1344
+ pass
1345
+
1346
+ def _hdf5_write_dataset(self,onde_file,parent,parent_path,name,onde_fileobj):
1347
+ """Write this object to a new hdf5 dataset
1348
+ """
1349
+ ref_dtype = h5py.special_dtype(ref = h5py.Reference)
1350
+
1351
+ byindex = self.refs.byindex
1352
+ h5_byindex = np.zeros(byindex.shape,dtype = ref_dtype)
1353
+ nditer = np.nditer(byindex, flags = ("multi_index","refs_ok"))
1354
+
1355
+ for objarray in nditer:
1356
+ obj = objarray[()]
1357
+ if obj is not None:
1358
+ # !!!*** This may need to be accelerated by caching the hdf5 object references per the various discussions on h5py indexing performance with Paul Wilcox, early 2026. https://github.com/COFREND/ONDE-format/issues/24
1359
+ h5_byindex[nditer.multi_index] = onde_file.fh[onde_file.file_object_dict[obj].hdf5_path].ref
1360
+ pass
1361
+ pass
1362
+ ds = parent.create_dataset(name,data=h5_byindex)
1363
+ pass
1364
+
1365
+ @classmethod
1366
+ def new(cls, file_or_graph, refs = None, shape = None, **kwargs):
1367
+ graph = None
1368
+
1369
+ if isinstance(file_or_graph, ONDEFileGraph):
1370
+ graph = file_or_graph
1371
+ pass
1372
+ elif file_or_graph is not None:
1373
+ graph = file_or_graph.graph
1374
+ pass
1375
+
1376
+ newobj= cls(None, refs = refs, shape = shape, **kwargs)
1377
+
1378
+ if file_or_graph is not None:
1379
+ return ONDEClassInstanceWrapper(_graph=graph,_obj=newobj)
1380
+ return newobj
1381
+
1382
+ @classmethod
1383
+ def load_from_hdf5(cls,onde_file,fileobjs_by_h5path,h5_fh,np_h5ref_array):
1384
+ refs = np.empty(np_h5ref_array.shape,dtype = "O")
1385
+ nditer = np.nditer(np_h5ref_array, flags = ("multi_index","refs_ok"))
1386
+
1387
+ for refarray in nditer:
1388
+ ref = refarray[()]
1389
+ if ref is not None and not "(null)" in str(ref):
1390
+ refs[nditer.multi_index] = ONDEObject.load_from_hdf5(onde_file,fileobjs_by_h5path,h5_fh,h5_fh[ref])
1391
+ pass
1392
+ pass
1393
+ instance = cls(None,refs = refs, shape = np_h5ref_array.shape)
1394
+ return instance
1395
+
1396
+ pass
1397
+
1398
+
1399
+ class ONDEObject(ONDEBase):
1400
+ """ONDEObject represents a (non-leaf) node in the graph
1401
+ that can point at other objects."""
1402
+ #ONDE_TYPE = None # List of classes starting with base class
1403
+ _ONDE_attrs = None # TwoWayDictionary by name of attributes that should be ONDEBase (or subclass) objects).
1404
+ _ONDE_dataset_attrs = None # Sub-set of ONDE_attrs that should use hdf5 dataset (as opposed to hdf5 attribute) storage.
1405
+
1406
+ def __init__(self, _orig, **kwargs):
1407
+ """Private constructor for internal use only.
1408
+ Use .new() classmethod or copy.copy()
1409
+ """
1410
+
1411
+ #ONDE_TYPE = None
1412
+ #if _orig is not None:
1413
+ # ONDE_TYPE = object.__getattribute__(_orig, "ONDE_TYPE")
1414
+ # pass
1415
+ #if "ONDE_TYPE" in kwargs:
1416
+ # ONDE_TYPE = kwargs["ONDE_TYPE"]
1417
+ # del kwargs["ONDE_TYPE"]
1418
+ # pass
1419
+
1420
+ _ONDE_attrs = None
1421
+ _ONDE_dataset_attrs = None
1422
+ if _orig is not None:
1423
+ _ONDE_attrs = object.__getattribute__(_orig, "_ONDE_attrs")
1424
+ _ONDE_dataset_attrs = object.__getattribute__(_orig,"_ONDE_dataset_attrs")
1425
+ pass
1426
+ if "_ONDE_attrs" in kwargs:
1427
+ #if _ONDE_attrs is not None:
1428
+ # _ONDE_attrs.update(kwargs["_ONDE_attrs"])
1429
+ # pass
1430
+ #else:
1431
+ _ONDE_attrs = kwargs["_ONDE_attrs"]
1432
+ # pass
1433
+ del kwargs["_ONDE_attrs"]
1434
+ pass
1435
+
1436
+ if "_ONDE_dataset_attrs" in kwargs:
1437
+ _ONDE_dataset_attrs = kwargs["_ONDE_dataset_attrs"]
1438
+ del kwargs["_ONDE_dataset_attrs"]
1439
+ pass
1440
+
1441
+ #object.__setattr__(self, "ONDE_TYPE", tuple(ONDE_TYPE))
1442
+ ONDE_attrs = TwoWayDictionary(_ONDE_attrs)
1443
+ if not "ONDE:TYPE" in ONDE_attrs:
1444
+ ONDE_attrs["ONDE:TYPE"] = ()
1445
+ pass
1446
+
1447
+ object.__setattr__(self, "_ONDE_attrs", ONDE_attrs)
1448
+
1449
+ if _ONDE_dataset_attrs is None :
1450
+ _ONDE_dataset_attrs = set()
1451
+ pass
1452
+ ONDE_dataset_attrs = set(_ONDE_dataset_attrs)
1453
+ object.__setattr__(self,"_ONDE_dataset_attrs",ONDE_dataset_attrs)
1454
+
1455
+ base_kwargs = {}
1456
+ for kwarg in kwargs:
1457
+ if kwarg.startswith("_"):
1458
+ base_kwargs[kwarg] = kwargs[kwarg]
1459
+ pass
1460
+ else:
1461
+ ONDE_attrs._set_attr(kwarg,kwargs[kwarg])
1462
+ pass
1463
+ pass
1464
+ super().__init__(_orig, **base_kwargs)
1465
+ pass
1466
+
1467
+ def __getattribute__(self, name):
1468
+ if name.startswith("_"):
1469
+ if name in {"_freeze", "_frozen", "_add_referencedby", "__class__", "__dir__", "_get_attr","_get_attr_dataset_storage", "_set_attr","_set_dataset_attr","_follow_path","_modification_scopes","_indices_for_object","_assign_pathel","_list_edges","_ONDE_attrs","_ONDE_dataset_attrs", "_has_attr","_repr","_repr_short","_hdf5_write_group","_list_attrs","_hdf5_write_dataset","_hdf5_write_attribute","_file_realizations","_file_realizations_lock","_onde_class_name","_get_item","_set_item","_list_items","_has_item"}:
1470
+ return object.__getattribute__(self, name)
1471
+ raise IndexError("ONDEObject: Attributes may not have leading underscores")
1472
+
1473
+ _get_attr = object.__getattribute__(self, "_get_attr")
1474
+
1475
+ return _get_attr(name)
1476
+
1477
+ def __setattr__(self, name, value):
1478
+ _frozen = object.__getattribute__(self, "_frozen")
1479
+ if _frozen:
1480
+ raise RuntimeError("Attempting to modify an object that is already frozen")
1481
+
1482
+ _set_attr = object.__getattribute__(self, "_set_attr")
1483
+ _set_attr(name, value)
1484
+
1485
+ pass
1486
+
1487
+ def _get_item(self,name):
1488
+ return self._get_attr(name)
1489
+
1490
+ def _set_item(self,name,value):
1491
+ return self._set_attr(name,value)
1492
+
1493
+ def __getitem__(self,name):
1494
+ return self._get_attr(name)
1495
+
1496
+ def __setitem__(self,name,value):
1497
+ self._set_attr(name,value)
1498
+ pass
1499
+
1500
+ def _get_attr(self, name):
1501
+ """
1502
+ Return the named conceptual attribute of an ONDE object.
1503
+ """
1504
+ if not isinstance(name,str):
1505
+ raise IndexError(f"Indexes must be strings")
1506
+ if name.startswith("_"):
1507
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
1508
+
1509
+ _ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
1510
+
1511
+ return _ONDE_attrs._get_attr(name)
1512
+
1513
+ def _get_attr_dataset_storage(self, name):
1514
+ """
1515
+ Return whether the named conceptual attribute of an ONDE object uses hdf5 dataset storage.
1516
+ """
1517
+
1518
+ if name.startswith("_"):
1519
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
1520
+
1521
+ _ONDE_dataset_attrs = object.__getattribute__(self, "_ONDE_dataset_attrs")
1522
+
1523
+ return name in _ONDE_dataset_attrs
1524
+
1525
+ def _has_attr(self, name):
1526
+ """
1527
+ Return whether the named conceptual attribute of an ONDE object exists.
1528
+ """
1529
+ _ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
1530
+ if name in _ONDE_attrs:
1531
+ return True
1532
+ return False
1533
+ def _has_item(self,name):
1534
+ return self._has_attr(name)
1535
+
1536
+ def __contains__(self,name):
1537
+ return self._has_attr(name)
1538
+
1539
+ def _set_attr(self, name, value):
1540
+ """
1541
+ Set the named conceptual attribute of an ONDE object. Use _set_dataset_attr() instead if the attribute should have hdf5 dataset storage.
1542
+ """
1543
+ _frozen = object.__getattribute__(self, "_frozen")
1544
+ if _frozen:
1545
+ raise RuntimeError("Attempting to modify an object that is already frozen")
1546
+
1547
+ if not isinstance(self, ONDEFileGraphSnapshot) and not ":" in name:
1548
+ raise ValueError(f"ONDEObject: Attempting to set field \"{name:s}\" with no colon-delimited defining class. Usually this means there is a typo in the field name, or that the field is not defined in your class definitions for some reason. If you are trying to add a custom field, be sure to use a custom namespace and accessory class or derived class. You can use bracket notation (e.g. object[\"CLASSNAME:FIELD\"]=...) to assign a field name that contains a colon manually if you do not have a formal class definition loaded")
1549
+
1550
+ if name.startswith("_"):
1551
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
1552
+
1553
+ _ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
1554
+ _ONDE_dataset_attrs = object.__getattribute__(self, "_ONDE_dataset_attrs")
1555
+
1556
+ if isinstance(value, ONDEClassInstanceWrapper) or isinstance(value, ONDEProxy):
1557
+ value = value._get_obj()
1558
+ pass
1559
+
1560
+ _ONDE_dataset_attrs.discard(name)
1561
+ return _ONDE_attrs._set_attr(name, value)
1562
+
1563
+ def _set_dataset_attr(self, name, value):
1564
+ """
1565
+ Set the named conceptual attribute of an ONDE object with dataset storage. Use _set_attr() instead if the attribute should have hdf5 attribute storage.
1566
+ """
1567
+ _frozen = object.__getattribute__(self, "_frozen")
1568
+ if _frozen:
1569
+ raise RuntimeError("Attempting to modify an object that is already frozen")
1570
+
1571
+ if not isinstance(self, ONDEFileGraphSnapshot) and not ":" in name:
1572
+ raise ValueError(f"ONDEObject: Attempting to set field \"{name:s}\" with no colon-delimited defining class. Usually this means there is a typo in the field name, or that the field is not defined in your class definitions for some reason. If you are trying to add a custom field, be sure to use a custom namespace and accessory class or derived class. You can use bracket notation (e.g. object[\"CLASSNAME:FIELD\"]=...) to assign a field name that contains a colon manually if you do not have a formal class definition loaded")
1573
+
1574
+ if name.startswith("_"):
1575
+ raise ValueError(f"Attributes such as \"{name:s}\" with leading underscores not allowed")
1576
+
1577
+ _ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
1578
+ _ONDE_dataset_attrs = object.__getattribute__(self, "_ONDE_dataset_attrs")
1579
+
1580
+ if isinstance(value, ONDEClassInstanceWrapper) or isinstance(value, ONDEProxy):
1581
+ value = value._get_obj()
1582
+ pass
1583
+
1584
+ _ONDE_dataset_attrs.add(name)
1585
+ return _ONDE_attrs._set_attr(name, value)
1586
+
1587
+ def _list_attrs(self):
1588
+ _ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
1589
+ return [attrname for attrname in _ONDE_attrs]
1590
+
1591
+ def __dir__(self):
1592
+ _ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
1593
+ return ["_freeze","_frozen"] + [attrname for attrname in _ONDE_attrs]
1594
+
1595
+ def _freeze(self):
1596
+ _frozen = object.__getattribute__(self, "_frozen")
1597
+ if _frozen:
1598
+ raise RuntimeError("Attempting to freeze an object that is already frozen")
1599
+ _ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
1600
+ _ONDE_attrs._freeze()
1601
+ for attrname in _ONDE_attrs:
1602
+ obj = getattr(_ONDE_attrs, attrname)
1603
+ obj._add_referencedby(self)
1604
+ if obj is not None and not obj._frozen:
1605
+ obj._freeze()
1606
+ pass
1607
+ pass
1608
+
1609
+ super()._freeze()
1610
+ pass
1611
+
1612
+ def _follow_path(self, path):
1613
+ if len(path) == 0:
1614
+ return self
1615
+
1616
+ path_entry = path[0]
1617
+
1618
+ if isinstance(path_entry, str):
1619
+ #return self._get_attr(path_entry)._follow_path(ONDEPath(path[1:]))
1620
+ # ONDEFileGraphSnapshot (our subclass) uses items not attributes. So here we explicitly use our own _get_attr() to expand out the path entry
1621
+ return ONDEObject._get_attr(self,path_entry)._follow_path(ONDEPath(path[1:]))
1622
+ else:
1623
+ raise AttributeError(f"Cannot index {self.__class__.__name__} by {path_entry}")
1624
+
1625
+ pass
1626
+
1627
+ def _assign_pathel(self, pathel, value):
1628
+ if self._frozen:
1629
+ raise RuntimeError(f"Cannot assign {str(value)} to {str(pathel)} element of frozen object")
1630
+ # ONDEFileGraphSnapshot (our subclass) uses items not attributes. So here we explicitly use our own _set_attr() to expand out the path entry
1631
+ #self._set_attr(pathel, value)
1632
+ ONDEObject._set_attr(self,pathel, value)
1633
+ pass
1634
+
1635
+ def _list_edges(self):
1636
+ _ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
1637
+ edgelist= list(_ONDE_attrs._keys())
1638
+ return edgelist
1639
+
1640
+ def _indices_for_object(self,obj):
1641
+ """identify all of the indices for self that reference object obj. Returns a frozenset."""
1642
+ _ONDE_attrs = object.__getattribute__(self, "_ONDE_attrs")
1643
+ return _ONDE_attrs(obj) # __call__ method does reverse lookup to return a set of indices
1644
+
1645
+ def _repr_short(self, onde_class=None):
1646
+ myclass=type(self).__name__
1647
+ if onde_class is not None:
1648
+ myclass=f"{onde_class:s}({myclass:s})"
1649
+ pass
1650
+
1651
+ return f"{myclass:s}(ID={id(self):x})"
1652
+
1653
+ def _repr(self, indentation, onde_class=None, extra_attrs={}):
1654
+ myclass=type(self).__name__
1655
+ if onde_class is not None:
1656
+ myclass=f"{onde_class:s}({myclass:s})"
1657
+ pass
1658
+
1659
+ new_extra_attrs=collections.OrderedDict()
1660
+ if onde_class is None:
1661
+ _ONDE_attrs=object.__getattribute__(self, '_ONDE_attrs')
1662
+ _ONDE_dataset_attrs = object.__getattribute__(self, "_ONDE_dataset_attrs")
1663
+
1664
+ keys=list(_ONDE_attrs._keys())
1665
+ shortkeys=keys
1666
+
1667
+ if isinstance(self,ONDEFileGraphSnapshot):
1668
+ shortkeys=self.shortkeys()
1669
+
1670
+ pass
1671
+
1672
+ for keyidx in range(len(keys)):
1673
+ attrname=keys[keyidx]
1674
+ attrnameshort=shortkeys[keyidx]
1675
+ storage_suffix = ""
1676
+ if attrname in _ONDE_dataset_attrs:
1677
+ storage_suffix = " (hdf5 dataset storage)"
1678
+ pass
1679
+ attrval=_ONDE_attrs[attrname]
1680
+ if attrval is not None:
1681
+ new_extra_attrs[attrnameshort]=object.__getattribute__(attrval, "_repr_short")() + storage_suffix
1682
+ pass
1683
+ else:
1684
+ new_extra_attrs[attrnameshort]="None" + storage_suffix
1685
+ pass
1686
+
1687
+ pass
1688
+ pass
1689
+ new_extra_attrs.update(extra_attrs)
1690
+ return repr_helper(indentation, myclass,
1691
+ ID=f"{id(self):x}",
1692
+ **new_extra_attrs)
1693
+
1694
+ def _onde_class_name(self):
1695
+ classname = "(None)"
1696
+ if "ONDE:TYPE" in self._ONDE_attrs:
1697
+ if isinstance(self._ONDE_attrs["ONDE:TYPE"],ONDEArray):
1698
+ if len(self._ONDE_attrs["ONDE:TYPE"].value) > 0:
1699
+ classname = self._ONDE_attrs["ONDE:TYPE"].value[0]
1700
+ pass
1701
+ pass
1702
+ pass
1703
+ return classname
1704
+
1705
+ def _hdf5_write_group(self,onde_file,parent,parent_path,name,onde_fileobj):
1706
+ """Write this object to a new hdf5 group
1707
+ """
1708
+ gr = parent.create_group(name)
1709
+ for attrname in self._ONDE_attrs:
1710
+ value = self._ONDE_attrs[attrname]
1711
+ if attrname in self._ONDE_dataset_attrs:
1712
+ if isinstance(value,ONDEObject) or isinstance(value,ONDEValue):
1713
+ raise ValueError(f"Error writing ONDE object of class {self._onde_class_name()}: Attribute {attrname} is marked for dataset storage, but is an {type(value).__name__} not an ONDEArray or ONDEReferenceArray")
1714
+ value._hdf5_write_dataset(onde_file,gr,gr.name,attrname,None)
1715
+ pass
1716
+ else:
1717
+ value._hdf5_write_attribute(onde_file,gr,gr.name,attrname,None)
1718
+ pass
1719
+ pass
1720
+
1721
+ pass
1722
+
1723
+ def _hdf5_write_attribute(self,onde_file,parent,parent_path,name,onde_fileobj):
1724
+ """Write a reference to this object to a new hdf5 attribute
1725
+ """
1726
+ parent.attrs[name] = onde_file.fh[onde_file.file_object_dict[self].hdf5_path].ref
1727
+ pass
1728
+
1729
+
1730
+ @classmethod
1731
+ def new(cls, file_or_graph,onde_classname, ONDE_TYPE = None, _ONDE_attrs = None,_ONDE_dataset_attrs = None, **kwargs):
1732
+ """Main constructor to call. If file_or_graph is provided, then the ONDEObject will be created but in fact the returned python object will be instead an ONDEClassInstanceWrapper with the context of that file or graph and the corresponding class definitions."""
1733
+
1734
+ graph = None
1735
+
1736
+ if isinstance(file_or_graph, ONDEFileGraph):
1737
+ graph = file_or_graph
1738
+ pass
1739
+ elif file_or_graph is not None:
1740
+ graph = file_or_graph.graph
1741
+ pass
1742
+
1743
+ if onde_classname is not None and ONDE_TYPE is None:
1744
+ class_derivation = [ onde_classname ]
1745
+ if graph is not None and graph.class_defs is not None:
1746
+ class_defs = graph.class_defs
1747
+ if onde_classname in class_defs.classes:
1748
+ onde_class = class_defs.classes[onde_classname]
1749
+ class_derivation = onde_class.class_derivation
1750
+ pass
1751
+ pass
1752
+
1753
+ ONDE_TYPE = ONDEArray.new(class_derivation, _frozen = True)
1754
+ pass
1755
+
1756
+ constructargs = {}
1757
+ if ONDE_TYPE is not None:
1758
+ constructargs["ONDE:TYPE"] = ONDE_TYPE
1759
+ pass
1760
+ if _ONDE_attrs is not None:
1761
+ constructargs["_ONDE_attrs"] = _ONDE_attrs
1762
+ pass
1763
+ if _ONDE_dataset_attrs is not None:
1764
+ constructargs["_ONDE_dataset_attrs"] = _ONDE_dataset_attrs
1765
+ pass
1766
+
1767
+ constructargs.update(kwargs)
1768
+ newobj = cls(None, **constructargs)
1769
+ #for attrname in kwargs:
1770
+ # setattr(newobj, attrname, kwargs[attrname])
1771
+ # pass
1772
+
1773
+ if file_or_graph is not None:
1774
+ return ONDEClassInstanceWrapper(_graph=graph,_obj=newobj)
1775
+ return newobj
1776
+
1777
+ @classmethod
1778
+ def load_from_hdf5(cls,onde_file,fileobjs_by_h5path,h5_fh,h5_group):
1779
+ _ONDE_attrs = collections.OrderedDict()
1780
+ _ONDE_dataset_attrs = set()
1781
+
1782
+ if h5_group.name in fileobjs_by_h5path:
1783
+ return fileobjs_by_h5path[h5_group.name].onde_instance
1784
+
1785
+ for attrname in h5_group.attrs:
1786
+ if not ":" in attrname:
1787
+ raise ValueError(f"Error loading fields from hdf5 group {h5_group.name:s} in file {onde_file.h5path:s}: field name {attrname:s} does not contain a colon separator between the defining class and the name.")
1788
+ h5_attr = h5_group.attrs[attrname]
1789
+ if isinstance(h5_attr,str) or isinstance(h5_attr,int) or isinstance(h5_attr,float):
1790
+ attr_obj = ONDEValue.new(h5_attr)
1791
+ _ONDE_attrs[attrname] = attr_obj
1792
+ pass
1793
+
1794
+ elif isinstance(h5_attr,np.ndarray):
1795
+ string_info = h5py.check_string_dtype(h5_attr.dtype)
1796
+ if string_info is not None:
1797
+ # array of strings
1798
+ pass
1799
+
1800
+ # Check if hdf5 reference
1801
+ ref_type = h5py.check_dtype(ref = h5_attr.dtype)
1802
+ if ref_type is not None:
1803
+ # array of references
1804
+
1805
+ attr_obj = ONDEReferenceArray.load_from_hdf5(onde_file,fileobjs_by_h5path,h5_fh,h5_attr)
1806
+ _ONDE_attrs[attrname] = attr_obj
1807
+ pass
1808
+ else:
1809
+ # Array of values
1810
+ attr_obj = ONDEArray.new(value = h5_attr)
1811
+ _ONDE_attrs[attrname] = attr_obj
1812
+ pass
1813
+ pass
1814
+ elif isinstance(h5_attr,h5py.Reference):
1815
+ attr_obj = ONDEObject.load_from_hdf5(onde_file, fileobjs_by_h5path, h5_fh, h5_fh[h5_attr])
1816
+ _ONDE_attrs[attrname] = attr_obj
1817
+
1818
+ pass
1819
+ else:
1820
+ raise ValueError(f"Unknown hdf5 attribute type: {type(h5_attr).__name__:s}")
1821
+ pass
1822
+ for (subname,subobj) in h5_group.items():
1823
+ if isinstance(subobj,h5py.Dataset):
1824
+ # Check if hdf5 reference
1825
+ ref_type = h5py.check_dtype(ref = subobj.dtype)
1826
+ if ref_type is not None:
1827
+ # array of references
1828
+
1829
+ attr_obj = ONDEReferenceArray.load_from_hdf5(onde_file,fileobjs_by_h5path,h5_fh,subobj[...])
1830
+ _ONDE_attrs[subname] = attr_obj
1831
+ _ONDE_dataset_attrs.add(subname)
1832
+ pass
1833
+ else:
1834
+ # Array of values
1835
+ attr_obj = ONDEArray.new(value = subobj[...])
1836
+ _ONDE_attrs[subname] = attr_obj
1837
+ _ONDE_dataset_attrs.add(subname)
1838
+ pass
1839
+ pass
1840
+ # elif isinstance(subobj,h5py.Group): # These lines will enable support for hierarchical ONDE files
1841
+ # attr_obj = ONDEObject.load_from_hdf5(onde_file, fileobjs_by_h5path, h5_fh, subobj)
1842
+ # _ONDE_attrs[subname] = attr_obj
1843
+ else:
1844
+ raise ValueError(f"Unknown h5type {type(subobj).__name__:s} at hdf5 path {h5_group.name:s}")
1845
+ pass
1846
+ instance = cls(None,_ONDE_attrs = _ONDE_attrs,_ONDE_dataset_attrs = _ONDE_dataset_attrs)
1847
+ fileobj = ONDEFileObject(onde_instance = instance,hdf5_path = h5_group.name)
1848
+ fileobjs_by_h5path[h5_group.name] = fileobj
1849
+ return instance
1850
+ pass
1851
+
1852
+ class ONDEFileGraphSnapshot(ONDEObject):
1853
+ """ Not allowed to be referenced by any other ONDEObject.
1854
+ The ONDE:TYPE field should be absent.
1855
+ Attributes represent entry points of the graph."""
1856
+ def __init__(self, _orig = None, **kwargs):
1857
+ super().__init__(_orig, **kwargs)
1858
+ # remove the ONDE:TYPE added by the ONDEObject constructor because we are not really a proper object
1859
+ _ONDE_attrs = object.__getattribute__(self,"_ONDE_attrs")
1860
+ del _ONDE_attrs["ONDE:TYPE"]
1861
+ pass
1862
+
1863
+ # Unlike ONDEObject, we index with _get_item, not with
1864
+ # _get_attr. Therefore we override the attribute methods
1865
+ # with the ones from ONDEBase.
1866
+
1867
+ _get_attr=ONDEBase._get_attr
1868
+ _has_attr=ONDEBase._has_attr
1869
+ _set_attr=ONDEBase._set_attr
1870
+
1871
+ # __getattribute__=ONDEBase.__getattribute__
1872
+ __getattribute__=object.__getattribute__
1873
+ __setattr__=ONDEBase.__setattr__
1874
+
1875
+
1876
+
1877
+ #def _set_item(self,index,value):
1878
+ # return ONDEObject._set_attr(self,index,value)
1879
+
1880
+ def _get_item(self,index):
1881
+ # Allow shorthand keys
1882
+ keys = self._list_items()
1883
+ if index in keys:
1884
+ return ONDEObject._get_attr(self,index)
1885
+
1886
+ prefixed = "2.25." + index
1887
+ matching = [ key for key in keys if key.startswith(prefixed) ]
1888
+ if len(matching) == 1:
1889
+ return ONDEObject._get_attr(self,matching[0])
1890
+
1891
+ raise KeyError(f"No Dataset with UUID matching given key \"{index:s}\"")
1892
+ def __getitem__(self,index):
1893
+ return self._get_item(index)
1894
+ #def __setitem__(self,index,value):
1895
+ # return self._set_item(index,value)
1896
+
1897
+ def _add_dataset(self,key,dataset):
1898
+ ONDEObject._set_attr(self,key,dataset)
1899
+ pass
1900
+
1901
+ def add(self,dataset):
1902
+ # Must maintain parallel logic in ONDEProxy._add()
1903
+ if "ONDE:TYPE" not in dataset or dataset["ONDE:TYPE"].value[0] != "ONDE_DATASET":
1904
+ raise ValueError(f"Attempting to add something other than a dataset to an ONDEFileGraphSnapshot")
1905
+
1906
+ if "ONDE:UUID" not in dataset:
1907
+ raise ValueError(f"Attempting to add a dataset without a UUID to an ONDEFileGraphSnapshot")
1908
+
1909
+ self._add_dataset(dataset["ONDE:UUID"].value,dataset)
1910
+ pass
1911
+
1912
+ def _list_items(self):
1913
+ return ONDEObject._list_attrs(self)
1914
+
1915
+ def keys(self):
1916
+ return self._list_items()
1917
+
1918
+ def shortkeys(self):
1919
+ full_keys = self._list_items()
1920
+ # shortable_keys is a dictionary listing all of the keys that we can use a shorthand for (i.e. those starting with 2.25.). Values are intialized to none but will be populated with the shorthand eventually
1921
+ shortable_keys = { key: None for key in full_keys if key.startswith("2.25.") }
1922
+ shortable_key_set = frozenset(shortable_keys.keys())
1923
+ #import pdb
1924
+ #pdb.set_trace()
1925
+ for shortable_key in shortable_key_set:
1926
+ for num_chars_for_unique in range(4,len(shortable_key)+1):
1927
+ prefix = shortable_key[:(5 + num_chars_for_unique)]
1928
+ num_keys_matching_prefix = len([ key for key in shortable_key_set if key[:(5 + num_chars_for_unique)] == prefix ])
1929
+ if prefix in full_keys:
1930
+ num_keys_matching_prefix += 1
1931
+ pass
1932
+ if num_keys_matching_prefix == 1:
1933
+ shortable_keys[shortable_key] = prefix[5:]
1934
+ break
1935
+ if num_chars_for_unique == len(shortable_key):
1936
+ #Didn't find any prefix
1937
+ shortable_keys[shortable_key] = shortable_key
1938
+ pass
1939
+ pass
1940
+ pass
1941
+ reduced_keys = [ shortable_keys[key] if key in shortable_keys else key for key in full_keys ]
1942
+ return reduced_keys
1943
+
1944
+
1945
+ def __iter__(self):
1946
+ return iter(self.keys())
1947
+
1948
+ #@classmethod
1949
+ #def new(cls, entry_points = None):
1950
+ # if entry_points is None:
1951
+ # entry_points = TwoWayDictionary()
1952
+ # pass
1953
+ # return cls(None, ONDE_TYPE = [], _ONDE_attrs = entry_points)
1954
+ @classmethod
1955
+ def new(cls, **kwargs):
1956
+ _frozen = False
1957
+ if "_frozen" in kwargs:
1958
+ _frozen = kwargs["_frozen"]
1959
+ del kwargs["_frozen"]
1960
+ pass
1961
+ newobj = cls(None)
1962
+ for attrname in kwargs:
1963
+ newobj._set_item(attrname, kwargs[attrname])
1964
+ pass
1965
+ if _frozen:
1966
+ newobj._freeze()
1967
+ pass
1968
+ return newobj
1969
+
1970
+ pass
1971
+
1972
+ class ONDEField(object):
1973
+ """Represents a field from the .csv spec."""
1974
+
1975
+ defining_class = None # Name of the defining class
1976
+ class_prefix = None # Prefix on the field
1977
+ name = None # Field name, not including prefix or separating colon
1978
+ comments = None # Documentation string
1979
+ mandatory = None # True for mandatory attributes/datasets
1980
+ storage = None # "D" for dataset storage, "A" for attribute storage, "A or D" for either
1981
+ content_class_string = None # Referenced type
1982
+ dimensionality_string = None # Dimensionality field from .csv
1983
+ size_or_content_string = None # Value field from .csv
1984
+ min_val_string = None # Min field from .csv
1985
+ max_val_string = None # Max field from .csv
1986
+
1987
+ def __init__(self, **kwargs):
1988
+ for arg in kwargs:
1989
+ if hasattr(self, arg):
1990
+ setattr(self, arg, kwargs[arg])
1991
+ pass
1992
+ else:
1993
+ raise ValueError(f"ONDEField; unknown attribute {arg:s}")
1994
+ pass
1995
+ pass
1996
+
1997
+ @classmethod
1998
+ def from_csv_line(cls, row, class_derivation):
1999
+ (classname, name, comments, mandatory_optional, dataset_attribute, content_class, dims, size_or_content, min_val, max_val, accessory_class) = row
2000
+
2001
+ name_split = name.split(":")
2002
+ class_prefix = name_split[0]
2003
+ class_derivation = (classname,) if class_derivation is None else class_derivation
2004
+
2005
+ if len(name_split) != 2:
2006
+ raise ValueError(f"Field name {name:s} should have one colon")
2007
+
2008
+ if class_prefix not in class_derivation and not (class_prefix == "ONDE" and name in ["ONDE:LABEL", "ONDE:TYPE_TAGS","ONDE:UUID"]):
2009
+ raise ValueError(f"Mismatch between class name or super classes and prefix defining {name:s}")
2010
+
2011
+ if mandatory_optional not in ["M", "O"]:
2012
+ raise ValueError(f"Mandatory/Optional field is not either O or M defining {name:s}")
2013
+
2014
+ if dataset_attribute not in ["A", "D","A or D"]:
2015
+ raise ValueError(f"Dataset/Attribute field is not either D or A defining {name:s}")
2016
+
2017
+ name_only = name_split[1]
2018
+ mandatory = mandatory_optional == "M"
2019
+ storage = dataset_attribute
2020
+
2021
+ return cls(
2022
+ defining_class=classname,
2023
+ class_prefix=class_prefix,
2024
+ name=name_only,
2025
+ comments=comments,
2026
+ mandatory=mandatory,
2027
+ storage=storage,
2028
+ content_class_string=content_class,
2029
+ dimensionality_string=dims,
2030
+ size_or_content_string=size_or_content,
2031
+ min_val_string = min_val,
2032
+ max_val_string = max_val
2033
+ )
2034
+ pass
2035
+
2036
+
2037
+ class ONDEAccessoryClass(object):
2038
+ """Represents an accessory class defined in the ONDE .csv spec."""
2039
+
2040
+ classname = None # Name of the accessory class
2041
+ attributes = None # Dictionary by name of ONDEField references
2042
+ comments = None # Comments about this class (sourced from the .csv file)
2043
+
2044
+ def __init__(self):
2045
+ self.attributes = collections.OrderedDict()
2046
+ pass
2047
+ pass
2048
+
2049
+ class ONDEClass(object):
2050
+ """Represents a class defined in the ONDE .csv spec."""
2051
+ classname = None
2052
+ class_derivation = None # tuple of strings starting with base class and ending with this current class
2053
+ superclass = None # Reference the ONDEClass object for our superclass, or None
2054
+ type_tags = None # Dictionary by name of truth value for mandatory
2055
+ attributes = None # Dictionary by name of ONDEField references
2056
+ comments = None # Comments about thhis class (sourced from the .csv file)
2057
+
2058
+ def __init__(self):
2059
+ self.attributes = collections.OrderedDict()
2060
+ self.type_tags = collections.OrderedDict()
2061
+ pass
2062
+ pass
2063
+
2064
+ class ONDEClassInstanceWrapper(object):
2065
+ """Represents an ONDEObject that is an instance of a known class. References the proxy to the underlying ONDEObject and the class definition."""
2066
+
2067
+ # only _proxy OR _obj can ever be set
2068
+ _proxy = None # an ONDEProxy for the ONDEObject instance
2069
+ _obj = None # an ONDEObject instance
2070
+ _graph = None # ONDEFileGraph, used only with _obj
2071
+ # _our_class = None # the ONDEClass instance
2072
+
2073
+ def __init__(self, **kwargs):
2074
+ # _dict = object.__getattribute__(self, "__dict__")
2075
+
2076
+ for arg in kwargs:
2077
+ if arg in type(self).__dict__:
2078
+ # setattr(self, arg, kwargs[arg])
2079
+ object.__setattr__(self, arg, kwargs[arg])
2080
+
2081
+ pass
2082
+ else:
2083
+ raise ValueError(f"ONDEClassInstanceWrapper; unknown attribute {arg:s}")
2084
+ pass
2085
+ pass
2086
+
2087
+ def _get_obj(self):
2088
+ if self._obj is not None:
2089
+ return self._obj
2090
+ return self._proxy._get_obj()
2091
+
2092
+ def _full_fieldname_from_attrname(self, _graph, obj, attrname):
2093
+ #_graph = object.__getattribute__(self,"_graph")
2094
+ if attrname in onde_basic_attrs[obj.__class__.__name__]:
2095
+ return (attrname, None)
2096
+
2097
+ if isinstance(obj,ONDEObject):
2098
+ classdefs = _graph.class_defs
2099
+ if attrname=="ONDE:TYPE":
2100
+ return (attrname, None)
2101
+ (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set) = classdefs.get_fields_dict(obj)
2102
+ if attrname in combined_fields_dict:
2103
+ field = combined_fields_dict[attrname]
2104
+ full_name = field.class_prefix + ":" + field.name
2105
+ return (full_name, field)
2106
+ else:
2107
+ # raise NameError(f"Unknown attribute {attrname:s} on ONDEObject of type {obj._ONDE_attrs['ONDE:TYPE'].value[-1]:s}.")
2108
+ return (None, None)
2109
+ pass
2110
+ return (attrname, None)
2111
+
2112
+ def _get_attr_or_item(self,get_method_name,key):
2113
+
2114
+ _proxy = object.__getattribute__(self,"_proxy")
2115
+ _obj = object.__getattribute__(self,"_obj")
2116
+ _graph = object.__getattribute__(self,"_graph")
2117
+
2118
+ if _proxy is not None:
2119
+ _obj = _proxy._get_obj()
2120
+ _graph = _proxy._graph
2121
+ pass
2122
+
2123
+ full_name = key
2124
+ if get_method_name == "_get_attr":
2125
+ (full_name, field) = self._full_fieldname_from_attrname(_graph, _obj, key)
2126
+ if full_name is None:
2127
+ full_name = key
2128
+ pass
2129
+
2130
+ if not _obj._has_attr(full_name):
2131
+ return None
2132
+
2133
+ pass
2134
+
2135
+ if _proxy is not None:
2136
+ get_method = getattr(_proxy,get_method_name)
2137
+ value = get_method(full_name) # e.g. _proxy._get_attr(key)
2138
+ if isinstance(value,ONDEProxy):
2139
+ value_obj = value._get_obj()
2140
+ if isinstance(value_obj,ONDEObject) or isinstance(value_obj,ONDEReferenceArray):
2141
+ return self.__class__.new_from_proxy(value)
2142
+ return value
2143
+ return value
2144
+ else:
2145
+
2146
+ get_method = getattr(_obj,get_method_name)
2147
+ value = get_method(full_name) # e.g. _obj._get_attr(name)
2148
+
2149
+ if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
2150
+
2151
+ value_wrapper = self.__class__.new_from_obj(_graph,value)
2152
+ return value_wrapper
2153
+ else:
2154
+
2155
+ return value
2156
+ pass
2157
+ pass
2158
+
2159
+
2160
+
2161
+ def _has_attr_or_item(self,has_method_name,key):
2162
+
2163
+ _proxy = object.__getattribute__(self,"_proxy")
2164
+ _obj = object.__getattribute__(self,"_obj")
2165
+ _graph = object.__getattribute__(self,"_graph")
2166
+
2167
+ if _proxy is not None:
2168
+ _obj = _proxy._get_obj()
2169
+ _graph = _proxy._graph
2170
+ pass
2171
+
2172
+ full_name = key
2173
+
2174
+ (full_name, field) = self._full_fieldname_from_attrname(_graph, _obj, key)
2175
+ if full_name is None:
2176
+ full_name = key
2177
+ pass
2178
+
2179
+ if has_method_name == "_has_attr":
2180
+ return _obj._has_attr(full_name)
2181
+
2182
+ if has_method_name == "_has_item":
2183
+ return _obj._has_item(full_name)
2184
+ return False
2185
+
2186
+ def _get_attr(self,name):
2187
+ return self._get_attr_or_item("_get_attr",name)
2188
+
2189
+ def __contains__(self,name):
2190
+ return self._has_item(name)
2191
+
2192
+ def _get_attr_dataset_storage(self,name):
2193
+ obj = self._get_obj()
2194
+ return obj._get_attr_dataset_storage(name)
2195
+
2196
+ def _set_attr_or_item(self,set_method_name,key,value):
2197
+ our_proxy = object.__getattribute__(self,"_proxy")
2198
+ our_obj = object.__getattribute__(self,"_obj")
2199
+ _graph = object.__getattribute__(self,"_graph")
2200
+
2201
+ if our_proxy is not None:
2202
+ our_obj = our_proxy._get_obj()
2203
+ _graph = our_proxy._graph
2204
+ pass
2205
+
2206
+ full_name = key
2207
+ field = None
2208
+ if set_method_name == "_set_attr" or set_method_name == "_set_dataset_attr":
2209
+ (full_name, field) = self._full_fieldname_from_attrname(_graph, our_obj, key)
2210
+ if full_name is None:
2211
+ full_name = key
2212
+ pass
2213
+ pass
2214
+
2215
+ if field is not None and set_method_name == "_set_attr":
2216
+ # For now, if field.storage is "A or D", we always just store it as a dataset.
2217
+ if field.storage != "A":
2218
+ set_method_name = "_set_dataset_attr"
2219
+ pass
2220
+ pass
2221
+
2222
+ if isinstance(value,ONDEClassInstanceWrapper):
2223
+ target_proxy = object.__getattribute__(value,"_proxy")
2224
+ target_obj = object.__getattribute__(value,"_obj")
2225
+ if our_proxy is not None and target_proxy is not None:
2226
+ set_method = getattr(our_proxy,set_method_name)
2227
+ set_method(full_name,target_proxy) # self._proxy._set_attr(key,target_proxy)
2228
+ pass
2229
+ elif self._proxy is not None and target_obj is not None:
2230
+ set_method = getattr(self._proxy,set_method_name)
2231
+ set_method(full_name,target_obj)
2232
+ pass
2233
+ elif self._obj is not None and target_proxy is not None:
2234
+ set_method = getattr(self._obj,set_method_name)
2235
+ set_method(full_name,target_proxy._get_obj()) # e.g. self._obj._set_attr(key,_proxy._get_obj())
2236
+ pass
2237
+ elif self._obj is not None and target_obj is not None:
2238
+ set_method = getattr(self._obj,set_method_name)
2239
+ set_method(full_name,target_obj)
2240
+ pass
2241
+ else:
2242
+ assert(False) # Exactly one of _obj and _proxy should be valid
2243
+ pass
2244
+ pass
2245
+ elif isinstance(value,ONDEProxy):
2246
+ if our_proxy is not None:
2247
+ set_method = getattr(our_proxy,set_method_name)
2248
+ set_method(full_name,value)
2249
+ pass
2250
+ elif our_obj is not None:
2251
+ set_method = getattr(our_obj,set_method_name)
2252
+ set_method(full_name,value._get_obj())
2253
+ pass
2254
+ else:
2255
+ assert(False)
2256
+ pass
2257
+ pass
2258
+ elif isinstance(value,ONDEBase):
2259
+ if our_proxy is not None:
2260
+ set_method = getattr(our_proxy,set_method_name)
2261
+ set_method(full_name,value)
2262
+ pass
2263
+ elif our_obj is not None:
2264
+ set_method = getattr(our_obj,set_method_name)
2265
+ set_method(full_name,value)
2266
+ pass
2267
+ else:
2268
+ assert(False)
2269
+ pass
2270
+ pass
2271
+ else:
2272
+ #raise ValueError(f"Attribute values should be ONDEBase or ONDEProxy or ONDEClassInstanceWrapper")
2273
+ #build an object from python structures
2274
+ target_obj = onde_from_python(value, field)
2275
+ if our_proxy is not None:
2276
+ set_method = getattr(our_proxy,set_method_name)
2277
+ set_method(full_name,target_obj)
2278
+ pass
2279
+ elif our_obj is not None:
2280
+ set_method = getattr(our_obj,set_method_name)
2281
+ set_method(full_name,target_obj)
2282
+ pass
2283
+ else:
2284
+ assert(False)
2285
+ pass
2286
+ pass
2287
+ pass
2288
+
2289
+
2290
+
2291
+ def _set_attr(self,name,value):
2292
+ self._set_attr_or_item("_set_attr",name,value)
2293
+ pass
2294
+
2295
+ def _set_dataset_attr(self,name,value):
2296
+ self._set_attr_or_item("_set_dataset_attr",name,value)
2297
+ pass
2298
+
2299
+ def _get_item(self,index):
2300
+ return self._get_attr_or_item("_get_item",index)
2301
+
2302
+ def _has_item(self,index):
2303
+ return self._has_attr_or_item("_has_item",index)
2304
+
2305
+ def _set_item(self,index,value):
2306
+ self._set_attr_or_item("_set_item",index,value)
2307
+ pass
2308
+
2309
+ def _get_data(self,name):
2310
+ _proxy = object.__getattribute__(self,"_proxy")
2311
+ _obj = object.__getattribute__(self,"_obj")
2312
+
2313
+ if _proxy is not None:
2314
+ value = _proxy._get_data(self,name)
2315
+
2316
+ return value
2317
+ else:
2318
+ _graph = object.__getattribute__(self,"_graph")
2319
+ value = _obj._get_data(self,name)
2320
+ return value
2321
+ pass
2322
+
2323
+ def _set_data(self,name,value):
2324
+ if isinstance(value,ONDEClassInstanceWrapper) or isinstance(value,ONDEBase) or isinstance(value,ONDEProxy):
2325
+ raise ValueError(f"_set_data(): value must not be an ONDEClassInstanceWrapper, a ONDEProxy, or an ONDEBase subclass")
2326
+ if self._proxy is not None:
2327
+ self._proxy._set_data(name,value)
2328
+ pass
2329
+ elif self._obj is not None:
2330
+ self._obj._set_data(name,value)
2331
+ pass
2332
+ else:
2333
+ assert(False)
2334
+ pass
2335
+ pass
2336
+
2337
+ def _basic_attr_set(self):
2338
+ if self._proxy is not None:
2339
+ obj = self._proxy._get_obj()
2340
+ pass
2341
+ else:
2342
+ obj = self._obj
2343
+ pass
2344
+
2345
+ fields = onde_basic_attrs[obj.__class__.__name__]
2346
+ return frozenset(fields)
2347
+
2348
+ def _class_field_set(self):
2349
+ if self._proxy is not None:
2350
+ obj = self._proxy._get_obj()
2351
+ pass
2352
+ else:
2353
+ obj = self._obj
2354
+ pass
2355
+
2356
+ if self._proxy is not None:
2357
+ _graph = self._proxy._graph
2358
+ pass
2359
+ else:
2360
+ _graph = self._graph
2361
+ pass
2362
+
2363
+ classdefs = _graph.class_defs
2364
+ if classdefs is not None:
2365
+
2366
+ if isinstance(obj, ONDEObject):
2367
+ (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(obj)
2368
+ return frozenset(list(full_fields_dict.keys()))
2369
+ pass
2370
+
2371
+ return frozenset()
2372
+
2373
+ def _shortkeys(self):
2374
+ obj = self._get_obj()
2375
+ if isinstance(obj,ONDEObject):
2376
+ if self._proxy is not None:
2377
+ _graph = self._proxy._graph
2378
+ pass
2379
+ else:
2380
+ _graph = self._graph
2381
+ pass
2382
+ obj_fields = self._assigned_attr_field_set()
2383
+
2384
+ classdefs = _graph.class_defs
2385
+ if classdefs is not None:
2386
+ (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(obj)
2387
+ # Make concise_set mutable
2388
+ concise_set = set(concise_set)
2389
+ for obj_field in obj_fields:
2390
+ if obj_field in full_fields_dict and obj_field not in concise_set:
2391
+ # This field is in the class definitions but it is not in the concise key set. Therefore there must be a shorthand in the concise set. So we don't need to consider this field.
2392
+ pass
2393
+ else:
2394
+ # Make sure this field is in the concise set
2395
+ concise_set.add(obj_field)
2396
+ pass
2397
+ pass
2398
+ return list(concise_set)
2399
+ return list(obj_fields)
2400
+ return obj._shortkeys()
2401
+
2402
+ def _keys(self):
2403
+ obj = self._get_obj()
2404
+
2405
+ if isinstance(obj,ONDEObject):
2406
+
2407
+ full_fields = self._class_field_set()
2408
+ obj_fields = self._assigned_attr_field_set()
2409
+ return list(full_fields | obj_fields)
2410
+ return obj._keys()
2411
+
2412
+ def _shorthand_field_set(self):
2413
+ if self._proxy is not None:
2414
+ obj = self._proxy._get_obj()
2415
+ pass
2416
+ else:
2417
+ obj = self._obj
2418
+ pass
2419
+
2420
+ if self._proxy is not None:
2421
+ _graph = self._proxy._graph
2422
+ pass
2423
+ else:
2424
+ _graph = self._graph
2425
+ pass
2426
+
2427
+ classdefs = _graph.class_defs
2428
+ if classdefs is not None:
2429
+
2430
+ if isinstance(obj, ONDEObject):
2431
+ (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(obj)
2432
+ return frozenset(list(shorthand_fields_dict.keys()))
2433
+ pass
2434
+
2435
+ return frozenset()
2436
+
2437
+ def _assigned_attr_field_set(self):
2438
+ """Fields defined by actual assigned attributes of this object"""
2439
+ if self._proxy is not None:
2440
+ obj = self._proxy._get_obj()
2441
+ pass
2442
+ else:
2443
+ obj = self._obj
2444
+ pass
2445
+
2446
+ return frozenset(obj._list_attrs())
2447
+
2448
+ def _all_field_set(self):
2449
+ return list(self._basic_attr_set()|self._class_field_set()|self._shorthand_field_set()|self._assigned_attr_field_set()).sorted()
2450
+
2451
+ def __dir__(self):
2452
+ if self._proxy is not None:
2453
+ obj = self._proxy._get_obj()
2454
+ pass
2455
+ else:
2456
+ obj = self._obj
2457
+ pass
2458
+
2459
+ if self._proxy is not None:
2460
+ _graph = self._proxy._graph
2461
+ pass
2462
+ else:
2463
+ _graph = self._graph
2464
+ pass
2465
+
2466
+ classdefs = _graph.class_defs
2467
+ if classdefs is not None:
2468
+
2469
+ if isinstance(obj, ONDEObject):
2470
+ (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(obj)
2471
+ return sorted(list(concise_set|self._basic_attr_set()))
2472
+ pass
2473
+
2474
+ return []
2475
+
2476
+
2477
+
2478
+ def __getattribute__(self,name):
2479
+ if name.startswith('_'):
2480
+ return object.__getattribute__(self,name)
2481
+
2482
+ return self._get_attr_or_item("_get_attr",name)
2483
+ obsolete = r"""
2484
+ if self._proxy is not None:
2485
+ obj = self._proxy._get_obj()
2486
+ _graph = self._proxy._graph
2487
+ pass
2488
+ else:
2489
+ obj = self._obj
2490
+ _graph = self._graph
2491
+ pass
2492
+
2493
+ field_dict = {}
2494
+ if isinstance(obj,ONDEObject):
2495
+ classdefs = _graph.class_defs
2496
+ field_dict = classdefs.get_fields_dict(obj)
2497
+ if name in field_dict:
2498
+ field = field_dict[name]
2499
+ full_name = field.class_prefix + ":" + field.name
2500
+ value = obj._ONDE_attrs[full_name]
2501
+ if self._proxy is not None:
2502
+ valueproxy = _proxy._get_attr(full_name)
2503
+ if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
2504
+ return self.__class__.new_from_proxy(valueproxy)
2505
+ else:
2506
+ return valueproxy
2507
+ pass
2508
+ else:
2509
+ if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
2510
+ return self.__class__.new_from_obj(self._graph, value)
2511
+ else:
2512
+ return value
2513
+ pass
2514
+ pass
2515
+ pass
2516
+ else:
2517
+ assert(isinstance(obj, ONDEReferenceArray))
2518
+ value = obj._get_attr(name)
2519
+
2520
+ if self._proxy is not None:
2521
+ valueproxy = _proxy._get_attr(name)
2522
+ if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
2523
+ return self.__class__.new_from_proxy(valueproxy)
2524
+ else:
2525
+ return valueproxy
2526
+ pass
2527
+ else:
2528
+ if isinstance(value,ONDEObject) or isinstance(value,ONDEReferenceArray):
2529
+ return self.__class__.new_from_obj(self._graph, value)
2530
+ else:
2531
+ return value
2532
+ pass
2533
+ pass
2534
+ assert(False)
2535
+ pass
2536
+ """
2537
+ def __setitem__(self,key,value):
2538
+ self._set_attr_or_item("_set_item",key,value)
2539
+ return
2540
+
2541
+ def __getitem__(self,key):
2542
+ return self._get_attr_or_item("_get_item",key)
2543
+
2544
+
2545
+ def __setattr__(self,name,value):
2546
+ if name.startswith('_'):
2547
+ raise ValueError('Cannot assign attribute with leading underscore')
2548
+ self._set_attr_or_item("_set_attr",name,value)
2549
+ return
2550
+ obsolete = r"""
2551
+ if self._proxy is not None:
2552
+ if isinstance(value, ONDEClassInstanceWrapper):
2553
+ _proxy = value._proxy
2554
+ _obj = value._obj
2555
+ if _proxy is not None:
2556
+ setattr(self._proxy, _proxy)
2557
+ pass
2558
+ elif _obj is not None:
2559
+ setattr(self._proxy, _obj)
2560
+ pass
2561
+ else:
2562
+ assert(False)
2563
+ pass
2564
+ pass
2565
+ else:
2566
+ setattr(self._proxy, value)
2567
+ pass
2568
+ pass
2569
+ elif self._obj is not None:
2570
+ if isinstance(value, ONDEClassInstanceWrapper):
2571
+ _proxy = value._proxy
2572
+ _obj = value._obj
2573
+ if _proxy is not None:
2574
+ setattr(self._obj, _proxy._get_obj())
2575
+ pass
2576
+ elif _obj is not None:
2577
+ setattr(self._obj, _obj)
2578
+ pass
2579
+ else:
2580
+ assert(False)
2581
+ pass
2582
+ pass
2583
+ else:
2584
+ if isinstance(value, ONDEProxy):
2585
+ setattr(self._obj, value._get_obj())
2586
+ pass
2587
+ else:
2588
+ setattr(self._obj, value)
2589
+ pass
2590
+ pass
2591
+ pass
2592
+ else:
2593
+ assert(False)
2594
+ pass
2595
+ pass
2596
+
2597
+ """
2598
+
2599
+ def _repr_short(self, onde_class = None):
2600
+ our_proxy = object.__getattribute__(self,"_proxy")
2601
+ our_obj = object.__getattribute__(self,"_obj")
2602
+ _graph = object.__getattribute__(self,"_graph")
2603
+
2604
+ if our_proxy is not None:
2605
+ our_obj = our_proxy._get_obj()
2606
+ _graph = our_proxy._graph
2607
+ pass
2608
+
2609
+ return "InstanceWrapper of " + our_obj._repr_short()
2610
+
2611
+ def _repr(self, indentation, onde_class=None, extra_attrs={}):
2612
+ our_proxy = object.__getattribute__(self,"_proxy")
2613
+ our_obj = object.__getattribute__(self,"_obj")
2614
+ _graph = object.__getattribute__(self,"_graph")
2615
+
2616
+ if our_proxy is not None:
2617
+ our_obj = our_proxy._get_obj()
2618
+ _graph = our_proxy._graph
2619
+ pass
2620
+
2621
+ classdefs = _graph.class_defs
2622
+ if classdefs is not None:
2623
+
2624
+ if isinstance(our_obj, ONDEObject):
2625
+ (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set)=classdefs.get_fields_dict(our_obj)
2626
+ attrdict=collections.OrderedDict()
2627
+ for attrname in sorted(list(concise_set)):
2628
+ attrval = self._get_attr(attrname)
2629
+ dataset_storage = self._get_attr_dataset_storage(attrname)
2630
+ storage_suffix = ""
2631
+ if dataset_storage:
2632
+ storage_suffix = " (hdf5 dataset storage)"
2633
+ pass
2634
+
2635
+ if attrval is not None:
2636
+ attrdict[attrname] = attrval._repr_short() + storage_suffix
2637
+ pass
2638
+ else:
2639
+ attrdict[attrname] = "None"
2640
+ pass
2641
+ pass
2642
+ return our_obj._repr(indentation,onde_class = f"InstanceWrapper({our_obj._ONDE_attrs['ONDE:TYPE'].value[-1]:s})", extra_attrs = attrdict)
2643
+ pass
2644
+ return our_obj._repr(indentation)
2645
+
2646
+ def __repr__(self):
2647
+ return self._repr(0)
2648
+
2649
+ @classmethod
2650
+ def new_from_proxy(cls, proxy):
2651
+ obj = proxy._get_obj()
2652
+ if not isinstance(obj,ONDEObject) and not isinstance(obj,ONDEReferenceArray):
2653
+ raise ValueError(f"ONDEClassInstanceWrapper only wraps non-leaf nodes, not {obj.__class__.__name__}")
2654
+ return cls(_proxy=proxy)
2655
+
2656
+ @classmethod
2657
+ def new_obj(cls, graph, onde_classname, **kwargs):
2658
+ if graph.class_defs is None:
2659
+ raise ValueError("Graph does not have class definitions loaded")
2660
+
2661
+ onde_class = graph.class_defs.classes[onde_classname]
2662
+ obj = ONDEObject.new(None,None,ONDE_TYPE = ONDEArray.new(onde_class.class_derivation, _frozen = True), **kwargs)
2663
+ # proxy = ONDEProxy(_path=None)
2664
+
2665
+ return cls(_graph=graph,_obj=obj)
2666
+
2667
+ @classmethod
2668
+ def new_from_obj(cls,graph,obj):
2669
+ if not isinstance(obj,ONDEBase):
2670
+ raise ValueError(f"Given object is of type {obj.__class__.__name__:s} and is not an ONDEBase instance")
2671
+
2672
+ # onde_classname = object.__getattribute__(_orig, "ONDE:TYPE")
2673
+ if graph.class_defs is None:
2674
+ raise ValueError("Graph does not have class definitions loaded")
2675
+ return cls(_graph=graph,_obj=obj)
2676
+
2677
+ pass
2678
+
2679
+ class ONDEClassDefinitions(object):
2680
+ """Represents the set of class definitions from the ONDE .csv file."""
2681
+ #NOTE: Immutable once constructed, except for field cache
2682
+ classes = None # Dictionary by name of ONDEClass
2683
+ acc_classes = None # Dictionary by name of accessory classes
2684
+ file_type = None # From the size_or_content of the blank ONDE:TYPE entry at the top of the csv file
2685
+ version = None # VERSION specification from the .csv file
2686
+
2687
+
2688
+
2689
+ field_cache = None #dictionary by (tuple_of_class_derivation, frozen_set_of_accessory_classes) of a dictionary by field names and field name abbreviations of ONDEField objects
2690
+
2691
+ def __init__(self):
2692
+ self.classes = collections.OrderedDict()
2693
+ self.acc_classes = collections.OrderedDict()
2694
+ self.field_cache = {}
2695
+ pass
2696
+
2697
+ def get_fields_dict(self,obj):
2698
+ """returns three dictionaries by field name of ONDEField: the shorthand dictionary, the full dictionary, the combined dictionary, and the concise set. Automatically omits any shorthands that conflict with actual attributes. The concise set includes the shortest name for each field plus any fields which don't correspond to ONDEField objects"""
2699
+ onde_type_value = obj._ONDE_attrs["ONDE:TYPE"]
2700
+ assert(isinstance(onde_type_value,ONDEArray))
2701
+
2702
+ class_derivation = tuple([str(classname) for classname in onde_type_value.value])
2703
+
2704
+ if "ONDE:TYPE_TAGS" in obj._ONDE_attrs:
2705
+ acc_class_value = obj._ONDE_attrs["ONDE:TYPE_TAGS"]
2706
+ assert(isinstance(acc_class_value,ONDEArray))
2707
+ type_tags = frozenset([str(type_tag) for typ_tag in acc_class_value.value])
2708
+ pass
2709
+ else:
2710
+ type_tags = frozenset()
2711
+ pass
2712
+
2713
+ field_cache_key = (class_derivation, type_tags, frozenset(obj._ONDE_attrs._keys()))
2714
+
2715
+ if field_cache_key in self.field_cache:
2716
+ return self.field_cache[field_cache_key]
2717
+
2718
+ shorthand_fields_dict = collections.OrderedDict()
2719
+ full_fields_dict = collections.OrderedDict()
2720
+ combined_fields_dict = collections.OrderedDict()
2721
+ derivation_index=0
2722
+ mandatory_acc_classes = set()
2723
+ shorthand_blacklist = set()
2724
+
2725
+ def add_field(fieldname,field,extra_keys):
2726
+ full_fields_dict[fieldname]=field
2727
+ combined_fields_dict[fieldname]=field
2728
+ for key in extra_keys:
2729
+ if key in shorthand_blacklist:
2730
+ continue
2731
+ if key in obj._ONDE_attrs:
2732
+ # Don't allow overriding an actual attribute of this object with a shorthand
2733
+ continue
2734
+ if key in shorthand_fields_dict:
2735
+ # key already present
2736
+ if shorthand_fields_dict[key].class_prefix==full_fields_dict[fieldname].class_prefix and shorthand_fields_dict[key].name==full_fields_dict[fieldname].name:
2737
+ # Override of a shorthand for the same thing already present. This is allowable.
2738
+ shorthand_fields_dict[key]=full_fields_dict[fieldname]
2739
+ combined_fields_dict[key]=full_fields_dict[fieldname]
2740
+ pass
2741
+ else:
2742
+ # Conflicting shorthands. Remove pre-existing entry and blacklist
2743
+
2744
+ del shorthand_fields_dict[key]
2745
+ del combined_fields_dict[key]
2746
+ shorthand_blacklist.add(key)
2747
+ pass
2748
+ pass
2749
+ else:
2750
+ shorthand_fields_dict[key]=full_fields_dict[fieldname]
2751
+ combined_fields_dict[key]=full_fields_dict[fieldname]
2752
+ pass
2753
+ pass
2754
+ pass
2755
+
2756
+ for classname in ("ONDE",)+class_derivation:
2757
+ # from base class to most derived
2758
+ derivation_index+=1
2759
+
2760
+ if not classname in self.classes:
2761
+ continue
2762
+
2763
+ if class_derivation[:(derivation_index-1)] != self.classes[classname].class_derivation:
2764
+ print(f"Object class derivation mismatch: {class_derivation[:(derivation_index-1)]} from object definition versus {self.classes[classname].class_derivation} from class definition.",file=sys.stderr)
2765
+ break
2766
+
2767
+
2768
+ # Gather fields from this class
2769
+ for fieldname in self.classes[classname].attributes:
2770
+ extra_keys=set()
2771
+ if ':' in fieldname:
2772
+ #Shorthand for field name by admitting colon
2773
+ extra_keys.add(fieldname[(fieldname.index(':')+1):])
2774
+ pass
2775
+
2776
+ add_field(fieldname,self.classes[classname].attributes[fieldname],extra_keys)
2777
+ pass
2778
+
2779
+ # Gather mandatory accessory classes
2780
+ for acc_class_name in self.classes[classname].type_tags:
2781
+ if self.classes[classname].type_tags[acc_class_name]:
2782
+ # This accessory class is mandatory
2783
+ mandatory_acc_classes.add(acc_class_name)
2784
+ pass
2785
+ pass
2786
+ pass
2787
+
2788
+ acc_classes = type_tags | mandatory_acc_classes # Set union
2789
+
2790
+ for acc_class_name in acc_classes:
2791
+
2792
+ # Gather fields from accessory classes
2793
+ acc_class = self.acc_classes[acc_class_name]
2794
+
2795
+ # Gather fields from this accessory class
2796
+ for fieldname in acc_class.attributes:
2797
+ extra_keys=set()
2798
+ if ':' in fieldname:
2799
+ #Shorthand for field name by admitting colon
2800
+ extra_keys.add(fieldname[(fieldname.index(':')+1):])
2801
+ pass
2802
+
2803
+ add_field(fieldname,acc_class.attributes[fieldname],extra_keys)
2804
+ pass
2805
+ pass
2806
+ # Construct concise set
2807
+ reverse_combined = {}
2808
+ for fieldname in combined_fields_dict:
2809
+ if id(combined_fields_dict[fieldname]) in reverse_combined:
2810
+ reverse_combined[id(combined_fields_dict[fieldname])].append(fieldname)
2811
+ pass
2812
+ else:
2813
+ reverse_combined[id(combined_fields_dict[fieldname])] = [fieldname]
2814
+ pass
2815
+ pass
2816
+
2817
+ concise_set=set()
2818
+ for objid in reverse_combined:
2819
+ shortest = sorted(reverse_combined[objid], key=len)[0]
2820
+ concise_set.add(shortest)
2821
+ pass
2822
+
2823
+ for attrname in obj._ONDE_attrs:
2824
+ if attrname not in full_fields_dict:
2825
+ concise_set.add(attrname)
2826
+ pass
2827
+ pass
2828
+
2829
+ concise_set_frozen=frozenset(concise_set)
2830
+
2831
+
2832
+ # import pdb
2833
+ # pdb.set_trace()
2834
+
2835
+ self.field_cache[field_cache_key] = (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set_frozen)
2836
+ return (shorthand_fields_dict,full_fields_dict,combined_fields_dict,concise_set_frozen)
2837
+
2838
+
2839
+ @classmethod
2840
+ def load_from_csv(cls, filename,extra_class_defs = None):
2841
+ class_defs = ONDEClassDefinitions()
2842
+
2843
+ if extra_class_defs is not None:
2844
+ raise ValueError("Extra class definitions not yet supported")
2845
+
2846
+ with open(filename,mode="r",encoding="utf-8-sig") as csvfh:
2847
+ reader = csv.reader(csvfh, delimiter=";")
2848
+
2849
+ for row in reader:
2850
+ if len(row) < 8:
2851
+ row += [""]*(8-len(row))
2852
+ pass
2853
+
2854
+ stripped_row = [col.strip() for col in row]
2855
+ (classname, name, comments, mandatory_optional, dataset_attribute, content_class, dims, size_or_content, min_val, max_val, accessory_class) = stripped_row
2856
+
2857
+ if classname == "Class":
2858
+ # header csv line
2859
+ continue
2860
+
2861
+ if name == "ONDE:VERSION":
2862
+ class_defs.version=size_or_content
2863
+ continue
2864
+
2865
+ if name == "ONDE:FILETYPE":
2866
+ # ignore filetype definition in .csv
2867
+ continue
2868
+
2869
+ if accessory_class == "True":
2870
+ accessory_class = True
2871
+ pass
2872
+ elif accessory_class == "False" or accessory_class == "":
2873
+ accessory_class = False
2874
+ pass
2875
+ else:
2876
+ raise ValueError(f"unknown accessory_class value in .csv: {accessory_class:s}")
2877
+
2878
+ if name == "ONDE:TYPE" and not accessory_class:
2879
+ # class definition
2880
+ if classname in class_defs.classes:
2881
+ raise ValueError(f"Class {classname:s} multiply defined in {filename:s}")
2882
+ newclass = ONDEClass()
2883
+ newclass.classname = classname
2884
+ newclass.comments = comments
2885
+ # size_or_content should be a list of base classes terminated with this class, written roughly like a python list of strings
2886
+ baseclasses = ast.literal_eval(size_or_content)
2887
+ assert(type(baseclasses) is list)
2888
+ if classname == "":
2889
+ # Blank ONDE:TYPE entry specifying the file type
2890
+ assert(len(baseclasses) == 1)
2891
+ assert(type(baseclasses[0]) is str)
2892
+ class_defs.file_type = baseclasses[0]
2893
+ superclasses = []
2894
+ superclass_def = None
2895
+ pass
2896
+ else:
2897
+ assert(baseclasses[-1] == classname)
2898
+
2899
+ # check superclasses
2900
+ superclasses = baseclasses[:-1]
2901
+
2902
+ if len(superclasses) > 0:
2903
+ immediate_superclass = superclasses[-1]
2904
+ if immediate_superclass not in class_defs.classes:
2905
+ raise ValueError(f"Superclass {immediate_superclass:s} of {classname:s} in {filename:s} is unknown")
2906
+ superclass_def = class_defs.classes[immediate_superclass]
2907
+ # Superclass definition list of class names should match our superclasses
2908
+ if tuple(superclasses) != superclass_def.class_derivation:
2909
+ raise ValueError(f"Superclass {immediate_superclass:s} derviation {str(superclass_def.class_derivation):s} does not match superclass list {str(tuple(superclasses)):s} from class {classname:s} in csv file {filename:s}")
2910
+ pass
2911
+ else:
2912
+ superclass_def = None
2913
+ pass
2914
+ pass
2915
+
2916
+ newclass.class_derivation = tuple(superclasses) + (classname,)
2917
+ newclass.superclass = superclass_def # ONDEClassObject
2918
+ class_defs.classes[classname] = newclass
2919
+ pass
2920
+
2921
+ elif name == "ONDE:TYPE" and accessory_class:
2922
+ # accessory class definition
2923
+
2924
+ if classname in class_defs.acc_classes:
2925
+ raise ValueError(f"Accessory class {classname:s} multiply defined in {filename:s}")
2926
+
2927
+ newclass = ONDEAccessoryClass()
2928
+ newclass.classname = classname
2929
+ newclass.comments = comments
2930
+ # size_or_content should be a list of base classes terminated with this class, written roughly like a python list of strings
2931
+ acc_classes = ast.literal_eval(size_or_content)
2932
+
2933
+ assert(type(acc_classes) is list)
2934
+ assert(len(acc_classes) == 1)
2935
+ assert(acc_classes[0] == classname)
2936
+
2937
+ class_defs.acc_classes[classname] = newclass
2938
+
2939
+ pass
2940
+
2941
+ elif name == "ONDE:TYPE_TAGS":
2942
+ # specification for a class having a particular accessory class
2943
+
2944
+ type_tags = ast.literal_eval(size_or_content)
2945
+
2946
+ for type_tag in type_tags:
2947
+ class_defs.classes[classname].type_tags[type_tag] = mandatory_optional == "M"
2948
+
2949
+ pass
2950
+
2951
+ pass
2952
+
2953
+ else:
2954
+ # populating class with a field
2955
+
2956
+ class_derivation = None
2957
+ class_or_acc = None
2958
+
2959
+ if classname in class_defs.classes:
2960
+ class_or_acc = class_defs.classes[classname]
2961
+ class_derivation = class_or_acc.class_derivation
2962
+ pass
2963
+ elif classname in class_defs.acc_classes:
2964
+ class_or_acc = class_defs.acc_classes[classname]
2965
+ pass
2966
+ else:
2967
+ raise ValueError(f"Class {classname:s} not found; not a known Class or AccessoryClass while defining attribute {name:s}")
2968
+
2969
+ if name in class_or_acc.attributes:
2970
+ raise ValueError(f"Attribute {name} already in Class or AccessoryClass")
2971
+
2972
+
2973
+ class_or_acc.attributes[name] = ONDEField.from_csv_line(stripped_row, class_derivation)
2974
+
2975
+ pass
2976
+ pass
2977
+ pass
2978
+
2979
+ return class_defs
2980
+ pass
2981
+
2982
+
2983
+ @dataclass
2984
+ class ScopeNode(object):
2985
+ referrers: Optional[set[ONDEBase]]=field(default_factory=set) # Set of referring objects.
2986
+ edges: Optional[set[Any]]=field(default_factory=set) # Set of edge indexes. If it is None, then all edges are in scope. Usually either each edge index is either a string for an ONDEObject or a tuple of integers for an ONDEReferenceArray.
2987
+ pass
2988
+
2989
+ def graph_replace_node__walk(starting_path, starting_obj,referring_obj, scope, scope_nodes, trans):
2990
+ """Trans is a transaction that has a mutable graph snapshot.
2991
+
2992
+ scope_nodes is a mutable dictionary, which may be already partially prepopulated, indexed by node, of ScopeNode instances.
2993
+
2994
+ scope is an ONDEOpScope.
2995
+ starting_obj is the object corresponding to starting_path. It is not assumed to already be inserted into scope_nodes. It is assumed that starting_obj has already been checked against the exclude_objs of scope.
2996
+ starting_path is an ONDEPath indicating the full path of starting_obj. It is assumed that starting_path has been already checked against the include_paths of scope.
2997
+
2998
+ Walk the graph starting at the given starting_path, ignoring explicitly excluded paths and nodes from the scope, while accumulating nodes into scope_nodes, keeping track of which edges are fair game (represented by a set corresponding to the node in scope_nodes), or where all edges are fair game (represented by None instead of the set)
2999
+ """
3000
+
3001
+ snap = trans.snap
3002
+
3003
+ # Register starting_obj into scope_nodes
3004
+ if starting_obj in scope_nodes:
3005
+ new=False
3006
+ scope_node = scope_nodes[starting_obj]
3007
+ pass
3008
+ else:
3009
+ new=True
3010
+ scope_node = ScopeNode()
3011
+ scope_nodes[starting_obj] = scope_node
3012
+ pass
3013
+
3014
+ #if starting_scope_set is not None:
3015
+ # If we are traversing this object, then all edges are in scope, so we replace any set with None
3016
+
3017
+ scope_node.edges = None
3018
+ scope_node.referrers.add(referring_obj)
3019
+
3020
+ if new:
3021
+ edges = starting_obj._list_edges()
3022
+
3023
+ for edge in edges:
3024
+ current_path = ONDEPath(starting_path + (edge,))
3025
+ current_obj = starting_obj._follow_path(ONDEPath((edge,)))
3026
+
3027
+ if current_path in scope.exclude_paths:
3028
+ continue
3029
+
3030
+ if current_obj in scope.exclude_objects:
3031
+ continue
3032
+
3033
+ graph_replace_node__walk(current_path, current_obj, starting_obj,scope, scope_nodes, trans)
3034
+
3035
+ pass
3036
+ pass
3037
+ pass
3038
+
3039
+
3040
+ @dataclass
3041
+ class ReplacedNode(object):
3042
+ replacement: Optional[ONDEBase]=None
3043
+ refersto: Optional[set[ONDEBase]]=field(default_factory=set) # Set of objects this node refers to that we might want to replace.
3044
+
3045
+ pass
3046
+
3047
+
3048
+ def graph_replace_node__reversewalk(current_node,refersto,scope_nodes,changed_nodes):
3049
+ if current_node not in changed_nodes:
3050
+ new = True
3051
+ changed_nodes[current_node] = ReplacedNode()
3052
+ pass
3053
+ else:
3054
+ new = False
3055
+ pass
3056
+
3057
+ changed_nodes[current_node].refersto.update(refersto)
3058
+
3059
+ if new:
3060
+ referrers = scope_nodes[current_node].referrers
3061
+
3062
+ for referrer in referrers:
3063
+ if referrer in changed_nodes:
3064
+ continue
3065
+
3066
+ graph_replace_node__reversewalk(referrer,set([current_node]),scope_nodes,changed_nodes)
3067
+ pass
3068
+ pass
3069
+ pass
3070
+
3071
+ def graph_replace_node(trans, scope, path, orig_node, replacement_node):
3072
+ # to do: proposed algorithm:
3073
+ #
3074
+ # step 1: follow each starting location path to its end; then, continue to walk the graph, ignoring explicitly excluded (edges or paths?) while accumulating all nodes into a dict (indexed by pre-existing nodes) of scope_nodes and keeping track of which edges are "fair game" for each node (i.e. all edges, if we were walking the graph, or edges called out explicitly in a starting location path)
3075
+ snap = trans.snap
3076
+ scope_nodes = collections.OrderedDict()
3077
+ scope_nodes[snap] = ScopeNode() # Give the snapshot itself a blank ScopeNode
3078
+
3079
+ for starting_path in scope.include_paths:
3080
+ if starting_path in scope.exclude_paths:
3081
+ continue
3082
+
3083
+
3084
+ current_parent = snap
3085
+ current_path = ONDEPath(())
3086
+ pending_path = starting_path
3087
+ current_scope_set = scope_nodes[snap].edges
3088
+
3089
+ # Follow the starting path, element by element, because we need to accumulate all referrers into the scope dictionary.
3090
+ while len(current_path) < len(starting_path)-1:
3091
+ current_scope_set.add(pending_path[0])
3092
+ current_path = ONDEPath(current_path + (pending_path[0],))
3093
+
3094
+ current_parent_parent=current_parent
3095
+ current_parent = current_parent._follow_path(ONDEPath((pending_path[0],)))
3096
+ #print("current_parent=",str(current_path))
3097
+ pending_path = ONDEPath(pending_path[1:])
3098
+ if current_parent in scope_nodes:
3099
+ current_scope_node = scope_nodes[current_parent]
3100
+ current_scope_set = current_scope_node.edges
3101
+ pass
3102
+ else:
3103
+ current_scope_node = ScopeNode()
3104
+ current_scope_set = current_scope_node.edges
3105
+ scope_nodes[current_parent] = current_scope_node
3106
+ pass
3107
+ current_scope_node.referrers.add(current_parent_parent)
3108
+ pass
3109
+
3110
+ if current_scope_set is not None:
3111
+ current_scope_set.add(pending_path[0])
3112
+ pass
3113
+
3114
+
3115
+
3116
+ assert(starting_path[-1] == pending_path[0])
3117
+ starting_parent = current_parent
3118
+ starting_obj = starting_parent._follow_path(ONDEPath((starting_path[-1],)))
3119
+ if starting_obj in scope.exclude_objects:
3120
+ continue
3121
+
3122
+ graph_replace_node__walk(starting_path, starting_obj, starting_parent, scope, scope_nodes, trans)
3123
+ pass
3124
+
3125
+ # print("scope_nodes=",scope_nodes)
3126
+
3127
+
3128
+ #
3129
+ # step 2: identify the node to be changed within the set (if it's not included, the new node is not referenced)
3130
+ #
3131
+ if orig_node not in scope_nodes:
3132
+ return
3133
+ # step 3: reverse walk the set of nodes, starting at the node to be changed, identifying these nodes into a new (and probably smaller) dict called changed_nodes, the keys of which are a set of nodes through which the change will propagate while the values are ReplacedNode objects with the replacement set to None
3134
+ #
3135
+ changed_nodes = collections.OrderedDict()
3136
+
3137
+ current_node = orig_node
3138
+
3139
+ graph_replace_node__reversewalk(current_node,set([]),scope_nodes,changed_nodes)
3140
+
3141
+
3142
+ #print("changed_nodes=",changed_nodes)
3143
+
3144
+ # step 4: iterate through the second set creating a replacement for each where we update changed_nodes, populating each entry's value with the replacement stored in a ReplacedNode object
3145
+ #
3146
+ changed_node_indexes = list(changed_nodes.keys())
3147
+
3148
+ for changed_node in changed_node_indexes:
3149
+ if changed_node is not orig_node:
3150
+ # check to see if the changed node is frozen; if has modification scopes been cleared?
3151
+ if changed_node._modification_scopes is None:
3152
+ replacement = changed_node.__class__(_orig = changed_node)
3153
+ pass
3154
+ else:
3155
+ replacement = changed_node
3156
+ pass
3157
+
3158
+ replacement._modification_scopes.add(scope)
3159
+
3160
+ pass
3161
+ else:
3162
+ replacement = replacement_node
3163
+ pass
3164
+ changed_nodes[changed_node].replacement = replacement
3165
+ pass
3166
+
3167
+
3168
+
3169
+ #print("changed_nodes=",changed_nodes)
3170
+
3171
+
3172
+
3173
+ # step 5: iterate through the replacements, identifying every "fair game" reference to changed_nodes, and re-pointing that to the replacements
3174
+ #
3175
+
3176
+ for replaced_node in changed_nodes:
3177
+ replacement = changed_nodes[replaced_node].replacement
3178
+ scope_node = scope_nodes[replaced_node]
3179
+ fair_game_edges = scope_node.edges
3180
+ refersto = changed_nodes[replaced_node].refersto
3181
+ # refersto is a set of ONDEBase that includes all of the outgoing-referenced-objects referred to by replacement that may need to be repointed at a newly created copy that is findable by changed_nodes
3182
+ # Only those edges listed in fair_game_edges (or all edges if fair_game_edges is None) need to be swapped out.
3183
+ for dest in refersto:
3184
+ dest_indices = replaced_node._indices_for_object(dest)
3185
+ if fair_game_edges is not None:
3186
+ dest_indices = dest_indices.intersection(fair_game_edges)
3187
+ pass
3188
+ for dest_index in dest_indices:
3189
+ replacement._assign_pathel(dest_index, changed_nodes[dest].replacement)
3190
+ pass
3191
+ pass
3192
+ pass
3193
+
3194
+
3195
+ # step 6: the result is potential replacement for the entry point for each scope starting location
3196
+
3197
+ # implementation plan:
3198
+ #
3199
+ # we need a set of node, with each node having a set of "fair game" edges, stored globally for this operation as a dict, indexed by nodes with the values being either None (all possible edges) or a set of edges identifiers
3200
+ #
3201
+ # we also need a set of references of all scope nodes that point at any given scope node of intereset
3202
+ #
3203
+ # for the task above, we'll need a ScopeNode class that has a set of "fair game" edges (or None indicating that all edges are fair game) and it will need a set of referring nodes; as we assemble the scope_nodes dictionary, any time we find a node that we have seen before, we add to the referring node set of the ScopeNode, rather than creating a new ScopeNode
3204
+
3205
+ pass
3206
+
3207
+ class ONDEProxy(object):
3208
+ """Mutable proxy reference to a graph entry that remembers context."""
3209
+ _graph = None # ONDEFileGraph object we started with
3210
+ _path = None # ONDEPath of the object we are proxying
3211
+ # _obj = None # The actual object we are proxying
3212
+ # _obj_snap = None # Snapshot from which we obtained _obj
3213
+ _trans = None # Transaction may be none not inside a transaction
3214
+ _scope = None # A scope separate from the one in _trans (if None, use the scope from _trans)
3215
+ _snap = None # Snapshot; only used if there is no transaction
3216
+
3217
+ def __init__(self, **kwargs):
3218
+ # _dict = object.__getattribute__(self, "__dict__")
3219
+
3220
+ for arg in kwargs:
3221
+ if arg in type(self).__dict__:
3222
+ # setattr(self, arg, kwargs[arg])
3223
+ object.__setattr__(self, arg, kwargs[arg])
3224
+
3225
+ pass
3226
+ else:
3227
+ raise ValueError(f"ONDEProxy; unknown attribute {arg:s}")
3228
+ pass
3229
+ pass
3230
+
3231
+ @classmethod
3232
+ def new_from_proxy(cls, parent, subpath):
3233
+ _graph = object.__getattribute__(parent, "_graph")
3234
+
3235
+ # parent_obj = object.__getattribute__(parent, "_obj")
3236
+ # _obj = parent_obj._get_attr(attr_name)
3237
+ # _obj_snap = object.__getattribute__(parent, "_obj_snap")
3238
+
3239
+ _parent_path = object.__getattribute__(parent, "_path")
3240
+ path = ONDEPath(_parent_path + (subpath,))
3241
+
3242
+ _trans = object.__getattribute__(parent, "_trans")
3243
+
3244
+ return cls(_graph=_graph, _path=path, _trans=_trans)
3245
+
3246
+ @classmethod
3247
+ def new_from_snapshot(cls, graph, snapshot):
3248
+ path = ONDEPath()
3249
+ return cls(_graph=graph, _path=path, _snap=snapshot)
3250
+
3251
+ @classmethod
3252
+ def new_from_scope(cls, scope):
3253
+ _trans = scope.trans
3254
+ _graph = _trans.graph
3255
+ _path = ONDEPath()
3256
+
3257
+ return cls(_graph=_graph, _path=_path, _trans=_trans,_scope=scope)
3258
+
3259
+ def _get_obj(self):
3260
+ trans = object.__getattribute__(self, "_trans")
3261
+
3262
+ if trans is not None:
3263
+ snap = trans.snap
3264
+ pass
3265
+
3266
+ else:
3267
+ graph = object.__getattribute__(self, "_graph")
3268
+ snap = graph.latest_snap
3269
+ pass
3270
+
3271
+ _path = object.__getattribute__(self, "_path")
3272
+ return snap._follow_path(_path)
3273
+
3274
+ def __getattribute__(self, name):
3275
+ if name.startswith("_"):
3276
+ if name in { "_set_attr","_set_dataset_attr", "_get_attr","_get_attr_dataset_storage","_get_item","_set_item","_get_data","_set_data","_set_attr_or_item_or_data","_get_obj","_follow_path","__class__","__dict__","_graph","_repr","_repr_short","_add_dataset"}:
3277
+ return object.__getattribute__(self, name)
3278
+ elif name in {"_freeze", "_frozen",}:
3279
+ obj = self._get_obj()
3280
+ return getattr(obj,name)
3281
+ raise ValueError(f"ONDEProxy: invalid underscore attribute {name}")
3282
+ obj = self._get_obj()
3283
+ if isinstance(obj,ONDEFileGraphSnapshot):
3284
+ # only attribute of a snapshot is .add(),
3285
+ # which we redirect to ourselves as ._add()
3286
+ if name == "add":
3287
+ return object.__getattribute__(self, "_add")
3288
+ elif name=="keys":
3289
+ return object.__getattribute__(obj, "keys")
3290
+ raise IndexError(f"No attribute {name:s} of a proxy for ONDEFileGraphSnapshot")
3291
+
3292
+ if isinstance(obj,ONDEObject):
3293
+ if isinstance(obj,ONDEFileGraphSnapshot) and name == "add":
3294
+ name = "_add"
3295
+ pass
3296
+
3297
+ _get_attr = object.__getattribute__(self, "_get_attr")
3298
+ return _get_attr(name)
3299
+
3300
+ #elif isinstance(obj,ONDEReferenceArray):
3301
+ # self._set_item(name,value)
3302
+ # pass
3303
+ else:
3304
+ assert(isinstance(obj,ONDEBase))
3305
+ _get_data = object.__getattribute__(self, "_get_data")
3306
+ return _get_data(name)
3307
+
3308
+ pass
3309
+
3310
+ def __setattr__(self,name,value):
3311
+ _set_attr = object.__getattribute__(self, "_set_attr")
3312
+ obj = self._get_obj()
3313
+ if isinstance(obj,ONDEObject):
3314
+ self._set_attr(name,value)
3315
+ pass
3316
+ #elif isinstance(obj,ONDEReferenceArray):
3317
+ # self._set_item(name,value)
3318
+ # pass
3319
+ else:
3320
+ assert(isinstance(obj,ONDEBase))
3321
+ self._set_data(name,value)
3322
+ pass
3323
+ pass
3324
+
3325
+ def _get_attr(self, name):
3326
+ obj = self._get_obj()
3327
+ attr_obj = obj._get_attr(name)
3328
+
3329
+ assert(isinstance(attr_obj, ONDEBase))
3330
+ return self.__class__.new_from_proxy(self, name)
3331
+
3332
+ #return attr_obj
3333
+
3334
+ def _get_attr_dataset_storage(self, name):
3335
+ obj = self._get_obj()
3336
+ return obj._get_attr_dataset_storage(name)
3337
+
3338
+
3339
+ def _has_attr(self, name):
3340
+ obj = self._get_obj()
3341
+ return obj._has_attr(name)
3342
+
3343
+
3344
+ def _set_attr(self, name, value):
3345
+ self._set_attr_or_item_or_data('_set_attr', name, value)
3346
+ pass
3347
+
3348
+ def _add(self,dataset):
3349
+ """add() method for proxying ONDEFileGraphSnapshot"""
3350
+ # Must maintain parallel logic in ONDEFileGraphSnapshot.add()
3351
+ obj=dataset
3352
+ if not isinstance(obj,ONDEBase):
3353
+ obj = dataset._get_obj()
3354
+ pass
3355
+ if "ONDE:TYPE" not in obj or obj["ONDE:TYPE"].value[0] != "ONDE_DATASET":
3356
+ raise ValueError(f"Attempting to add something other than a dataset to an ONDEFileGraphSnapshot")
3357
+
3358
+ if "ONDE:UUID" not in dataset:
3359
+ raise ValueError(f"Attempting to add a dataset without a UUID to an ONDEFileGraphSnapshot")
3360
+ self._set_attr_or_item_or_data("_add_dataset",dataset["ONDE:UUID"].value,dataset)
3361
+ pass
3362
+
3363
+ def _set_dataset_attr(self, name, value):
3364
+ self._set_attr_or_item_or_data('_set_dataset_attr', name, value)
3365
+ pass
3366
+
3367
+
3368
+ def _get_item(self, key):
3369
+ obj = self._get_obj()
3370
+ item_obj = obj._get_item(key)
3371
+
3372
+ if isinstance(item_obj, ONDEBase):
3373
+
3374
+ # if obj is an ONDEFileGraphSnapshot, then we have flexible indexing with shorthands but for the proxy we need the real key
3375
+ if isinstance(obj, ONDEFileGraphSnapshot):
3376
+ item_obj_keys = [ k for k in obj.keys() if obj._get_item(key) is item_obj ]
3377
+ assert (len(item_obj_keys)==1)
3378
+ key = item_obj_keys[0]
3379
+ pass
3380
+
3381
+ return self.__class__.new_from_proxy(self, key)
3382
+
3383
+ return item_obj
3384
+
3385
+ def _set_item(self, index, value):
3386
+ self._set_attr_or_item_or_data('_set_item', index, value)
3387
+ pass
3388
+
3389
+ def __getitem__(self,index):
3390
+ return self._get_item(index)
3391
+
3392
+ def __setitem__(self,index,value):
3393
+ return self._set_item(index,value)
3394
+
3395
+ def _get_data(self, name):
3396
+ obj = self._get_obj()
3397
+ data_obj = obj._get_data(name)
3398
+
3399
+ return data_obj
3400
+
3401
+ def _set_data(self, name, value):
3402
+ self._set_attr_or_item_or_data('_set_data', name, value)
3403
+ pass
3404
+
3405
+ def _add_dataset(self,key,dataset):
3406
+ assert(key==dataset["ONDE:UUID"].value)
3407
+ self._set_attr_or_item_or_data("_add_dataset",key,dataset)
3408
+ pass
3409
+
3410
+ def _set_attr_or_item_or_data(self, set_method_name, key, value):
3411
+ ''' Use the named set method (_set_attr,_set_dataset_attr, _set_item, or _set_data) to assign the element specified by key to the given value'''
3412
+
3413
+ _trans = object.__getattribute__(self, "_trans")
3414
+ #import pdb
3415
+ #pdb.set_trace()
3416
+
3417
+ if _trans is None:
3418
+ _graph = object.__getattribute__(self, "_graph")
3419
+
3420
+ transaction = ONDETransaction(_graph)
3421
+
3422
+ with transaction as scope:
3423
+ proxy=scope.graph
3424
+ # proxy now has a potentially updated snapshot that we
3425
+
3426
+ # should use for the transaction. Need to use our path to
3427
+ # recreate our object and then call _set_attr on that.
3428
+ path = object.__getattribute__(self,"_path")
3429
+ newproxy = proxy._follow_path(path)
3430
+ set_method = getattr(newproxy, set_method_name) # extract the bound method e.g. newproxy._set_attr
3431
+ set_method(key, value)
3432
+ pass
3433
+
3434
+ pass
3435
+
3436
+ else:
3437
+ # create replacement node that has the desired change
3438
+ # obj = object.__getattribute__(self, "_obj")
3439
+
3440
+ # create a new object of obj's class, passing the original that we want to copy as its first constructor parameter; the ONDE classes are built to handle this
3441
+ _get_obj = object.__getattribute__(self, "_get_obj")
3442
+ obj = _get_obj()
3443
+ scope = object.__getattribute__(self, "_scope")
3444
+ path = object.__getattribute__(self, "_path")
3445
+
3446
+ if scope is None:
3447
+ scope = _trans.scope
3448
+ pass
3449
+
3450
+ if scope.include_paths is None:
3451
+ # Null scope: use most constrained scope for each op
3452
+ scope = ONDEOpScope(_trans,[path])
3453
+ pass
3454
+
3455
+ # if obj._frozen:
3456
+ if obj._modification_scopes is None:
3457
+ replacement = obj.__class__(_orig=obj)
3458
+ # object.__setattr__(replacement, "_modification_scopes", set([scope]))
3459
+ replacement._modification_scopes.add(scope)
3460
+ new = True
3461
+ pass
3462
+ else:
3463
+ replacement = obj
3464
+ new = False
3465
+ pass
3466
+
3467
+ set_method = getattr(replacement, set_method_name) # e.g. replacement._set_attr
3468
+ set_method(key, value)
3469
+ #replacement._freeze() Freezing happens at the end of the transaction
3470
+ #if new: # We could avoid the replacement for preexisting modifications if we knew that the scope of the previous modification matched our scope.
3471
+ if new or scope not in obj._modification_scopes:
3472
+ replacement._modification_scopes.add(scope)
3473
+ if len(path) > 0:
3474
+ graph_replace_node(_trans, scope, path, obj, replacement)
3475
+ pass
3476
+ else:
3477
+ assert(obj is _trans.snap and replacement is _trans.snap)
3478
+ pass
3479
+
3480
+ pass
3481
+
3482
+ pass
3483
+
3484
+ pass
3485
+
3486
+ def _follow_path(self,path):
3487
+ obj = self._get_obj()
3488
+ dest=obj._follow_path(path)
3489
+
3490
+
3491
+ _graph = object.__getattribute__(self, "_graph")
3492
+
3493
+ _our_path = object.__getattribute__(self, "_path")
3494
+ full_path = ONDEPath(_our_path + path)
3495
+
3496
+ _trans = object.__getattribute__(self, "_trans")
3497
+
3498
+ return self.__class__(_graph=_graph, _path=full_path, _trans=_trans)
3499
+
3500
+ def _repr_short(self, onde_class=None):
3501
+ return object.__getattribute__(self._get_obj(), "_repr_short")(onde_class)
3502
+
3503
+ def _repr(self, indentation, onde_class=None, extra_attrs={}):
3504
+ return repr_helper(indentation, f"Writeable proxy of {object.__getattribute__(self._get_obj(), '_repr')(indentation+2, onde_class=onde_class, extra_attrs=extra_attrs):s}")
3505
+
3506
+ def __repr__(self):
3507
+ return f"Writeable proxy of {repr(self._get_obj()):s}"
3508
+ pass
3509
+
3510
+
3511
+ def _traverse_snapshot_fileneeded_objs(class_defs,cur_obj,add_cur_to_needed,needed_objs):
3512
+ """Traverse the graph of ONDEBase objects depth-first starting at cur_obj, accumulating ONDEObjects that will need to be written into the file into the ordered dictionary needed_objs. It is an ordered dictionary because we will use its ordering for the object creation so that objects are created before they are used. The values stored into the ordered dictionary are all None."""
3513
+ edges = cur_obj._list_edges()
3514
+
3515
+ for edge in edges:
3516
+ sub_obj = cur_obj._follow_path((edge,))
3517
+ if sub_obj in needed_objs:
3518
+ continue # sub_obj has already been processed
3519
+ sub_is_needed = type(sub_obj) is ONDEObject
3520
+ _traverse_snapshot_fileneeded_objs(class_defs,sub_obj,sub_is_needed,needed_objs)
3521
+ pass
3522
+
3523
+ if add_cur_to_needed:
3524
+ needed_objs[cur_obj] = None
3525
+ pass
3526
+ pass
3527
+
3528
+
3529
+ def _search_h5_for_onde_datasets(h5_fh,h5_group,dataset_h5_groups,h5_groups_seen):
3530
+ """ dataset_h5_groups is a dictionary by hdf5 path of hdf5 group objects that contain an ONDE:TYPE attribute starting with ONDE_DATASET
3531
+
3532
+
3533
+ h5_groups_seen is a set of hdf5 group paths seen during the traversal
3534
+ """
3535
+
3536
+ if "ONDE:TYPE" in h5_group.attrs and h5_group.attrs["ONDE:TYPE"][0] == "ONDE_DATASET":
3537
+ dataset_h5_groups[h5_group.name] = h5_group
3538
+ pass
3539
+
3540
+ h5_groups_seen.add(h5_group.name)
3541
+
3542
+ for (name,item) in h5_group.items():
3543
+ if isinstance(item,h5py.Group) and item.name not in h5_groups_seen:
3544
+ _search_h5_for_onde_datasets(h5_fh,item,dataset_h5_groups,h5_groups_seen)
3545
+ pass
3546
+ pass
3547
+ pass
3548
+
3549
+ class ONDEFile(object):
3550
+ """Represents an HDF5 file that may contain ONDE objects. Unlike most other data structures it is not thread safe (only one thread at a time should be manipulating an ONDEFile). However it is safe for multiple threads to simultaneously access and even modify (by the usual ONDEProxy methods) objects from a single file, so long as only a single thread attempts to write any changes to disk."""
3551
+ file_object_dict = None # Dictionary by frozen ONDEBase object of ONDEFileObject.
3552
+ class_defs = None # ONDEClassDefinitions object
3553
+ h5path = None # file path used
3554
+ mode = None # mode used to open the file
3555
+ fh = None # h5py file handle.
3556
+ graph = None # ONDEFileGraph object (or subclass)
3557
+ db_maxidx = None # integer representing the highest index used in the PyONDE_DB hdf5 group
3558
+ version = None
3559
+ filetype = None
3560
+
3561
+ def __init__(self, class_defs = None, h5path = None, mode = None, fh = None,version = None, filetype = None):
3562
+ self.class_defs = class_defs
3563
+ self.h5path = h5path
3564
+ self.mode = mode
3565
+ self.fh = fh
3566
+ self.version = version
3567
+ self.filetype = filetype
3568
+
3569
+ self.file_object_dict = {}
3570
+ self.graph = ONDEFileGraph(class_defs = self.class_defs)
3571
+
3572
+ if self.mode not in {"r","r+","w","w-","x","a"}:
3573
+ raise ValueError(f"Unknown mode: {self.mode:s}")
3574
+
3575
+ if self.mode == "r+" or self.mode == "a" or self.mode == "r":
3576
+
3577
+ if version is not None:
3578
+ raise ValueError("Attempting to set ONDE_VERSION on existing file.")
3579
+ if filetype is not None:
3580
+ raise ValueError("Attempting to set ONDE_FILETYPE on existing file.")
3581
+ self.load() # Attempt to read in current contents.
3582
+ pass
3583
+
3584
+ pass
3585
+
3586
+ def __getitem__(self,index):
3587
+ return self.graph._get_item(index)
3588
+
3589
+ def __setitem__(self,index,value):
3590
+ return self.graph._set_item(index,value)
3591
+
3592
+ def keys(self):
3593
+ return self.graph.keys()
3594
+
3595
+ def __iter__(self):
3596
+ return iter(self.keys())
3597
+
3598
+ def load(self):
3599
+ self.version = self.fh.attrs["ONDE_VERSION"]
3600
+ self.filetype = set(self.fh.attrs["ONDE_FILETYPE"])
3601
+
3602
+ # check for PyONDE_DB and db_maxidx
3603
+ self.db_maxidx = 0
3604
+ if "PyONDE_DB" in self.fh:
3605
+ db = self.fh["PyONDE_DB"]
3606
+ for db_key in db.keys():
3607
+ if db_key.isdigit():
3608
+ db_keynum = int(db_key)
3609
+ if db_keynum > self.db_maxidx:
3610
+ self.db_maxidx = db_keynum
3611
+ pass
3612
+ pass
3613
+ pass
3614
+ pass
3615
+ #import pdb
3616
+ #pdb.set_trace()
3617
+ # Search through group structure for datasets.
3618
+ dataset_h5_groups = collections.OrderedDict() # dictionary by hdf5 path of hdf5 group objects that contain an ONDE:TYPE attribute starting with ONDE_DATASET
3619
+ h5_groups_seen = set() # set of hdf5 group paths seen during the traversal
3620
+ _search_h5_for_onde_datasets(self.fh,self.fh,dataset_h5_groups,h5_groups_seen)
3621
+
3622
+ fileobjs_by_h5path = {}
3623
+
3624
+ dataset_instances_by_h5path = collections.OrderedDict()
3625
+ for group_path in dataset_h5_groups:
3626
+ dataset_instances_by_h5path[group_path] = ONDEObject.load_from_hdf5(self,fileobjs_by_h5path,self.fh,dataset_h5_groups[group_path])
3627
+ pass
3628
+
3629
+ # replace existing contents of self.group as an atomic transaction
3630
+ with ONDETransaction(self.graph) as tr:
3631
+ for key in list(tr.graph.keys()):
3632
+ del tr.graph[key]
3633
+ pass
3634
+
3635
+ for h5path in dataset_instances_by_h5path:
3636
+
3637
+ instance = dataset_instances_by_h5path[h5path]
3638
+ if not "ONDE:UUID" in instance:
3639
+ raise ValueError(f"Error loading dataset from file {self.h5path:s} at hdf5 path {h5path:s}: No ONDE:UUID field specified")
3640
+
3641
+ # unique_id = instance["ONDE:UUID"].value
3642
+ # tr.graph[str(unique_id)] = instance
3643
+ tr.graph.add(instance)
3644
+ pass
3645
+ pass
3646
+ pass
3647
+
3648
+
3649
+
3650
+ def flush(self):
3651
+ snap = self.graph.latest_snap
3652
+
3653
+ # Ensure ONDE_FILETYPE and ONDE_VERSION are present
3654
+ if "ONDE_VERSION" not in self.fh.attrs:
3655
+ if self.version is not None:
3656
+ self.fh.attrs["ONDE_VERSION"] = self.version
3657
+ pass
3658
+ else:
3659
+ self.fh.attrs["ONDE_VERSION"] = self.class_defs.version
3660
+ pass
3661
+ pass
3662
+
3663
+ if "ONDE_FILETYPE" not in self.fh.attrs:
3664
+ if self.filetype is not None:
3665
+ self.fh.attrs["ONDE_FILETYPE"] = list(self.filetype)
3666
+ pass
3667
+ else:
3668
+ self.fh.attrs["ONDE_FILETYPE"] = ["ONDE_DATASET"]
3669
+ pass
3670
+ pass
3671
+
3672
+ # Make sure PyONDE_DB HDF5 group exists.
3673
+ if not "PyONDE_DB" in self.fh:
3674
+ self.fh.create_group("PyONDE_DB")
3675
+ self.db_maxidx = 0
3676
+ pass
3677
+
3678
+ db=self.fh["PyONDE_DB"]
3679
+ db_path = db.name
3680
+
3681
+ present_objs = set(self.file_object_dict.keys()) # set of ONDEBase objects already present in the file
3682
+ needed_objs = collections.OrderedDict()
3683
+ _traverse_snapshot_fileneeded_objs(self.class_defs,snap,False,needed_objs) # this function will fill needed_objs with an ordered set (implemented as keys of an ordered dictionary) of ONDEBase objects that are needed in the file. The ordering is depth-first so that leaf nodes get included prior to branch nodes that reference them. The False parameter indicates that the snap itself does NOT get included.
3684
+
3685
+ needed_objs_set = set(needed_objs.keys())
3686
+ # We need to remove objects that are listed in present_objs but not needed_objs
3687
+ to_remove = present_objs.difference(needed_objs_set)
3688
+
3689
+ # We need to add objects that are listed in needed_objs but not present_objs
3690
+ to_add = needed_objs_set.difference(present_objs)
3691
+
3692
+ # Perform the removal
3693
+ for obj in to_remove:
3694
+ fileobj = self.file_object_dict[obj]
3695
+
3696
+ #if fileobj.hdf5_attrname is not None:
3697
+ # # Object is stored as an attribute ... unnecessary becuase attributes will be written as part of their parent object.
3698
+ # attr_parent = self.fh[fileobj.hdf5_path]
3699
+ # del attr_parent[fileobj.hdf5_attrname]
3700
+ # pass
3701
+ #else:
3702
+ hdf5_obj=self.fh[fileobj.hdf5_path]
3703
+ hdf5_parent = hdf5_obj.parent
3704
+ del hdf5_parent[posixpath.basename(hdf5_obj.name)]
3705
+
3706
+
3707
+ fileobj.close()
3708
+ del self.file_object_dict[obj]
3709
+ pass
3710
+
3711
+
3712
+ # add in the needed objects. Loop ordering is because needed_objs is actually an OrderedDict.
3713
+ for needed_obj in needed_objs:
3714
+ if not needed_obj in to_add:
3715
+ continue
3716
+
3717
+ obj_name = f"{self.db_maxidx + 1:05d}"
3718
+ self.db_maxidx += 1
3719
+
3720
+ fileobj = ONDEFileObject(onde_instance = needed_obj,hdf5_path = posixpath.join(db_path,obj_name))
3721
+
3722
+ needed_obj._hdf5_write_group(onde_file = self,parent = db,parent_path = db_path,name = obj_name,onde_fileobj = fileobj)
3723
+
3724
+ self.file_object_dict[needed_obj] = fileobj
3725
+
3726
+ pass
3727
+
3728
+
3729
+ self.fh.flush()
3730
+ pass
3731
+
3732
+
3733
+
3734
+ def close(self):
3735
+
3736
+ if self.mode in {"r+","w","w-","x","a"}:
3737
+ self.flush()
3738
+ pass
3739
+
3740
+ self.fh.close()
3741
+ for key in self.file_object_dict:
3742
+ self.file_object_dict[key].close()
3743
+ pass
3744
+ self.file_object_dict = None
3745
+ self.fh = None
3746
+ pass
3747
+
3748
+ def __enter__(self):
3749
+ return self
3750
+
3751
+ def __exit__(self, exc_type, exc_value, tb):
3752
+ self.close()
3753
+ return False
3754
+
3755
+ def _repr(self, indentation, onde_class=None, extra_attrs={}):
3756
+ return repr_helper(indentation, "ONDEFile", _extra_info = f"open on \"{self.h5path:s}\"")+"\n"+self.graph._repr(indentation+2)
3757
+
3758
+ def __repr__(self):
3759
+ return self._repr(0)
3760
+
3761
+ @classmethod
3762
+ def new(cls, h5path, mode, class_defs_path=None, extra_class_defs = None, onde_version = None):
3763
+ fh = h5py.File(h5path, mode)
3764
+
3765
+ class_defs = None
3766
+ if class_defs_path is None and onde_version is not None:
3767
+ class_defs_path = os.path.join(os.path.split(sys.modules[cls.__module__].__file__)[0],"onde_versions",f"ONDE_fields_v{onde_version:s}.csv")
3768
+ pass
3769
+ if class_defs_path is not None:
3770
+ class_defs = ONDEClassDefinitions.load_from_csv(class_defs_path,extra_class_defs = extra_class_defs)
3771
+ pass
3772
+
3773
+ return cls(class_defs = class_defs, h5path = h5path, mode = mode, fh = fh)
3774
+
3775
+ pass
3776
+
3777
+ class ONDEDatasetFile(ONDEFile):
3778
+ """Represents an ONDE 1.0 file that is expected to contain one or more ONDE_DATASET instances."""
3779
+ pass
3780
+
3781
+ class ONDEFileObject(object):
3782
+ """Represents a frozen ONDEBase subclass instance that is represented in an HDF5 file."""
3783
+ onde_instance = None # Reference to an ONDEBase subclass instance.
3784
+ hdf5_path = None # Path of the given object in the file.
3785
+ #hdf5_attrname = None # If this FileObject is stored as an attribute, then this is the attribute name. Unnecessary because attributes will be written as part of their parent object.
3786
+ # hdf5_obj = None # Actual h5py object.
3787
+
3788
+ def __init__(self, onde_instance = None, hdf5_path = None): #, hdf5_obj = None):
3789
+ self.onde_instance = onde_instance
3790
+ self.hdf5_path = hdf5_path
3791
+ #self.hdf5_obj = hdf5_obj
3792
+
3793
+ with self.onde_instance._file_realizations_lock:
3794
+ self.onde_instance._file_realizations.add(self)
3795
+ pass
3796
+
3797
+ pass
3798
+
3799
+ def close(self):
3800
+ self.hdf5_path = None
3801
+ #self.hdf5_obj = None
3802
+
3803
+ with self.onde_instance._file_realizations_lock:
3804
+ self.onde_instance._file_realizations.remove(self)
3805
+ pass
3806
+ self.onde_instance = None
3807
+ pass
3808
+ pass
3809
+
3810
+
3811
+
3812
+ # rough api of transactions
3813
+ # with ONDETransaction(graph, include_scope, exclude_scope) as transaction:
3814
+ # transaction.obj.attr = "hello"
3815
+
3816
+
3817
+ #class ONDEOperation(object):
3818
+ # """Represents an operation that can be part of a transaction"""
3819
+ # scope = None # ONDEOpScope reference or None representing default (local) scope
3820
+ # pass
3821
+
3822
+
3823
+ #class ONDEAssignValue(ONDEOperation):
3824
+ # """Represents assignment of the value within an ONDEValue object"""
3825
+ # attr_name = None # Usually "value"
3826
+ # attr_value = None # String, int, float, immutable array, etc.
3827
+ # pass