sdata 1.5.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. sdata/__init__.py +140 -0
  2. sdata/base.py +684 -0
  3. sdata/contrib/__init__.py +0 -0
  4. sdata/contrib/piexif/__init__.py +12 -0
  5. sdata/contrib/piexif/_common.py +94 -0
  6. sdata/contrib/piexif/_dump.py +346 -0
  7. sdata/contrib/piexif/_exceptions.py +2 -0
  8. sdata/contrib/piexif/_exif.py +638 -0
  9. sdata/contrib/piexif/_insert.py +60 -0
  10. sdata/contrib/piexif/_load.py +270 -0
  11. sdata/contrib/piexif/_remove.py +56 -0
  12. sdata/contrib/piexif/_transplant.py +45 -0
  13. sdata/contrib/piexif/_webp.py +240 -0
  14. sdata/contrib/piexif/helper.py +66 -0
  15. sdata/contrib/ranger/__init__.py +8 -0
  16. sdata/contrib/ranger/collections/__init__.py +0 -0
  17. sdata/contrib/ranger/collections/rangebucketmap.py +346 -0
  18. sdata/contrib/ranger/collections/rangemap.py +396 -0
  19. sdata/contrib/ranger/collections/rangeset.py +483 -0
  20. sdata/contrib/ranger/range/__init__.py +0 -0
  21. sdata/contrib/ranger/range/cut.py +285 -0
  22. sdata/contrib/ranger/range/range.py +644 -0
  23. sdata/contrib/sobol_seq.py +394 -0
  24. sdata/contrib/sortedcontainers/LICENSE +13 -0
  25. sdata/contrib/sortedcontainers/__init__.py +74 -0
  26. sdata/contrib/sortedcontainers/sorteddict.py +789 -0
  27. sdata/contrib/sortedcontainers/sortedlist.py +2646 -0
  28. sdata/contrib/sortedcontainers/sortedset.py +733 -0
  29. sdata/contrib/sqlitedict.py +598 -0
  30. sdata/contrib/timeflake/__init__.py +40 -0
  31. sdata/contrib/timeflake/flake.py +69 -0
  32. sdata/contrib/timeflake/utils.py +47 -0
  33. sdata/deprecated/__init__.py +10 -0
  34. sdata/deprecated/data.py +1503 -0
  35. sdata/did/__init__.py +129 -0
  36. sdata/did/_http.py +67 -0
  37. sdata/did/base58btc.py +49 -0
  38. sdata/did/did_github.py +152 -0
  39. sdata/did/did_peer4.py +155 -0
  40. sdata/did/did_web.py +222 -0
  41. sdata/did/diddoc.py +75 -0
  42. sdata/did/ed25519.py +225 -0
  43. sdata/did/eddsa.py +107 -0
  44. sdata/did/errors.py +40 -0
  45. sdata/did/issue_vc.py +193 -0
  46. sdata/did/jose.py +126 -0
  47. sdata/did/keys.py +104 -0
  48. sdata/did/utils_didkey.py +161 -0
  49. sdata/did/verify_vp.py +158 -0
  50. sdata/doe.py +101 -0
  51. sdata/dtypes.py +510 -0
  52. sdata/experiments/__init__.py +331 -0
  53. sdata/experiments/blob0.py +131 -0
  54. sdata/experiments/file.py +42 -0
  55. sdata/experiments/filegroup.py +144 -0
  56. sdata/experiments/image_old.py +313 -0
  57. sdata/experiments/ks2.py +15 -0
  58. sdata/experiments/lapshear.py +8 -0
  59. sdata/experiments/material.py +7 -0
  60. sdata/experiments/parameters_deprecated.py +394 -0
  61. sdata/experiments/workbook.py +226 -0
  62. sdata/format.py +125 -0
  63. sdata/imagemeta.py +533 -0
  64. sdata/interactive.py +118 -0
  65. sdata/iolib/__init__.py +51 -0
  66. sdata/iolib/hashtree.py +264 -0
  67. sdata/iolib/json1sqlitestore.py +554 -0
  68. sdata/iolib/jsonsqlitestore.py +704 -0
  69. sdata/iolib/pud.py +242 -0
  70. sdata/iolib/reader.py +262 -0
  71. sdata/iolib/writer.py +419 -0
  72. sdata/metadata.py +882 -0
  73. sdata/parameter.py +414 -0
  74. sdata/schema.py +273 -0
  75. sdata/sclass/__init__.py +111 -0
  76. sdata/sclass/bfo_classes.py +362 -0
  77. sdata/sclass/blob.py +362 -0
  78. sdata/sclass/content.py +87 -0
  79. sdata/sclass/dataframe.py +1251 -0
  80. sdata/sclass/dataframegroup.py +187 -0
  81. sdata/sclass/dependency_graph.py +85 -0
  82. sdata/sclass/filereference.py +124 -0
  83. sdata/sclass/image.py +218 -0
  84. sdata/sclass/lazy_namespace.py +155 -0
  85. sdata/sclass/process.py +383 -0
  86. sdata/sclass/process_graph.py +221 -0
  87. sdata/sclass/zipfile.py +135 -0
  88. sdata/semantic.py +346 -0
  89. sdata/suuid.py +136 -0
  90. sdata/timestamp.py +336 -0
  91. sdata/tools.py +14 -0
  92. sdata/units.py +602 -0
  93. sdata/vocab.py +149 -0
  94. sdata-1.5.0.dist-info/METADATA +264 -0
  95. sdata-1.5.0.dist-info/RECORD +98 -0
  96. sdata-1.5.0.dist-info/WHEEL +5 -0
  97. sdata-1.5.0.dist-info/licenses/LICENSE-MIT +21 -0
  98. sdata-1.5.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1503 @@
1
+ # -*-coding: utf-8-*-
2
+ from __future__ import division
3
+
4
+ import sdata
5
+
6
+ '''
7
+ basic sdata types
8
+ '''
9
+ from sdata import __version__
10
+ import sys
11
+ import os
12
+ import uuid
13
+ from collections import OrderedDict
14
+ import logging
15
+ logger = logging.getLogger("sdata")
16
+ import numpy as np
17
+ import pandas as pd
18
+ import shutil
19
+ import copy
20
+ from sdata.metadata import Metadata, Attribute, extract_name_unit
21
+ from sdata.timestamp import now_utc_str, now_local_str, today_str
22
+ import inspect
23
+ import json
24
+ import hashlib
25
+ import base64
26
+ import re
27
+ from sdata.contrib.sqlitedict import SqliteDict
28
+
29
+ if sys.version_info < (3, 0):
30
+ from StringIO import StringIO
31
+ else:
32
+ from io import BytesIO, StringIO
33
+
34
+ class Sdata_Name_Exeption(Exception): pass
35
+ class Sdata_Uuid_Exeption(Exception): pass
36
+
37
+ if sys.version_info < (3, 6):
38
+ import sha3
39
+
40
+ try:
41
+ import openpyxl
42
+ except:
43
+ logger.warning("openpyxl is not available -> no xlsx import")
44
+
45
+ def uuid_from_str(name):
46
+ return uuid.uuid3(uuid.NAMESPACE_DNS, name)
47
+
48
+ def nameunit_from_colname(column_name):
49
+ """
50
+
51
+ """
52
+ #pattern = r"^(.*?)\s*\[(.+)\]$"
53
+ pattern = r"^(.*?)\s*\[([^\]]+)\]$"
54
+
55
+ # Anwenden des Regex
56
+ match = re.match(pattern, column_name)
57
+
58
+ if match:
59
+ name = match.group(1) # Der Name der Spalte
60
+ unit = match.group(2) # Die Einheit
61
+ else:
62
+ name = column_name
63
+ unit = "-"
64
+ return name.strip(), unit.strip()
65
+
66
+ class Data(object):
67
+ """Base sdata object"""
68
+ ATTR_NAMES = []
69
+
70
+ SDATA_VERSION = "!sdata_version"
71
+ SDATA_NAME = "!sdata_name"
72
+ SDATA_SNAME = "!sdata_sname"
73
+ SDATA_UUID = "!sdata_uuid"
74
+ SDATA_SUUID = "!sdata_suuid"
75
+ SDATA_CTIME = "!sdata_ctime"
76
+ SDATA_MTIME = "!sdata_mtime"
77
+ SDATA_PARENT = "!sdata_parent"
78
+ SDATA_CLASS = "!sdata_class"
79
+ SDATA_PROJECT = "!sdata_project"
80
+ SDATA_URL = "!sdata_url"
81
+ SDATA_COLUMN = "!sdata_column"
82
+
83
+ SDATA_ATTRIBUTES = [SDATA_VERSION, SDATA_NAME, SDATA_UUID, SDATA_SUUID, SDATA_CLASS, SDATA_PARENT, SDATA_PROJECT,
84
+ SDATA_CTIME, SDATA_MTIME, SDATA_URL]
85
+
86
+ def __init__(self, **kwargs):
87
+ """create Data object
88
+
89
+ .. code-block:: python
90
+
91
+ df = pd.DataFrame([1,2,3])
92
+ data = sdata.Data(name='my name',
93
+ uuid='38b26864e7794f5182d38459bab85842',
94
+ table=df,
95
+ description="A remarkable description")
96
+
97
+
98
+ :param name: name of the data object
99
+ :param table: pandas.DataFrame to store
100
+ :param uuid: uuid of the object
101
+ :param metadata: sdata.Metadata object
102
+ :param description: a string to describe the object
103
+ """
104
+
105
+ # RFC 0012: Data ist deprecated -> sdata.sclass.dataframe.DataFrame.
106
+ # Nur bei DIREKTER Instanziierung warnen, damit Domänen-/Experiment-
107
+ # Subklassen (Pud, TestProgram, …) nicht gestört werden (deren Ablösung
108
+ # ist Teil von Stufe 2 / 2.0).
109
+ if type(self) is Data:
110
+ import warnings
111
+ warnings.warn(
112
+ "sdata.Data is deprecated (RFC 0012); use "
113
+ "sdata.sclass.dataframe.DataFrame. Removal in sdata 2.0.",
114
+ DeprecationWarning, stacklevel=2)
115
+
116
+ # self._uuid = None
117
+ # self._name = None
118
+ self._prefix = None
119
+
120
+ # ToDo: add getter and setter for metadata
121
+ # self.metadata = kwargs.get("metadata") or Metadata()
122
+
123
+ self.metadata = Metadata()
124
+
125
+ # set default sdata attributes
126
+ self.metadata.add(self.SDATA_VERSION, __version__, dtype="str", description="sdata package version")
127
+ self.metadata.add(self.SDATA_NAME, "N.N.", dtype="str", description="name of the data object")
128
+ self.metadata.add(self.SDATA_SNAME, "", dtype="str", description="sname of the data object")
129
+ self.metadata.add(self.SDATA_UUID, "", dtype="str", description="Universally Unique Identifier")
130
+ self.metadata.add(self.SDATA_SUUID, "", dtype="str", description="Super Universally Unique Identifier")
131
+ self.metadata.add(self.SDATA_PARENT, "", dtype="str", description="uuid of the parent sdata object")
132
+ self.metadata.add(self.SDATA_CLASS, self.__class__.__name__, dtype="str", description="sdata class")
133
+ self.metadata.add(self.SDATA_CTIME, now_utc_str(), dtype="str", description="creation date")
134
+ self.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str", description="modification date")
135
+ self.metadata.add(self.SDATA_URL, "", dtype="str", description="url of the data object")
136
+ self.metadata.add(self.SDATA_PROJECT, "", dtype="str", description="project name")
137
+
138
+ metadata = kwargs.get("metadata")
139
+ if metadata is not None:
140
+ # logger.debug("Data got Metadata {}".format(metadata))
141
+ if metadata and isinstance(metadata, Metadata):
142
+ for attribute in metadata.attributes.values():
143
+ # logger.debug("Data.Metadata.add {0.name}:{0.value}".format(attribute))
144
+ self.metadata.add(attribute)
145
+
146
+ # auto correct
147
+ if kwargs.get("auto_correct") is None or kwargs.get("auto_correct") is True:
148
+ self.auto_correct = True
149
+ else:
150
+ self.auto_correct = False
151
+ # logger.debug("sdata: set auto_correct={}".format(self.auto_correct))
152
+
153
+ if kwargs.get("name") is not None:
154
+ self.name = kwargs.get("name")
155
+
156
+ self.prefix = kwargs.get("prefix") or ""
157
+ self._gen_default_attributes(kwargs.get("default_attributes") or self.ATTR_NAMES)
158
+ self._group = OrderedDict()
159
+ self._table = None # pd.DataFrame()
160
+ self.table = kwargs.get("table", None)
161
+ self._description = ""
162
+ self.description = kwargs.get("description", "")
163
+ self.project = kwargs.get("project", "N.N.")
164
+ self.metadata.add(self.SDATA_PROJECT, self.project, dtype="str", description="project name")
165
+
166
+ if (kwargs.get("uuid")=="" or kwargs.get("uuid") is not None) and not self.metadata.get(self.SDATA_UUID).value and kwargs.get("uuid")!="hash":
167
+ # logger.info("uuid in kwargs")
168
+ try:
169
+ self._set_uuid(kwargs.get("uuid")) # store given uuid str or generate a new uuid
170
+ except Sdata_Uuid_Exeption as exp:
171
+ if self.auto_correct is True:
172
+ logger.warning("got invalid uuid -> generate a new uuid")
173
+ self._set_uuid(uuid.uuid4())
174
+ else:
175
+ raise
176
+ elif (kwargs.get("uuid")=="" or kwargs.get("uuid") is None) and self.metadata.get(self.SDATA_UUID).value != "":
177
+ # logger.info("uuid in metadata")
178
+ pass
179
+ elif kwargs.get("uuid")=="hash":
180
+ sha3_256 = self.gen_uuid_from_state()
181
+ # logger.info("gen uuid from sha3_256 {}".format(sha3_256))
182
+ new_uuid = uuid_from_str(sha3_256)
183
+ self._set_uuid(new_uuid.hex)
184
+ else:
185
+ # logger.info("uuid new")
186
+ self._set_uuid(uuid.uuid4())
187
+
188
+ if "suuid" not in kwargs:
189
+ self.gen_suuid()
190
+ else:
191
+ suuid = sdata.SUUID.from_suuid_str(kwargs.get("suuid"))
192
+ self.suuid = kwargs.get("suuid")
193
+ self.sname = suuid.sname
194
+
195
+ if "sname" in kwargs:
196
+ self.sname = kwargs.get("sname")
197
+
198
+
199
+ if "parent" in kwargs:
200
+ self.metadata.add(self.SDATA_PARENT, kwargs.get("parent"))
201
+
202
+ def gen_suuid(self):
203
+ self.suuid = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).suuid_str
204
+ self.sname = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).sname
205
+
206
+ def gen_uuid_from_state(self):
207
+ """generate the same uuid for the same data
208
+
209
+ :return: uuid
210
+ """
211
+ s = hashlib.sha3_256()
212
+ metadata = self.metadata.copy()
213
+ metadata.attributes.pop(self.SDATA_UUID)
214
+ metadata.attributes.pop(self.SDATA_MTIME)
215
+ metadata.attributes.pop(self.SDATA_CTIME)
216
+ metadatastr = metadata.to_json().encode(errors="replace")
217
+ s.update(metadatastr)
218
+ if self.table is not None:
219
+ tablestr = self.table.to_json().encode(errors="replace")
220
+ s.update(tablestr)
221
+ s.update(self.description.encode(errors="replace"))
222
+ return s.hexdigest()
223
+
224
+ def __eq__(self, other):
225
+ """compare Data checksum
226
+
227
+ :param other: sdata.Data object
228
+ :return: True or False
229
+ """
230
+ if not isinstance(other, self.__class__):
231
+ logger.debug("you should not compare {} with {}!".format(self.__class__.__name__, other.__class__.__name__))
232
+ return False
233
+ return self.sha3_256 == other.sha3_256
234
+
235
+ def rename(self, name, random=True):
236
+ """change name, uuid and suuid
237
+ """
238
+ self.name = name
239
+ project = self.metadata.get(self.SDATA_PROJECT).value
240
+ if random is False:
241
+ suuid = sdata.SUUID.from_name(self.__class__.__name__, name, ns_name=project)
242
+ else:
243
+ suuid = sdata.SUUID(self.__class__.__name__, name)
244
+ self._set_suuid(suuid.suuid_str)
245
+ self._set_uuid(suuid.huuid)
246
+ return self
247
+
248
+ @property
249
+ def m(self):
250
+ """get metadata"""
251
+ return self.metadata
252
+
253
+ @property
254
+ def attrs(self):
255
+ """return user metadata as dict"""
256
+ return self.metadata.get_udict()
257
+
258
+ def update_mtime(self):
259
+ """update modification time
260
+
261
+ :return:
262
+ """
263
+ self.metadata.add(self.SDATA_MTIME, now_utc_str())
264
+
265
+ @property
266
+ def sha3_256_table(self):
267
+ """Return a SHA3 hash of the sData.table object with a hashbit length of 32 bytes.
268
+
269
+ .. code-block:: python
270
+
271
+ sdata.Data(name="1", uuid=sdata.uuid_from_str("1")).sha3_256_table
272
+
273
+ 'c468e659891eb5dea6eb6baf73f51ca0688792bf9ad723209dc22730903f6efa'
274
+
275
+ :return: hashlib.sha3_256.hexdigest()
276
+ """
277
+ s = hashlib.sha3_256()
278
+ if self.table is not None:
279
+ tablestr = self.table.to_json().encode(errors="replace")
280
+ s.update(tablestr)
281
+ return s.hexdigest()
282
+
283
+ @property
284
+ def sha3_256_metadata(self):
285
+ s = hashlib.sha3_256()
286
+ metadatastr = self.metadata.to_json().encode(errors="replace")
287
+ s.update(metadatastr)
288
+ return s.hexdigest()
289
+
290
+ @property
291
+ def sha3_256(self):
292
+ """Return a SHA3 hash of the sData object with a hashbit length of 32 bytes.
293
+
294
+ .. code-block:: python
295
+
296
+ sdata.Data(name="1", uuid=sdata.uuid_from_str("1")).sha3_256
297
+
298
+ 'c468e659891eb5dea6eb6baf73f51ca0688792bf9ad723209dc22730903f6efa'
299
+
300
+ :return: hashlib.sha3_256.hexdigest()
301
+ """
302
+ s = hashlib.sha3_256()
303
+ metadatastr = self.metadata.to_json().encode(errors="replace")
304
+ s.update(metadatastr)
305
+ if self.table is not None:
306
+ # tablestr = self.table.to_json().encode(errors="replace")
307
+ s.update(self.sha3_256_table.encode(errors="replace"))
308
+ s.update(self.description.encode(errors="replace"))
309
+ return s.hexdigest()
310
+
311
+ def update_hash(self, hashobject):
312
+ """A hash represents the object used to calculate a checksum of a
313
+ string of information.
314
+
315
+ .. code-block:: python
316
+
317
+ data = sdata.Data()
318
+
319
+ md5 = hashlib.md5()
320
+ data.update_hash(md5)
321
+ md5.hexdigest()
322
+ 'bbf323bdcb0bf961803b5504a8a60d69'
323
+
324
+ sha1 = hashlib.sha1()
325
+ data.update_hash(sha1)
326
+ sha1.hexdigest()
327
+ '3c59368c7735c1ecaf03ebd4c595bb6e73e90f0c'
328
+
329
+ hashobject = hashlib.sha3_256()
330
+ data.update_hash(hashobject).hexdigest()
331
+ 'c468e659891eb5dea6eb6baf73f51ca0688792bf9ad723209dc22730903f6efa'
332
+
333
+ data.update_hash(hashobject).digest()
334
+ b'M8...'
335
+
336
+ :param hash: hash object, e.g. hashlib.sha1()
337
+ :return: hash
338
+ """
339
+ if not (hasattr(hashobject, "update") and hasattr(hashobject, "hexdigest")):
340
+ logger.error("Data.update_hash: given hashfunction is invalid")
341
+ raise Exception("Data.update_hash: given hashfunction is invalid")
342
+
343
+ metadatastr = self.metadata.to_json().encode(errors="replace")
344
+ hashobject.update(metadatastr)
345
+ if self.table is not None:
346
+ tablestr = self.table.to_json().encode(errors="replace")
347
+ hashobject.update(tablestr)
348
+ hashobject.update(self.description.encode(errors="replace"))
349
+ return hashobject
350
+
351
+ def describe(self):
352
+ """Generate descriptive info of the data
353
+
354
+ .. code-block:: python
355
+
356
+ df = pd.DataFrame([1,2,3])
357
+ data = sdata.Data(name='my name',
358
+ uuid='38b26864e7794f5182d38459bab85842',
359
+ table=df,
360
+ description="A remarkable description")
361
+ data.describe()
362
+
363
+ .. code-block:: none
364
+
365
+ 0
366
+ metadata 3
367
+ table_rows 3
368
+ table_columns 1
369
+ description 24
370
+
371
+
372
+ :return: pd.DataFrame
373
+ """
374
+ df = pd.DataFrame({0: []}, dtype=object)
375
+ df.loc["metadata", 0] = self.metadata.size
376
+ if self.table is None:
377
+ df.loc["table_rows"] = 0
378
+ df.loc["table_columns"] = 0
379
+ else:
380
+ df.loc["table_rows"] = len(self.table)
381
+ df.loc["table_columns"] = len(self.table.columns)
382
+ df.loc["description", 0] = len(self.description)
383
+ return df
384
+
385
+ def _gen_default_attributes(self, default_attributes):
386
+ """create default Attributes in data.metadata"""
387
+ for attr_name, value, dtype, unit, description, required in default_attributes:
388
+ self.metadata.set_attr(name=attr_name, value=value, dtype=dtype, description=description)
389
+
390
+ def _get_suuid(self):
391
+ return self.metadata.get(self.SDATA_SUUID).value
392
+
393
+ def _set_suuid(self, value):
394
+ self.metadata.set_attr(self.SDATA_SUUID, value)
395
+
396
+ suuid = property(fget=_get_suuid, fset=_set_suuid, doc="suuid of the object")
397
+
398
+
399
+ def _get_sname(self):
400
+ return self.metadata.get(self.SDATA_SNAME).value
401
+
402
+ def _set_sname(self, value):
403
+ self.metadata.set_attr(self.SDATA_SNAME, value)
404
+
405
+ sname = property(fget=_get_sname, fset=_set_sname, doc="sname of the object")
406
+
407
+ def _get_uuid(self):
408
+ return self.metadata.get(self.SDATA_UUID).value
409
+ # return self._uuid
410
+
411
+ def _set_uuid(self, value):
412
+ if isinstance(value, str):
413
+ try:
414
+ uuid.UUID(value)
415
+ self.metadata.set_attr(self.SDATA_UUID, uuid.UUID(value).hex)
416
+ except ValueError as exp:
417
+ logger.warning("data.uuid: %s" % exp)
418
+ raise Sdata_Uuid_Exeption("got invalid uuid str '{}'".format(str(value)))
419
+ elif isinstance(value, uuid.UUID):
420
+ self.metadata.set_attr(self.SDATA_UUID, value.hex)
421
+ else:
422
+ logger.error("Data.uuid: invalid uuid '{}'".format(value))
423
+ raise Exception("Data.uuid: invalid uuid '{}'".format(value))
424
+
425
+ uuid = property(fget=_get_uuid, fset=_set_uuid, doc="uuid of the object")
426
+
427
+ def _get_name(self):
428
+ # return self._name
429
+ return self.metadata.get(self.SDATA_NAME).value
430
+
431
+ def _set_name(self, value):
432
+ if isinstance(value, str):
433
+ try:
434
+ self.metadata.set_attr(self.SDATA_NAME, str(value)[:256])
435
+ except ValueError as exp:
436
+ logger.warning("data.name: %s" % exp)
437
+ else:
438
+ # self._name = str(value)[:256]
439
+ self.metadata.set_attr(self.SDATA_NAME, str(value)[:256])
440
+
441
+ name = property(fget=_get_name, fset=_set_name, doc="name of the object")
442
+
443
+ def _get_project(self):
444
+ return self.metadata.get(self.SDATA_PROJECT).value
445
+
446
+ def _set_project(self, value):
447
+ if isinstance(value, str):
448
+ try:
449
+ self.metadata.set_attr(self.SDATA_PROJECT, str(value)[:256])
450
+ except ValueError as exp:
451
+ logger.warning("data.project: %s" % exp)
452
+ else:
453
+ # self._name = str(value)[:256]
454
+ self.metadata.set_attr(self.SDATA_PROJECT, str(value)[:256])
455
+
456
+ project = property(fget=_get_project, fset=_set_project, doc="name of the project")
457
+
458
+ def _get_description(self):
459
+ return self._description
460
+
461
+ def _set_description(self, value):
462
+ if isinstance(value, str):
463
+ try:
464
+ self._description = str(value)
465
+ except ValueError as exp:
466
+ logger.warning("data.name: %s" % exp)
467
+ else:
468
+ self._description = str(value)
469
+
470
+ description = property(fget=_get_description, fset=_set_description, doc="description of the object")
471
+
472
+ @property
473
+ def filename(self):
474
+
475
+ validchars = "-_.() "
476
+ out = ""
477
+
478
+ name = "{}".format(self.name)
479
+
480
+ for c in name:
481
+ if str.isalpha(c) or str.isdigit(c) or (c in validchars):
482
+ out += c
483
+ else:
484
+ out += "_"
485
+ return out
486
+
487
+ def _get_prefix(self):
488
+ return self._prefix
489
+
490
+ def _set_prefix(self, value):
491
+ if isinstance(value, str):
492
+ try:
493
+ self._prefix = value[:256]
494
+ except ValueError as exp:
495
+ logger.warning("data.prefix: %s" % exp)
496
+ else:
497
+ self._prefix = str(value)[:256]
498
+
499
+ prefix = property(fget=_get_prefix, fset=_set_prefix, doc="prefix of the object name")
500
+
501
+ def _get_table(self):
502
+ return self._table
503
+
504
+ def _set_table(self, df):
505
+ if isinstance(df, pd.DataFrame):
506
+ self._table = df
507
+ if self._table.index.name is None:
508
+ self._table.index.name = "index"
509
+
510
+ table = property(fget=_get_table, fset=_set_table, doc="table object(pandas.DataFrame)")
511
+ df = table
512
+
513
+ def description_to_df(self):
514
+ """get description as DataFrame
515
+
516
+ :return: DataFrame of description lines
517
+ """
518
+ return pd.DataFrame(self.description.splitlines())
519
+
520
+ def description_from_df(self, df):
521
+ """set description from DataFrame of lines
522
+
523
+ :return:
524
+ """
525
+ if df is not None and isinstance(df, pd.DataFrame) and len(df)>0:
526
+ lines = df.iloc[:, 0]
527
+ lines = lines.astype(str)
528
+ self.description = "\n".join(lines.values)
529
+
530
+ def to_folder(self, path, dtype="csv"):
531
+ """export data to folder
532
+
533
+ :param path:
534
+ :param dtype:
535
+ :return:
536
+ """
537
+
538
+ if dtype not in ["csv", "xlsx"]:
539
+ dtype = "xlsx"
540
+ if not os.path.exists(path):
541
+ try:
542
+ os.makedirs(path)
543
+ except OSError as exp:
544
+ logger.error(exp)
545
+ else:
546
+ self.clear_folder(path)
547
+
548
+ self.metadata.set_attr(name="class", value=self.__class__.__name__, description="object class", unit="-",
549
+ dtype="str")
550
+ self.metadata.set_attr(name="uuid", value=self.uuid, description="object uuid", unit="-", dtype="str")
551
+ self.metadata.set_attr(name="name", value=self.name, description="object name", unit="-", dtype="str")
552
+
553
+ if dtype == "csv":
554
+ metadata_filepath = os.path.join(path, "metadata.csv")
555
+ logger.debug("export meta csv '{}'".format(metadata_filepath))
556
+ self.metadata.to_csv(metadata_filepath)
557
+
558
+ # table export
559
+ if isinstance(self._table, pd.DataFrame) and len(self._table) > 0:
560
+ exportpath = os.path.join(path, "{}.csv".format(self.osname))
561
+ self._table.to_csv(exportpath, index=False)
562
+ if dtype == "xlsx":
563
+ if not isinstance(self._table, pd.DataFrame):
564
+ self.table = pd.DataFrame()
565
+ exportpath = os.path.join(path, "{}.xlsx".format(self.osname))
566
+ self.to_xlsx(exportpath)
567
+ # group export
568
+ for data in self.group.values():
569
+ exportpath = os.path.join(path, "{}-{}".format(data.__class__.__name__.lower(), data.osname))
570
+ data.to_folder(exportpath, dtype=dtype)
571
+ return path
572
+
573
+ def set_column_metadata(self):
574
+ """convert 'Force [N]' -> 'Force', 'N'
575
+
576
+ add Attribute for each column
577
+
578
+ sdata_column_0: value="Force", unit="N"
579
+ sdata_column_1
580
+ ...
581
+ sdata_column_n
582
+
583
+
584
+ """
585
+ if self.df is None:
586
+ return
587
+ for i, col in enumerate(self.df.columns):
588
+ name, unit = nameunit_from_colname(col)
589
+ scolname = f"{self.SDATA_COLUMN}_{i}"
590
+ self.metadata.add(scolname, value=name, unit=unit, label=col)
591
+
592
+ def set_columnnames_from_metadata(self):
593
+ """Use '!sdata_column_*' to reformat columns names
594
+
595
+ '!sdata_column_0', value='Force', unit='N' -> 'Force [N]'
596
+
597
+ > f'{name} [{unit}]'
598
+
599
+ """
600
+ if self.df is None:
601
+ return
602
+
603
+ cols = []
604
+ for i, col in enumerate(self.df.columns):
605
+ scolname = f"{self.SDATA_COLUMN}_{i}"
606
+ attr = self.metadata.get(scolname)
607
+ if attr is None:
608
+ name, unit = nameunit_from_colname(col)
609
+ else:
610
+ name = attr.value
611
+ unit = attr.unit
612
+ new_colname = f'{name} [{unit}]'
613
+ cols.append(new_colname)
614
+ self.df.columns = cols
615
+
616
+ @classmethod
617
+ def from_folder(cls, path):
618
+ """sdata object instance
619
+
620
+ :param path:
621
+ :return:
622
+ """
623
+ # data = Data.from_folder(path)
624
+
625
+ data = cls()
626
+ if not os.path.exists(path):
627
+ logger.error("from_folder error: path '{}' not exists.".format(path))
628
+ return data
629
+
630
+ data.metadata = data._load_metadata(path)
631
+ try:
632
+ data.uuid = data.metadata.get_attr("uuid").value
633
+ data.name = data.metadata.get_attr("name").value
634
+ except Exception as exp:
635
+ logger.error("Data.from_folder: {}".format(data.metadata.to_dict()))
636
+ raise
637
+
638
+ # table import
639
+ files = [x for x in os.listdir(path) if
640
+ not os.path.isdir(os.path.join(path, x)) and not x.startswith("metadata")]
641
+ if len(files) == 1:
642
+ assert len(files) == 1, "invalid number of files for Table '{}'".format(files)
643
+ importpath = os.path.join(path, files[0])
644
+ print("read table {}".format(importpath))
645
+ # data._table = pd.read_csv(importpath)
646
+
647
+ if not os.path.exists(path):
648
+ return cls()
649
+ metadata = cls._load_metadata(path)
650
+ data = cls()
651
+ data.metadata = metadata
652
+ data.uuid = data.metadata.get_attr("uuid").value
653
+ data.name = data.metadata.get_attr("name").value
654
+
655
+ folders = [x for x in os.listdir(path) if os.path.isdir(os.path.join(path, x))]
656
+ for folder in folders:
657
+ subfolder = os.path.join(path, folder)
658
+ data_ = data.from_folder(subfolder)
659
+ subdata = data_.from_folder(subfolder)
660
+ data.add_data(subdata)
661
+ return data
662
+
663
+ @staticmethod
664
+ def clear_folder(path):
665
+ """delete subfolder in export folder
666
+
667
+ :param path: path
668
+ :return: None
669
+ """
670
+
671
+ def is_valid(path):
672
+ prefix = path.split("-")[0]
673
+ if prefix in [x.lower() for x in SDATACLS.keys()]:
674
+ return True
675
+ else:
676
+ return False
677
+
678
+ subfolders = [x for x in os.listdir(path) if os.path.isdir(os.path.join(path, x))]
679
+ valid_subfolders = [x for x in subfolders if is_valid(x)]
680
+ for subfolder in valid_subfolders:
681
+ try:
682
+ subfolder = os.path.join(path, subfolder)
683
+ logger.debug("clear_folder: rm {}".format(subfolder))
684
+ shutil.rmtree(subfolder)
685
+ except OSError as exp:
686
+ raise
687
+
688
+ @staticmethod
689
+ def _load_metadata(path):
690
+ """load metadata from csv
691
+
692
+ :returns: Metadata instance"""
693
+ metadata_filepath = os.path.join(path, "metadata.csv")
694
+ if os.path.exists(metadata_filepath):
695
+ metadata = Metadata().from_csv(metadata_filepath)
696
+ else:
697
+ metadata = Metadata()
698
+ return metadata
699
+
700
+ @staticmethod
701
+ def _get_class_from_metadata(metadata):
702
+ """get class object from metadata
703
+
704
+ :returns: relevant sdata class object"""
705
+ classattr = metadata.get_attr("class")
706
+ if classattr is not None:
707
+ sdataclassname = classattr.value
708
+ sdatacls = SDATACLS.get(sdataclassname)
709
+ if sdataclassname not in SDATACLS:
710
+ logger.warning("unsupported cls '{}'".format(sdataclassname))
711
+ sdatacls = Data
712
+ else:
713
+ logger.warning("cls not defined '{}'".format(metadata))
714
+ sdatacls = None
715
+ return sdatacls
716
+
717
+ @property
718
+ def osname(self):
719
+ """:returns: os compatible name (ascii?)"""
720
+ return self.asciiname.lower()
721
+
722
+ @property
723
+ def asciiname(self):
724
+ name = copy.copy(self.name)
725
+ mapper = [("ä", "ae"), ("ö", "oe"), ("ü", "ue"), ("Ä", "Ae"), ("Ö", "Oe"), ("Ü", "Ue"),
726
+ ("ß", "sz"), (" ", "_"), ("/", "_"), ("\\", "_")]
727
+ for k, v in mapper:
728
+ name = name.replace(k, v)
729
+ return name.encode('ascii', 'replace').decode("ascii")
730
+
731
+ def verify_attributes(self):
732
+ """check mandatory attributes"""
733
+ invalid_attrs = []
734
+ # attr_defs = ["name", "value", "dtype", "unit", "description", "required"]
735
+ for attr_defs in self.ATTR_NAMES:
736
+ required = attr_defs[5]
737
+ if required is False:
738
+ continue
739
+ attr = self.metadata.get_attr(attr_defs[0])
740
+ if attr is None:
741
+ invalid_attrs.append(attr_defs[0])
742
+ elif attr.value is None:
743
+ invalid_attrs.append(attr.name)
744
+ return invalid_attrs
745
+
746
+ def __str__(self):
747
+ return f"({self.__class__.__name__} '{self.name}':{self.uuid})"
748
+
749
+ __repr__ = __str__
750
+ @property
751
+ def mdf(self):
752
+ return self.metadata.df
753
+
754
+ @property
755
+ def udf(self):
756
+ return self.metadata.udf
757
+
758
+ @property
759
+ def sdf(self):
760
+ return self.metadata.sdf
761
+
762
+ @property
763
+ def md(self):
764
+ return self.metadata
765
+
766
+ def get_group(self):
767
+ return self._group
768
+
769
+ group = property(get_group, doc="get group")
770
+
771
+ def keys(self):
772
+ """get all child objects uuids
773
+
774
+ :return: list of uuid's
775
+ """
776
+ return list(self.group.keys())
777
+
778
+ def values(self):
779
+ """get all child objects
780
+
781
+ :return: list of child objects
782
+ """
783
+ return list(self.group.values())
784
+
785
+ def items(self):
786
+ """get all child objects
787
+
788
+ :return: [(child uuid, child objects), ]
789
+ """
790
+ return list(self.group.items())
791
+
792
+ def clear_group(self):
793
+ """clear group dict"""
794
+ self._group = OrderedDict()
795
+
796
+ def add_data(self, data):
797
+ """add data, if data.name is unique"""
798
+ if hasattr(data, "metadata"):
799
+ names = [dat.name.lower() for uid, dat in self.group.items()]
800
+ if data.name.lower() in names:
801
+ logger.error("{}: name '{}' aready exists".format(data.__class__.__name__, data.name))
802
+ return
803
+ self.group[data.uuid] = data
804
+ else:
805
+ logger.warning(f"ignore data {data}, {data.__class__.__name__} (wrong type!)")
806
+
807
+ def get_data_by_uuid(self, uid):
808
+ """get data by uuid"""
809
+ return self.group.get(uid)
810
+
811
+ def get_data_by_name(self, name):
812
+ """:return obj by name"""
813
+ d = dict([(obj.name, uid) for uid, obj in self.group.items()])
814
+ uid = d.get(name)
815
+ return self.get_data_by_uuid(uid)
816
+
817
+ def tree_folder(self, dir, padding=" ", print_files=True, hidden_files=False, last=True):
818
+ """print tree folder structure"""
819
+ if last is False:
820
+ print(padding[:-1] + '├─' + os.path.basename(os.path.abspath(dir)))
821
+ else:
822
+ print(padding[:-1] + '└─' + os.path.basename(os.path.abspath(dir)))
823
+ padding = padding + ' '
824
+ files = []
825
+ if print_files:
826
+ files = [x for x in sorted(os.listdir(dir)) if not x.startswith(".")]
827
+ else:
828
+ files = [x for x in sorted(os.listdir(dir)) if os.path.isdir(dir + os.sep + x)]
829
+
830
+ # metadata first
831
+ metafiles = [f for f in files if f.startswith("metadata")]
832
+ files = [x for x in files if x not in metafiles]
833
+ files = metafiles + sorted(files)
834
+
835
+ for count, file in enumerate(sorted(files)):
836
+ # print(padding + '|')
837
+ path = dir + os.sep + file
838
+ if os.path.isdir(path):
839
+ if count == (len(files) - 1):
840
+ self.tree_folder(path, padding + ' ', print_files, last=True)
841
+ else:
842
+ self.tree_folder(path, padding + '|', print_files, last=False)
843
+ else:
844
+ if count == (len(files) - 1):
845
+ print(padding + '└─' + file)
846
+ else:
847
+ print(padding + '├─' + file)
848
+
849
+ def dir(self):
850
+ """returns a nested list of all child objects
851
+
852
+ :return: list of sdata.Data objects
853
+ """
854
+ return [(x.name, x.dir()) for x in self.group.values()]
855
+
856
+ def to_dataframe(self):
857
+ """Export the DataFrame with additional metadata and description.
858
+
859
+ Returns:
860
+ pandas.DataFrame: A copy of the DataFrame with added attributes.
861
+ """
862
+ df = self.df.copy()
863
+ df.attrs["!sdata"] = {
864
+ "metadata": self.metadata.to_dict(),
865
+ "description": self.description
866
+ }
867
+ return df
868
+
869
+
870
+ def to_parquet(self, filepath=None, **kwargs):
871
+ """
872
+
873
+ """
874
+ engine = kwargs.get("engine", "pyarrow")
875
+ df = self.df.copy()
876
+ #df["!sdata_index"] = df.index
877
+ #df.reset_index(inplace=True)
878
+ df.attrs["!sdata"] = {"metadata": self.metadata.to_dict(),
879
+ "description": self.description}
880
+ if filepath is not None:
881
+ df.to_parquet(filepath, engine=engine)
882
+ else:
883
+ return df.to_parquet(engine=engine)
884
+
885
+ @classmethod
886
+ def from_parquet(cls, filepath):
887
+ """load data from parquet file
888
+
889
+ :param filepath:
890
+ :return: Data object
891
+ """
892
+ try:
893
+ if os.path.exists(filepath):
894
+ df = pd.read_parquet(filepath)
895
+ tt = cls(name=filepath)
896
+ tt.table = df
897
+
898
+ attrs = df.attrs.get("!sdata")
899
+ try:
900
+ tt.metadata = tt.metadata.from_dict(attrs["metadata"])
901
+ except:
902
+ logger.warning(f"ignore metadata {filepath}")
903
+ try:
904
+ tt.description = attrs.get("description")
905
+ except:
906
+ logger.warning(f"ignore description {filepath}")
907
+ if attrs is not None:
908
+ tt.df.attrs.pop("!sdata")
909
+ else:
910
+ raise Exception("parquet file '{}' not available".format(filepath))
911
+ return tt
912
+ except Exception as exp:
913
+ raise
914
+
915
+ def to_xlsx_byteio(self):
916
+ """get xlsx as byteio
917
+
918
+ :return: BytesIO
919
+ """
920
+
921
+ def adjust_col_width(sheetname, df, writer, width=40):
922
+ worksheet = writer.sheets[sheetname] # pull worksheet object
923
+ worksheet.set_column(0, 0, width)
924
+ for idx, col in enumerate(df): # loop through all columns
925
+ # series = df[col]
926
+ # max_len = max((
927
+ # series.astype(str, raise_on_error=False).map(len).max(), # len of largest item
928
+ # len(str(series.name)) # len of column name/header
929
+ # )) + 1 # adding a little extra space
930
+ worksheet.set_column(idx + 1, idx + 1, width)
931
+
932
+ output = BytesIO()
933
+ writer = pd.ExcelWriter(output, engine='xlsxwriter')
934
+ self.metadata.df.to_excel(writer, sheet_name='metadata')
935
+ adjust_col_width('metadata', self.metadata.df, writer)
936
+
937
+ self.df.to_excel(writer, sheet_name='table')
938
+ adjust_col_width('table', self.table, writer, width=15)
939
+
940
+ df_description = pd.DataFrame(self.description.splitlines())
941
+ df_description.to_excel(writer, sheet_name='description', index=False, header=None)
942
+ adjust_col_width('description', df_description, writer, width=200)
943
+
944
+ writer.close()
945
+ processed_data = output.getvalue()
946
+ return processed_data
947
+
948
+ def to_xlsx_base64(self):
949
+ """get xlsx as byteio base64 encoded
950
+
951
+ :return: base64
952
+ """
953
+ val = self.to_xlsx_byteio()
954
+ b64 = base64.b64encode(val)
955
+ return b64
956
+
957
+ def get_download_link(self):
958
+ """Generates a link allowing the data in a given panda dataframe to be downloaded
959
+ in: dataframe
960
+ out: href string
961
+ """
962
+ b64 = self.to_xlsx_base64()
963
+ return '<a href="data:application/octet-stream;base64,{1}" download="{0}.xlsx">Download {0}.xlsx file</a>'.format(
964
+ self.osname, b64.decode())
965
+
966
+ def to_xlsx(self, filepath=None):
967
+ """export atrributes and data to excel
968
+
969
+ :param filepath:
970
+ :return:
971
+ """
972
+
973
+ def adjust_col_width(sheetname, df, writer, width=40):
974
+ worksheet = writer.sheets[sheetname] # pull worksheet object
975
+ worksheet.set_column(0, 0, width)
976
+ for idx, col in enumerate(df): # loop through all columns
977
+ # series = df[col]
978
+ # max_len = max((
979
+ # series.astype(str, raise_on_error=False).map(len).max(), # len of largest item
980
+ # len(str(series.name)) # len of column name/header
981
+ # )) + 1 # adding a little extra space
982
+ worksheet.set_column(idx + 1, idx + 1, width)
983
+
984
+ with pd.ExcelWriter(filepath) as writer:
985
+
986
+ # metadata
987
+ # dfm = pd.DataFrame.from_dict(self.metadata, orient="index", columns=["value"])
988
+ dfm = self.metadata.to_dataframe()
989
+
990
+ # dfm = dfm.sort_index()
991
+ dfm.index.name = "key"
992
+ dfm.to_excel(writer, sheet_name='metadata', index=False)
993
+ adjust_col_width('metadata', dfm, writer)
994
+
995
+ # data
996
+ if self.table is not None:
997
+ if self._table.index.name is None:
998
+ self._table.index.name = "index"
999
+ self.table.to_excel(writer, sheet_name='table')
1000
+ adjust_col_width('table', self.table, writer, width=15)
1001
+ else:
1002
+ df = pd.DataFrame()
1003
+ df.index.name = "index"
1004
+ df.to_excel(writer, sheet_name='table')
1005
+ adjust_col_width('table', df, writer, width=15)
1006
+
1007
+ df_description = pd.DataFrame(self.description.splitlines())
1008
+ df_description.to_excel(writer, sheet_name='description', index=False, header=None)
1009
+ adjust_col_width('description', df_description, writer, width=200)
1010
+
1011
+ # # raw data
1012
+ # self.df_raw.index.name = "index"
1013
+ # self.df_raw.to_excel(writer, sheet_name='df_raw')
1014
+ # adjust_col_width('df_raw', self.df_raw, writer, width=15)
1015
+
1016
+ @classmethod
1017
+ def from_xlsx(cls, filepath):
1018
+ """save table as xlsx
1019
+
1020
+ :param filepath:
1021
+ :return:
1022
+ """
1023
+ try:
1024
+ if os.path.exists(filepath):
1025
+ tt = cls(name=filepath)
1026
+
1027
+ # read df
1028
+ try:
1029
+ tt.table = pd.read_excel(filepath, sheet_name="table", index_col=0)
1030
+ except:
1031
+ logger.info("no table data in '{}'".format(filepath))
1032
+ dfm = pd.read_excel(filepath, sheet_name="metadata")
1033
+ dfm.set_index(dfm.name.values, inplace=True)
1034
+ # dfm["value"] = dfm["value"].replace(np.nan, None)
1035
+ dfm["description"] = dfm["description"].replace(np.nan, '')
1036
+ dfm["label"] = dfm["label"].replace(np.nan, '')
1037
+
1038
+ # read metadata
1039
+ tt.metadata = tt.metadata.from_dataframe(dfm)
1040
+ for attr in tt.metadata.sdata_attributes.values():
1041
+ if attr.value == "nan":
1042
+ attr.value = ""
1043
+
1044
+ # read description
1045
+ try:
1046
+ dfd = pd.read_excel(filepath, sheet_name="description", header=None)
1047
+ tt.description = "\n".join(dfd[0])
1048
+ except:
1049
+ logger.info("no description in '{}'".format(filepath))
1050
+ return tt
1051
+ else:
1052
+ raise Exception("excel file '{}' not available".format(filepath))
1053
+ except Exception as exp:
1054
+ raise
1055
+
1056
+ def to_json(self, filepath=None):
1057
+ """export Data in json format
1058
+
1059
+ :param filepath: export file path (default:None)
1060
+ :return: json str
1061
+ """
1062
+
1063
+ if self.table is not None:
1064
+ json_table = self.table.to_dict()
1065
+ else:
1066
+ json_table = {}
1067
+
1068
+ j = {"metadata": self.metadata.to_dict(),
1069
+ "table": json_table,
1070
+ "description": self.description
1071
+ }
1072
+ if filepath:
1073
+ with open(filepath, "w") as fh:
1074
+ json.dump(j, fh)
1075
+ else:
1076
+ return json.dumps(j)
1077
+
1078
+ @classmethod
1079
+ def from_json(cls, s=None, filepath=None):
1080
+ """create Data from json str or file
1081
+
1082
+ :param s: json str
1083
+ :param filepath:
1084
+ :return: sdata.Data
1085
+ """
1086
+ data = cls(name="N.N.")
1087
+ if s is None and filepath is not None:
1088
+ with open(filepath, "r") as fh:
1089
+ d = json.load(fh)
1090
+ elif s is None and filepath is None:
1091
+ logger.error("data.from_json: no json data available")
1092
+ return
1093
+ elif s is not None and filepath is None:
1094
+ d = json.loads(s)
1095
+ else:
1096
+ logger.error("data.from_json: unexpected error")
1097
+ d = None
1098
+
1099
+ if d:
1100
+ if "metadata" in d.keys():
1101
+ data.metadata.update_from_dict(d["metadata"])
1102
+ else:
1103
+ logger.error("Data.from_json: table not available")
1104
+
1105
+ if "table" in d.keys():
1106
+ data.table = pd.DataFrame.from_dict(d["table"])
1107
+ # data.table = pd.read_json(json.dumps(d["table"]))
1108
+ # data.table = pd.read_json(d["table"])
1109
+ else:
1110
+ logger.error("Data.from_json: metadata not available")
1111
+
1112
+ if "description" in d.keys():
1113
+ data.description = d["description"]
1114
+ else:
1115
+ logger.error("Data.from_json: description not available")
1116
+
1117
+ return data
1118
+
1119
+ @classmethod
1120
+ def from_url(cls, url=None, stype=None):
1121
+ """create Data from json str or file
1122
+
1123
+ :param url: url
1124
+ :param stype: "json" ("xlsx", "csv")
1125
+ :return: sdata.Data
1126
+ """
1127
+
1128
+ supported_stypes = ["json"]
1129
+
1130
+ if stype not in supported_stypes:
1131
+ raise NotADirectoryError("stype '{}' is not supported".format(stype))
1132
+ return
1133
+
1134
+ # 'requests' ist optional; Fallback auf die Standardbibliothek (urllib).
1135
+ try:
1136
+ import requests
1137
+ raw = requests.get(url).text
1138
+ except ImportError:
1139
+ import urllib.request
1140
+ with urllib.request.urlopen(url) as _resp:
1141
+ raw = _resp.read().decode("utf-8", "replace")
1142
+ if stype=="json":
1143
+ data = cls.from_json(raw)
1144
+ return data
1145
+
1146
+
1147
+ def to_csv(self, filepath=None):
1148
+ """export sdata.Data to csv
1149
+
1150
+ :param filepath:
1151
+ :return:
1152
+ """
1153
+
1154
+ exportlines = []
1155
+ exportlines.append(self.metadata.to_csv_header(prefix="#;", sep=";", filepath=None))
1156
+ if self.df is not None:
1157
+ exportlines.append(self.df.to_csv(sep=";"))
1158
+ exportstr = "".join(exportlines)
1159
+
1160
+ if filepath is None:
1161
+ return exportstr
1162
+ else:
1163
+ with open(filepath, "w") as fh:
1164
+ fh.write(exportstr)
1165
+
1166
+ @classmethod
1167
+ def from_csv(cls, filepath, sep=";"):
1168
+ """import sdata.Data from csv
1169
+
1170
+ :param filepath:
1171
+ :param sep: separator (default=";")
1172
+ :return: sdata.Data
1173
+ """
1174
+ data = cls()
1175
+ df = pd.read_csv(filepath, sep=";", comment="#", index_col=0)
1176
+
1177
+
1178
+ #attr
1179
+ sio = open(filepath, "r")
1180
+ sio.seek(0)
1181
+ attritute_list = []
1182
+ for line in sio:
1183
+ if line.startswith("#;"):
1184
+ line = line.rstrip("\n")
1185
+ line = line.split(sep)
1186
+ attritute_list.append(line[1:8])
1187
+ data.metadata = Metadata.from_list(attritute_list)
1188
+ data.table = df
1189
+ return data
1190
+
1191
+
1192
+ @classmethod
1193
+ def from_csv_string(cls, s=None, sep=";"):
1194
+ """import sdata.Data from csv
1195
+
1196
+ :param s: csv str
1197
+ :param filepath:
1198
+ :param sep: separator (default=";")
1199
+ :return: sdata.Data
1200
+ """
1201
+ if s is None and filepath is None:
1202
+ logging.error("specify string if filepath")
1203
+ return
1204
+ data = cls()
1205
+ df = None
1206
+ try:
1207
+ sio = StringIO(s)
1208
+ df = pd.read_csv(sio, sep=";", comment="#")
1209
+ sio.seek(0)
1210
+ except Exception as exp:
1211
+ logging.error(f"{exp}")
1212
+ raise
1213
+
1214
+ attritute_list = []
1215
+ for line in sio:
1216
+ if line.startswith("#;"):
1217
+ line = line.rstrip("\n")
1218
+ line = line.split(sep)
1219
+ attritute_list.append(line[1:8])
1220
+ data.metadata = Metadata.from_list(attritute_list)
1221
+ data.table = df
1222
+ return data
1223
+
1224
+ def to_hdf5(self, filepath, **kwargs):
1225
+ """export sdata.Data to hdf5
1226
+
1227
+ :param filepath:
1228
+ :param complib: default='zlib' ['zlib', 'lzo', 'bzip2', 'blosc', 'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy', 'blosc:zlib', 'blosc:zstd']
1229
+ :param complevel: default=9 [0-9]
1230
+
1231
+ :return:
1232
+ """
1233
+ if not isinstance(self.df, pd.DataFrame):
1234
+ df = pd.DataFrame()
1235
+ else:
1236
+ df = self.df
1237
+ kwargs["mode"] = "w"
1238
+ if kwargs.get("complib") is None:
1239
+ kwargs["complib"] = "zlib"
1240
+
1241
+ if kwargs.get("complevel") is None:
1242
+ kwargs["complevel"] = 9
1243
+
1244
+ with pd.HDFStore(filepath, **kwargs) as hdf:
1245
+ hdf.put('metadata'.format(self.uuid), self.metadata.df, format='fixed', data_columns=True)
1246
+ hdf.put('table'.format(self.uuid), df, format='fixed', data_columns=True)
1247
+ hdf.put('description'.format(self.uuid), self.description_to_df(), format='fixed', data_columns=True)
1248
+
1249
+ def to_sqlite(self, filepath, **kwargs):
1250
+ """export sdata.Data to sqlite
1251
+
1252
+ :param filepath:
1253
+ :param kwargs:
1254
+ :return:
1255
+ """
1256
+
1257
+ if not isinstance(self.df, pd.DataFrame):
1258
+ df = pd.DataFrame()
1259
+ else:
1260
+ df = self.df
1261
+ df.columns = df.columns.astype(str)
1262
+
1263
+ with SqliteDict(filepath, autocommit=True) as sqdict:
1264
+ sqdict['metadata'] = self.metadata.df.to_parquet()
1265
+ sqdict['table'] = df.to_parquet()
1266
+
1267
+ description = self.description_to_df().copy()
1268
+ description.columns = description.columns.astype(str)
1269
+ sqdict['description'] = description.to_parquet()
1270
+
1271
+ @classmethod
1272
+ def from_sqlite(cls, filepath, **kwargs):
1273
+ """import sdata.Data from sqlite
1274
+
1275
+ :param filepath:
1276
+ :param kwargs:
1277
+ :return: sdata.Data
1278
+ """
1279
+
1280
+ with SqliteDict(filepath) as sqdict:
1281
+ metadata = sdata.Metadata.from_dataframe(pd.read_parquet(BytesIO(sqdict['metadata'])))
1282
+ df_table = pd.read_parquet(BytesIO(sqdict['table']))
1283
+ df_description = pd.read_parquet(BytesIO(sqdict['description']))
1284
+ data = Data(metadata=metadata, table=df_table)
1285
+ data.description_from_df(df_description)
1286
+ return data
1287
+
1288
+ @classmethod
1289
+ def metadata_from_hdf5(cls, filepath, **kwargs):
1290
+ """import sdata.Data.Metadata from hdf5
1291
+
1292
+ :param filepath:
1293
+ :return: sdata.Data
1294
+ """
1295
+ if not os.path.exists:
1296
+ logger.error("hdf5 file '{}' not available".format(filepath))
1297
+ return
1298
+
1299
+ with pd.HDFStore(filepath, mode="r+") as hdf:
1300
+ metadata_path = "/metadata".format(uuid)
1301
+ df_metadata = hdf.get(metadata_path)
1302
+ metadata = Metadata.from_dataframe(df_metadata)
1303
+ return metadata
1304
+
1305
+ @classmethod
1306
+ def from_hdf5(cls, filepath, **kwargs):
1307
+ """import sdata.Data from hdf5
1308
+
1309
+ :param filepath:
1310
+ :return: sdata.Data
1311
+ """
1312
+ if not os.path.exists:
1313
+ logger.error("hdf5 file '{}' not available".format(filepath))
1314
+ return
1315
+
1316
+ with pd.HDFStore(filepath, mode="r+") as hdf:
1317
+ metadata_path = "/metadata".format(uuid)
1318
+ table_path = "/table".format(uuid)
1319
+ description_path = "/description".format(uuid)
1320
+ df_metadata = hdf.get(metadata_path)
1321
+ df_table = hdf.get(table_path)
1322
+ df_description = hdf.get(description_path)
1323
+ metadata = Metadata.from_dataframe(df_metadata)
1324
+ # logger.debug("hdf {}".format(metadata.get("!sdata_uuid").value))
1325
+ data = Data(metadata=metadata, table=df_table)
1326
+ data.description_from_df(df_description)
1327
+
1328
+ return data
1329
+
1330
+ def to_html(self, filepath, xlsx=True, style=None):
1331
+ """export Data to html
1332
+
1333
+ :param filepath:
1334
+ :param xlsx:
1335
+ :param style:
1336
+ :return:
1337
+ """
1338
+
1339
+ table_values = self.df.head()
1340
+ table_headers = self.df.columns
1341
+
1342
+ table_description_values = self.df.describe()
1343
+ table_description_headers = self.df.describe().columns
1344
+
1345
+ metadata_values = self.metadata.df.head().values
1346
+ metadata_headers = self.metadata.df.columns
1347
+
1348
+ if xlsx is True:
1349
+ xlsx_tag = self.get_download_link()
1350
+ else:
1351
+ xlsx_tag = ""
1352
+
1353
+ from tabulate import tabulate # optional; nur für HTML-Repr benötigt
1354
+ param = {"title":"{0} [{1}]".format(self.osname,
1355
+ self.uuid),
1356
+ "description":self.description,
1357
+ "metadata": tabulate(metadata_values, metadata_headers, tablefmt="html"),
1358
+ "table": tabulate(table_values, table_headers, tablefmt="html"),
1359
+ "table_description": tabulate(table_description_values, table_description_headers, tablefmt="html"),
1360
+ "xlsx_tag":xlsx_tag,
1361
+ "sdata":"created with sdata v{}.".format(__version__),
1362
+ "now":"{}".format(now_utc_str()),
1363
+ }
1364
+
1365
+ tmpl = """<!DOCTYPE html>
1366
+ <html lang="de">
1367
+ <head>
1368
+ <meta charset="utf-8">
1369
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
1370
+ <title>{title}</title>
1371
+ <style>
1372
+ h1 {{
1373
+ background-color: #00FFFF77;
1374
+ color: black;
1375
+ }}
1376
+ h2 {{
1377
+ background-color: #00FFFF44;
1378
+ color: black;
1379
+ }}
1380
+ h3 {{
1381
+ background-color: #00FFFF11;
1382
+ color: black;
1383
+ }}
1384
+ p {{
1385
+ color: black;
1386
+ }}
1387
+ table, th, td, caption {{
1388
+ border: 1px solid #a0a0a0;
1389
+ }}
1390
+
1391
+ table {{
1392
+ border-collapse: collapse;
1393
+ border-spacing: 1em;
1394
+ border-width: thin 0 0 thin;
1395
+ margin: 0 0 1em;
1396
+ table-layout: auto;
1397
+ max-width: 100%;
1398
+ text-align: right;
1399
+ }}
1400
+ th, td {{
1401
+ font-weight: normal;
1402
+ text-align: left;
1403
+ border-spacing: 1em;
1404
+ padding: .1em .3em;
1405
+ }}
1406
+ th, caption {{
1407
+ background-color: #f1f3f4;
1408
+ font-weight: 700;
1409
+ }}
1410
+ </style>
1411
+ </head>
1412
+ <body>
1413
+ <h1>{title}</h1>
1414
+ <h2>Download</h2>
1415
+ <p">{xlsx_tag}</p>
1416
+ <h2>Description</h2>
1417
+ <p>{description}</p>
1418
+ <h2>Metadata</h2>
1419
+ {metadata}
1420
+ <h2>Table</h2>
1421
+ {table}
1422
+ <h3>Table Description</h3>
1423
+ {table_description}
1424
+ <p>{sdata}</p>
1425
+ <p>{now}</p>
1426
+ </body>
1427
+ </html>""".format(**param)
1428
+ try:
1429
+ with open(filepath, "w") as fh:
1430
+ fh.write(tmpl)
1431
+ except Exception as exp:
1432
+ raise
1433
+
1434
+ def copy(self, **kwargs):
1435
+ """create a copy of the Data object
1436
+
1437
+ .. code-block:: python
1438
+
1439
+ data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab85842", description="this is remarkable")
1440
+ datac = data.copy()
1441
+ print("data {0.uuid}".format(data))
1442
+ print("datac {0.uuid}".format(datac))
1443
+ print("datac.metadata['!sdata_parent'] {0.value}".format(datac.metadata["sdata_parent"]))
1444
+
1445
+ .. code-block::
1446
+
1447
+ data 38b26864e7794f5182d38459bab85842
1448
+ datac 2c4eb15900af435d8cd9c8573ca777e2
1449
+ datac.metadata['!sdata_parent'] 38b26864e7794f5182d38459bab85842
1450
+
1451
+ :return: Data
1452
+ """
1453
+ data = copy.deepcopy(self)
1454
+ data.metadata.add(self.SDATA_PARENT, self.suuid)
1455
+ data.metadata.add(self.SDATA_UUID, self.gen_uuid())
1456
+ data.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str")
1457
+ if "uuid" in kwargs:
1458
+ data.uuid = kwargs.get("uuid")
1459
+ if "name" in kwargs:
1460
+ data.name = kwargs.get("name")
1461
+ logger.debug(f"make copy of {self.uuid} -> {data.uuid}")
1462
+ data.gen_suuid()
1463
+
1464
+ return data
1465
+
1466
+ def gen_uuid(self):
1467
+ """generate new uuid string
1468
+
1469
+ :return: str, e.g. '5fa04a3738e4431dbc34eccea5e795c4'
1470
+ """
1471
+ return uuid.uuid4().hex
1472
+
1473
+ def refactor(self, fix_columns=True, add_table_metadata=True):
1474
+ """helper function
1475
+
1476
+ * to cleanup dataframe column name
1477
+ * to define Attributes for all dataframe columns
1478
+ """
1479
+ if isinstance(self.table, pd.DataFrame):
1480
+ mapper = {}
1481
+ for old_colname in self.table.columns:
1482
+ name, unit = extract_name_unit(old_colname)
1483
+ if fix_columns:
1484
+ mapper[old_colname] = name
1485
+ if add_table_metadata:
1486
+ old_attr = self.metadata.get(old_colname)
1487
+ if old_attr:
1488
+ logger.info("skip: {}".format(old_attr))
1489
+ self.metadata.relabel(old_colname, name)
1490
+ else:
1491
+ self.metadata.add(name=name, description=old_colname, unit=unit, dtype="float")
1492
+ self.table.rename(columns=mapper, inplace=True)
1493
+
1494
+ SDATACLS = {"Data": Data,
1495
+ }
1496
+ #
1497
+ # __all__ = ["Data", Data]
1498
+
1499
+
1500
+ def print_classes():
1501
+ for name, obj in inspect.getmembers(sys.modules[__name__]):
1502
+ if inspect.isclass(obj):
1503
+ print(obj)