sdata 1.5.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdata/__init__.py +140 -0
- sdata/base.py +684 -0
- sdata/contrib/__init__.py +0 -0
- sdata/contrib/piexif/__init__.py +12 -0
- sdata/contrib/piexif/_common.py +94 -0
- sdata/contrib/piexif/_dump.py +346 -0
- sdata/contrib/piexif/_exceptions.py +2 -0
- sdata/contrib/piexif/_exif.py +638 -0
- sdata/contrib/piexif/_insert.py +60 -0
- sdata/contrib/piexif/_load.py +270 -0
- sdata/contrib/piexif/_remove.py +56 -0
- sdata/contrib/piexif/_transplant.py +45 -0
- sdata/contrib/piexif/_webp.py +240 -0
- sdata/contrib/piexif/helper.py +66 -0
- sdata/contrib/ranger/__init__.py +8 -0
- sdata/contrib/ranger/collections/__init__.py +0 -0
- sdata/contrib/ranger/collections/rangebucketmap.py +346 -0
- sdata/contrib/ranger/collections/rangemap.py +396 -0
- sdata/contrib/ranger/collections/rangeset.py +483 -0
- sdata/contrib/ranger/range/__init__.py +0 -0
- sdata/contrib/ranger/range/cut.py +285 -0
- sdata/contrib/ranger/range/range.py +644 -0
- sdata/contrib/sobol_seq.py +394 -0
- sdata/contrib/sortedcontainers/LICENSE +13 -0
- sdata/contrib/sortedcontainers/__init__.py +74 -0
- sdata/contrib/sortedcontainers/sorteddict.py +789 -0
- sdata/contrib/sortedcontainers/sortedlist.py +2646 -0
- sdata/contrib/sortedcontainers/sortedset.py +733 -0
- sdata/contrib/sqlitedict.py +598 -0
- sdata/contrib/timeflake/__init__.py +40 -0
- sdata/contrib/timeflake/flake.py +69 -0
- sdata/contrib/timeflake/utils.py +47 -0
- sdata/deprecated/__init__.py +10 -0
- sdata/deprecated/data.py +1503 -0
- sdata/did/__init__.py +129 -0
- sdata/did/_http.py +67 -0
- sdata/did/base58btc.py +49 -0
- sdata/did/did_github.py +152 -0
- sdata/did/did_peer4.py +155 -0
- sdata/did/did_web.py +222 -0
- sdata/did/diddoc.py +75 -0
- sdata/did/ed25519.py +225 -0
- sdata/did/eddsa.py +107 -0
- sdata/did/errors.py +40 -0
- sdata/did/issue_vc.py +193 -0
- sdata/did/jose.py +126 -0
- sdata/did/keys.py +104 -0
- sdata/did/utils_didkey.py +161 -0
- sdata/did/verify_vp.py +158 -0
- sdata/doe.py +101 -0
- sdata/dtypes.py +510 -0
- sdata/experiments/__init__.py +331 -0
- sdata/experiments/blob0.py +131 -0
- sdata/experiments/file.py +42 -0
- sdata/experiments/filegroup.py +144 -0
- sdata/experiments/image_old.py +313 -0
- sdata/experiments/ks2.py +15 -0
- sdata/experiments/lapshear.py +8 -0
- sdata/experiments/material.py +7 -0
- sdata/experiments/parameters_deprecated.py +394 -0
- sdata/experiments/workbook.py +226 -0
- sdata/format.py +125 -0
- sdata/imagemeta.py +533 -0
- sdata/interactive.py +118 -0
- sdata/iolib/__init__.py +51 -0
- sdata/iolib/hashtree.py +264 -0
- sdata/iolib/json1sqlitestore.py +554 -0
- sdata/iolib/jsonsqlitestore.py +704 -0
- sdata/iolib/pud.py +242 -0
- sdata/iolib/reader.py +262 -0
- sdata/iolib/writer.py +419 -0
- sdata/metadata.py +882 -0
- sdata/parameter.py +414 -0
- sdata/schema.py +273 -0
- sdata/sclass/__init__.py +111 -0
- sdata/sclass/bfo_classes.py +362 -0
- sdata/sclass/blob.py +362 -0
- sdata/sclass/content.py +87 -0
- sdata/sclass/dataframe.py +1251 -0
- sdata/sclass/dataframegroup.py +187 -0
- sdata/sclass/dependency_graph.py +85 -0
- sdata/sclass/filereference.py +124 -0
- sdata/sclass/image.py +218 -0
- sdata/sclass/lazy_namespace.py +155 -0
- sdata/sclass/process.py +383 -0
- sdata/sclass/process_graph.py +221 -0
- sdata/sclass/zipfile.py +135 -0
- sdata/semantic.py +346 -0
- sdata/suuid.py +136 -0
- sdata/timestamp.py +336 -0
- sdata/tools.py +14 -0
- sdata/units.py +602 -0
- sdata/vocab.py +149 -0
- sdata-1.5.0.dist-info/METADATA +264 -0
- sdata-1.5.0.dist-info/RECORD +98 -0
- sdata-1.5.0.dist-info/WHEEL +5 -0
- sdata-1.5.0.dist-info/licenses/LICENSE-MIT +21 -0
- sdata-1.5.0.dist-info/top_level.txt +1 -0
sdata/deprecated/data.py
ADDED
|
@@ -0,0 +1,1503 @@
|
|
|
1
|
+
# -*-coding: utf-8-*-
|
|
2
|
+
from __future__ import division
|
|
3
|
+
|
|
4
|
+
import sdata
|
|
5
|
+
|
|
6
|
+
'''
|
|
7
|
+
basic sdata types
|
|
8
|
+
'''
|
|
9
|
+
from sdata import __version__
|
|
10
|
+
import sys
|
|
11
|
+
import os
|
|
12
|
+
import uuid
|
|
13
|
+
from collections import OrderedDict
|
|
14
|
+
import logging
|
|
15
|
+
logger = logging.getLogger("sdata")
|
|
16
|
+
import numpy as np
|
|
17
|
+
import pandas as pd
|
|
18
|
+
import shutil
|
|
19
|
+
import copy
|
|
20
|
+
from sdata.metadata import Metadata, Attribute, extract_name_unit
|
|
21
|
+
from sdata.timestamp import now_utc_str, now_local_str, today_str
|
|
22
|
+
import inspect
|
|
23
|
+
import json
|
|
24
|
+
import hashlib
|
|
25
|
+
import base64
|
|
26
|
+
import re
|
|
27
|
+
from sdata.contrib.sqlitedict import SqliteDict
|
|
28
|
+
|
|
29
|
+
if sys.version_info < (3, 0):
|
|
30
|
+
from StringIO import StringIO
|
|
31
|
+
else:
|
|
32
|
+
from io import BytesIO, StringIO
|
|
33
|
+
|
|
34
|
+
class Sdata_Name_Exeption(Exception): pass
|
|
35
|
+
class Sdata_Uuid_Exeption(Exception): pass
|
|
36
|
+
|
|
37
|
+
if sys.version_info < (3, 6):
|
|
38
|
+
import sha3
|
|
39
|
+
|
|
40
|
+
try:
|
|
41
|
+
import openpyxl
|
|
42
|
+
except:
|
|
43
|
+
logger.warning("openpyxl is not available -> no xlsx import")
|
|
44
|
+
|
|
45
|
+
def uuid_from_str(name):
|
|
46
|
+
return uuid.uuid3(uuid.NAMESPACE_DNS, name)
|
|
47
|
+
|
|
48
|
+
def nameunit_from_colname(column_name):
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
"""
|
|
52
|
+
#pattern = r"^(.*?)\s*\[(.+)\]$"
|
|
53
|
+
pattern = r"^(.*?)\s*\[([^\]]+)\]$"
|
|
54
|
+
|
|
55
|
+
# Anwenden des Regex
|
|
56
|
+
match = re.match(pattern, column_name)
|
|
57
|
+
|
|
58
|
+
if match:
|
|
59
|
+
name = match.group(1) # Der Name der Spalte
|
|
60
|
+
unit = match.group(2) # Die Einheit
|
|
61
|
+
else:
|
|
62
|
+
name = column_name
|
|
63
|
+
unit = "-"
|
|
64
|
+
return name.strip(), unit.strip()
|
|
65
|
+
|
|
66
|
+
class Data(object):
|
|
67
|
+
"""Base sdata object"""
|
|
68
|
+
ATTR_NAMES = []
|
|
69
|
+
|
|
70
|
+
SDATA_VERSION = "!sdata_version"
|
|
71
|
+
SDATA_NAME = "!sdata_name"
|
|
72
|
+
SDATA_SNAME = "!sdata_sname"
|
|
73
|
+
SDATA_UUID = "!sdata_uuid"
|
|
74
|
+
SDATA_SUUID = "!sdata_suuid"
|
|
75
|
+
SDATA_CTIME = "!sdata_ctime"
|
|
76
|
+
SDATA_MTIME = "!sdata_mtime"
|
|
77
|
+
SDATA_PARENT = "!sdata_parent"
|
|
78
|
+
SDATA_CLASS = "!sdata_class"
|
|
79
|
+
SDATA_PROJECT = "!sdata_project"
|
|
80
|
+
SDATA_URL = "!sdata_url"
|
|
81
|
+
SDATA_COLUMN = "!sdata_column"
|
|
82
|
+
|
|
83
|
+
SDATA_ATTRIBUTES = [SDATA_VERSION, SDATA_NAME, SDATA_UUID, SDATA_SUUID, SDATA_CLASS, SDATA_PARENT, SDATA_PROJECT,
|
|
84
|
+
SDATA_CTIME, SDATA_MTIME, SDATA_URL]
|
|
85
|
+
|
|
86
|
+
def __init__(self, **kwargs):
|
|
87
|
+
"""create Data object
|
|
88
|
+
|
|
89
|
+
.. code-block:: python
|
|
90
|
+
|
|
91
|
+
df = pd.DataFrame([1,2,3])
|
|
92
|
+
data = sdata.Data(name='my name',
|
|
93
|
+
uuid='38b26864e7794f5182d38459bab85842',
|
|
94
|
+
table=df,
|
|
95
|
+
description="A remarkable description")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
:param name: name of the data object
|
|
99
|
+
:param table: pandas.DataFrame to store
|
|
100
|
+
:param uuid: uuid of the object
|
|
101
|
+
:param metadata: sdata.Metadata object
|
|
102
|
+
:param description: a string to describe the object
|
|
103
|
+
"""
|
|
104
|
+
|
|
105
|
+
# RFC 0012: Data ist deprecated -> sdata.sclass.dataframe.DataFrame.
|
|
106
|
+
# Nur bei DIREKTER Instanziierung warnen, damit Domänen-/Experiment-
|
|
107
|
+
# Subklassen (Pud, TestProgram, …) nicht gestört werden (deren Ablösung
|
|
108
|
+
# ist Teil von Stufe 2 / 2.0).
|
|
109
|
+
if type(self) is Data:
|
|
110
|
+
import warnings
|
|
111
|
+
warnings.warn(
|
|
112
|
+
"sdata.Data is deprecated (RFC 0012); use "
|
|
113
|
+
"sdata.sclass.dataframe.DataFrame. Removal in sdata 2.0.",
|
|
114
|
+
DeprecationWarning, stacklevel=2)
|
|
115
|
+
|
|
116
|
+
# self._uuid = None
|
|
117
|
+
# self._name = None
|
|
118
|
+
self._prefix = None
|
|
119
|
+
|
|
120
|
+
# ToDo: add getter and setter for metadata
|
|
121
|
+
# self.metadata = kwargs.get("metadata") or Metadata()
|
|
122
|
+
|
|
123
|
+
self.metadata = Metadata()
|
|
124
|
+
|
|
125
|
+
# set default sdata attributes
|
|
126
|
+
self.metadata.add(self.SDATA_VERSION, __version__, dtype="str", description="sdata package version")
|
|
127
|
+
self.metadata.add(self.SDATA_NAME, "N.N.", dtype="str", description="name of the data object")
|
|
128
|
+
self.metadata.add(self.SDATA_SNAME, "", dtype="str", description="sname of the data object")
|
|
129
|
+
self.metadata.add(self.SDATA_UUID, "", dtype="str", description="Universally Unique Identifier")
|
|
130
|
+
self.metadata.add(self.SDATA_SUUID, "", dtype="str", description="Super Universally Unique Identifier")
|
|
131
|
+
self.metadata.add(self.SDATA_PARENT, "", dtype="str", description="uuid of the parent sdata object")
|
|
132
|
+
self.metadata.add(self.SDATA_CLASS, self.__class__.__name__, dtype="str", description="sdata class")
|
|
133
|
+
self.metadata.add(self.SDATA_CTIME, now_utc_str(), dtype="str", description="creation date")
|
|
134
|
+
self.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str", description="modification date")
|
|
135
|
+
self.metadata.add(self.SDATA_URL, "", dtype="str", description="url of the data object")
|
|
136
|
+
self.metadata.add(self.SDATA_PROJECT, "", dtype="str", description="project name")
|
|
137
|
+
|
|
138
|
+
metadata = kwargs.get("metadata")
|
|
139
|
+
if metadata is not None:
|
|
140
|
+
# logger.debug("Data got Metadata {}".format(metadata))
|
|
141
|
+
if metadata and isinstance(metadata, Metadata):
|
|
142
|
+
for attribute in metadata.attributes.values():
|
|
143
|
+
# logger.debug("Data.Metadata.add {0.name}:{0.value}".format(attribute))
|
|
144
|
+
self.metadata.add(attribute)
|
|
145
|
+
|
|
146
|
+
# auto correct
|
|
147
|
+
if kwargs.get("auto_correct") is None or kwargs.get("auto_correct") is True:
|
|
148
|
+
self.auto_correct = True
|
|
149
|
+
else:
|
|
150
|
+
self.auto_correct = False
|
|
151
|
+
# logger.debug("sdata: set auto_correct={}".format(self.auto_correct))
|
|
152
|
+
|
|
153
|
+
if kwargs.get("name") is not None:
|
|
154
|
+
self.name = kwargs.get("name")
|
|
155
|
+
|
|
156
|
+
self.prefix = kwargs.get("prefix") or ""
|
|
157
|
+
self._gen_default_attributes(kwargs.get("default_attributes") or self.ATTR_NAMES)
|
|
158
|
+
self._group = OrderedDict()
|
|
159
|
+
self._table = None # pd.DataFrame()
|
|
160
|
+
self.table = kwargs.get("table", None)
|
|
161
|
+
self._description = ""
|
|
162
|
+
self.description = kwargs.get("description", "")
|
|
163
|
+
self.project = kwargs.get("project", "N.N.")
|
|
164
|
+
self.metadata.add(self.SDATA_PROJECT, self.project, dtype="str", description="project name")
|
|
165
|
+
|
|
166
|
+
if (kwargs.get("uuid")=="" or kwargs.get("uuid") is not None) and not self.metadata.get(self.SDATA_UUID).value and kwargs.get("uuid")!="hash":
|
|
167
|
+
# logger.info("uuid in kwargs")
|
|
168
|
+
try:
|
|
169
|
+
self._set_uuid(kwargs.get("uuid")) # store given uuid str or generate a new uuid
|
|
170
|
+
except Sdata_Uuid_Exeption as exp:
|
|
171
|
+
if self.auto_correct is True:
|
|
172
|
+
logger.warning("got invalid uuid -> generate a new uuid")
|
|
173
|
+
self._set_uuid(uuid.uuid4())
|
|
174
|
+
else:
|
|
175
|
+
raise
|
|
176
|
+
elif (kwargs.get("uuid")=="" or kwargs.get("uuid") is None) and self.metadata.get(self.SDATA_UUID).value != "":
|
|
177
|
+
# logger.info("uuid in metadata")
|
|
178
|
+
pass
|
|
179
|
+
elif kwargs.get("uuid")=="hash":
|
|
180
|
+
sha3_256 = self.gen_uuid_from_state()
|
|
181
|
+
# logger.info("gen uuid from sha3_256 {}".format(sha3_256))
|
|
182
|
+
new_uuid = uuid_from_str(sha3_256)
|
|
183
|
+
self._set_uuid(new_uuid.hex)
|
|
184
|
+
else:
|
|
185
|
+
# logger.info("uuid new")
|
|
186
|
+
self._set_uuid(uuid.uuid4())
|
|
187
|
+
|
|
188
|
+
if "suuid" not in kwargs:
|
|
189
|
+
self.gen_suuid()
|
|
190
|
+
else:
|
|
191
|
+
suuid = sdata.SUUID.from_suuid_str(kwargs.get("suuid"))
|
|
192
|
+
self.suuid = kwargs.get("suuid")
|
|
193
|
+
self.sname = suuid.sname
|
|
194
|
+
|
|
195
|
+
if "sname" in kwargs:
|
|
196
|
+
self.sname = kwargs.get("sname")
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
if "parent" in kwargs:
|
|
200
|
+
self.metadata.add(self.SDATA_PARENT, kwargs.get("parent"))
|
|
201
|
+
|
|
202
|
+
def gen_suuid(self):
|
|
203
|
+
self.suuid = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).suuid_str
|
|
204
|
+
self.sname = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).sname
|
|
205
|
+
|
|
206
|
+
def gen_uuid_from_state(self):
|
|
207
|
+
"""generate the same uuid for the same data
|
|
208
|
+
|
|
209
|
+
:return: uuid
|
|
210
|
+
"""
|
|
211
|
+
s = hashlib.sha3_256()
|
|
212
|
+
metadata = self.metadata.copy()
|
|
213
|
+
metadata.attributes.pop(self.SDATA_UUID)
|
|
214
|
+
metadata.attributes.pop(self.SDATA_MTIME)
|
|
215
|
+
metadata.attributes.pop(self.SDATA_CTIME)
|
|
216
|
+
metadatastr = metadata.to_json().encode(errors="replace")
|
|
217
|
+
s.update(metadatastr)
|
|
218
|
+
if self.table is not None:
|
|
219
|
+
tablestr = self.table.to_json().encode(errors="replace")
|
|
220
|
+
s.update(tablestr)
|
|
221
|
+
s.update(self.description.encode(errors="replace"))
|
|
222
|
+
return s.hexdigest()
|
|
223
|
+
|
|
224
|
+
def __eq__(self, other):
|
|
225
|
+
"""compare Data checksum
|
|
226
|
+
|
|
227
|
+
:param other: sdata.Data object
|
|
228
|
+
:return: True or False
|
|
229
|
+
"""
|
|
230
|
+
if not isinstance(other, self.__class__):
|
|
231
|
+
logger.debug("you should not compare {} with {}!".format(self.__class__.__name__, other.__class__.__name__))
|
|
232
|
+
return False
|
|
233
|
+
return self.sha3_256 == other.sha3_256
|
|
234
|
+
|
|
235
|
+
def rename(self, name, random=True):
|
|
236
|
+
"""change name, uuid and suuid
|
|
237
|
+
"""
|
|
238
|
+
self.name = name
|
|
239
|
+
project = self.metadata.get(self.SDATA_PROJECT).value
|
|
240
|
+
if random is False:
|
|
241
|
+
suuid = sdata.SUUID.from_name(self.__class__.__name__, name, ns_name=project)
|
|
242
|
+
else:
|
|
243
|
+
suuid = sdata.SUUID(self.__class__.__name__, name)
|
|
244
|
+
self._set_suuid(suuid.suuid_str)
|
|
245
|
+
self._set_uuid(suuid.huuid)
|
|
246
|
+
return self
|
|
247
|
+
|
|
248
|
+
@property
|
|
249
|
+
def m(self):
|
|
250
|
+
"""get metadata"""
|
|
251
|
+
return self.metadata
|
|
252
|
+
|
|
253
|
+
@property
|
|
254
|
+
def attrs(self):
|
|
255
|
+
"""return user metadata as dict"""
|
|
256
|
+
return self.metadata.get_udict()
|
|
257
|
+
|
|
258
|
+
def update_mtime(self):
|
|
259
|
+
"""update modification time
|
|
260
|
+
|
|
261
|
+
:return:
|
|
262
|
+
"""
|
|
263
|
+
self.metadata.add(self.SDATA_MTIME, now_utc_str())
|
|
264
|
+
|
|
265
|
+
@property
|
|
266
|
+
def sha3_256_table(self):
|
|
267
|
+
"""Return a SHA3 hash of the sData.table object with a hashbit length of 32 bytes.
|
|
268
|
+
|
|
269
|
+
.. code-block:: python
|
|
270
|
+
|
|
271
|
+
sdata.Data(name="1", uuid=sdata.uuid_from_str("1")).sha3_256_table
|
|
272
|
+
|
|
273
|
+
'c468e659891eb5dea6eb6baf73f51ca0688792bf9ad723209dc22730903f6efa'
|
|
274
|
+
|
|
275
|
+
:return: hashlib.sha3_256.hexdigest()
|
|
276
|
+
"""
|
|
277
|
+
s = hashlib.sha3_256()
|
|
278
|
+
if self.table is not None:
|
|
279
|
+
tablestr = self.table.to_json().encode(errors="replace")
|
|
280
|
+
s.update(tablestr)
|
|
281
|
+
return s.hexdigest()
|
|
282
|
+
|
|
283
|
+
@property
|
|
284
|
+
def sha3_256_metadata(self):
|
|
285
|
+
s = hashlib.sha3_256()
|
|
286
|
+
metadatastr = self.metadata.to_json().encode(errors="replace")
|
|
287
|
+
s.update(metadatastr)
|
|
288
|
+
return s.hexdigest()
|
|
289
|
+
|
|
290
|
+
@property
|
|
291
|
+
def sha3_256(self):
|
|
292
|
+
"""Return a SHA3 hash of the sData object with a hashbit length of 32 bytes.
|
|
293
|
+
|
|
294
|
+
.. code-block:: python
|
|
295
|
+
|
|
296
|
+
sdata.Data(name="1", uuid=sdata.uuid_from_str("1")).sha3_256
|
|
297
|
+
|
|
298
|
+
'c468e659891eb5dea6eb6baf73f51ca0688792bf9ad723209dc22730903f6efa'
|
|
299
|
+
|
|
300
|
+
:return: hashlib.sha3_256.hexdigest()
|
|
301
|
+
"""
|
|
302
|
+
s = hashlib.sha3_256()
|
|
303
|
+
metadatastr = self.metadata.to_json().encode(errors="replace")
|
|
304
|
+
s.update(metadatastr)
|
|
305
|
+
if self.table is not None:
|
|
306
|
+
# tablestr = self.table.to_json().encode(errors="replace")
|
|
307
|
+
s.update(self.sha3_256_table.encode(errors="replace"))
|
|
308
|
+
s.update(self.description.encode(errors="replace"))
|
|
309
|
+
return s.hexdigest()
|
|
310
|
+
|
|
311
|
+
def update_hash(self, hashobject):
|
|
312
|
+
"""A hash represents the object used to calculate a checksum of a
|
|
313
|
+
string of information.
|
|
314
|
+
|
|
315
|
+
.. code-block:: python
|
|
316
|
+
|
|
317
|
+
data = sdata.Data()
|
|
318
|
+
|
|
319
|
+
md5 = hashlib.md5()
|
|
320
|
+
data.update_hash(md5)
|
|
321
|
+
md5.hexdigest()
|
|
322
|
+
'bbf323bdcb0bf961803b5504a8a60d69'
|
|
323
|
+
|
|
324
|
+
sha1 = hashlib.sha1()
|
|
325
|
+
data.update_hash(sha1)
|
|
326
|
+
sha1.hexdigest()
|
|
327
|
+
'3c59368c7735c1ecaf03ebd4c595bb6e73e90f0c'
|
|
328
|
+
|
|
329
|
+
hashobject = hashlib.sha3_256()
|
|
330
|
+
data.update_hash(hashobject).hexdigest()
|
|
331
|
+
'c468e659891eb5dea6eb6baf73f51ca0688792bf9ad723209dc22730903f6efa'
|
|
332
|
+
|
|
333
|
+
data.update_hash(hashobject).digest()
|
|
334
|
+
b'M8...'
|
|
335
|
+
|
|
336
|
+
:param hash: hash object, e.g. hashlib.sha1()
|
|
337
|
+
:return: hash
|
|
338
|
+
"""
|
|
339
|
+
if not (hasattr(hashobject, "update") and hasattr(hashobject, "hexdigest")):
|
|
340
|
+
logger.error("Data.update_hash: given hashfunction is invalid")
|
|
341
|
+
raise Exception("Data.update_hash: given hashfunction is invalid")
|
|
342
|
+
|
|
343
|
+
metadatastr = self.metadata.to_json().encode(errors="replace")
|
|
344
|
+
hashobject.update(metadatastr)
|
|
345
|
+
if self.table is not None:
|
|
346
|
+
tablestr = self.table.to_json().encode(errors="replace")
|
|
347
|
+
hashobject.update(tablestr)
|
|
348
|
+
hashobject.update(self.description.encode(errors="replace"))
|
|
349
|
+
return hashobject
|
|
350
|
+
|
|
351
|
+
def describe(self):
|
|
352
|
+
"""Generate descriptive info of the data
|
|
353
|
+
|
|
354
|
+
.. code-block:: python
|
|
355
|
+
|
|
356
|
+
df = pd.DataFrame([1,2,3])
|
|
357
|
+
data = sdata.Data(name='my name',
|
|
358
|
+
uuid='38b26864e7794f5182d38459bab85842',
|
|
359
|
+
table=df,
|
|
360
|
+
description="A remarkable description")
|
|
361
|
+
data.describe()
|
|
362
|
+
|
|
363
|
+
.. code-block:: none
|
|
364
|
+
|
|
365
|
+
0
|
|
366
|
+
metadata 3
|
|
367
|
+
table_rows 3
|
|
368
|
+
table_columns 1
|
|
369
|
+
description 24
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
:return: pd.DataFrame
|
|
373
|
+
"""
|
|
374
|
+
df = pd.DataFrame({0: []}, dtype=object)
|
|
375
|
+
df.loc["metadata", 0] = self.metadata.size
|
|
376
|
+
if self.table is None:
|
|
377
|
+
df.loc["table_rows"] = 0
|
|
378
|
+
df.loc["table_columns"] = 0
|
|
379
|
+
else:
|
|
380
|
+
df.loc["table_rows"] = len(self.table)
|
|
381
|
+
df.loc["table_columns"] = len(self.table.columns)
|
|
382
|
+
df.loc["description", 0] = len(self.description)
|
|
383
|
+
return df
|
|
384
|
+
|
|
385
|
+
def _gen_default_attributes(self, default_attributes):
|
|
386
|
+
"""create default Attributes in data.metadata"""
|
|
387
|
+
for attr_name, value, dtype, unit, description, required in default_attributes:
|
|
388
|
+
self.metadata.set_attr(name=attr_name, value=value, dtype=dtype, description=description)
|
|
389
|
+
|
|
390
|
+
def _get_suuid(self):
|
|
391
|
+
return self.metadata.get(self.SDATA_SUUID).value
|
|
392
|
+
|
|
393
|
+
def _set_suuid(self, value):
|
|
394
|
+
self.metadata.set_attr(self.SDATA_SUUID, value)
|
|
395
|
+
|
|
396
|
+
suuid = property(fget=_get_suuid, fset=_set_suuid, doc="suuid of the object")
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def _get_sname(self):
|
|
400
|
+
return self.metadata.get(self.SDATA_SNAME).value
|
|
401
|
+
|
|
402
|
+
def _set_sname(self, value):
|
|
403
|
+
self.metadata.set_attr(self.SDATA_SNAME, value)
|
|
404
|
+
|
|
405
|
+
sname = property(fget=_get_sname, fset=_set_sname, doc="sname of the object")
|
|
406
|
+
|
|
407
|
+
def _get_uuid(self):
|
|
408
|
+
return self.metadata.get(self.SDATA_UUID).value
|
|
409
|
+
# return self._uuid
|
|
410
|
+
|
|
411
|
+
def _set_uuid(self, value):
|
|
412
|
+
if isinstance(value, str):
|
|
413
|
+
try:
|
|
414
|
+
uuid.UUID(value)
|
|
415
|
+
self.metadata.set_attr(self.SDATA_UUID, uuid.UUID(value).hex)
|
|
416
|
+
except ValueError as exp:
|
|
417
|
+
logger.warning("data.uuid: %s" % exp)
|
|
418
|
+
raise Sdata_Uuid_Exeption("got invalid uuid str '{}'".format(str(value)))
|
|
419
|
+
elif isinstance(value, uuid.UUID):
|
|
420
|
+
self.metadata.set_attr(self.SDATA_UUID, value.hex)
|
|
421
|
+
else:
|
|
422
|
+
logger.error("Data.uuid: invalid uuid '{}'".format(value))
|
|
423
|
+
raise Exception("Data.uuid: invalid uuid '{}'".format(value))
|
|
424
|
+
|
|
425
|
+
uuid = property(fget=_get_uuid, fset=_set_uuid, doc="uuid of the object")
|
|
426
|
+
|
|
427
|
+
def _get_name(self):
|
|
428
|
+
# return self._name
|
|
429
|
+
return self.metadata.get(self.SDATA_NAME).value
|
|
430
|
+
|
|
431
|
+
def _set_name(self, value):
|
|
432
|
+
if isinstance(value, str):
|
|
433
|
+
try:
|
|
434
|
+
self.metadata.set_attr(self.SDATA_NAME, str(value)[:256])
|
|
435
|
+
except ValueError as exp:
|
|
436
|
+
logger.warning("data.name: %s" % exp)
|
|
437
|
+
else:
|
|
438
|
+
# self._name = str(value)[:256]
|
|
439
|
+
self.metadata.set_attr(self.SDATA_NAME, str(value)[:256])
|
|
440
|
+
|
|
441
|
+
name = property(fget=_get_name, fset=_set_name, doc="name of the object")
|
|
442
|
+
|
|
443
|
+
def _get_project(self):
|
|
444
|
+
return self.metadata.get(self.SDATA_PROJECT).value
|
|
445
|
+
|
|
446
|
+
def _set_project(self, value):
|
|
447
|
+
if isinstance(value, str):
|
|
448
|
+
try:
|
|
449
|
+
self.metadata.set_attr(self.SDATA_PROJECT, str(value)[:256])
|
|
450
|
+
except ValueError as exp:
|
|
451
|
+
logger.warning("data.project: %s" % exp)
|
|
452
|
+
else:
|
|
453
|
+
# self._name = str(value)[:256]
|
|
454
|
+
self.metadata.set_attr(self.SDATA_PROJECT, str(value)[:256])
|
|
455
|
+
|
|
456
|
+
project = property(fget=_get_project, fset=_set_project, doc="name of the project")
|
|
457
|
+
|
|
458
|
+
def _get_description(self):
|
|
459
|
+
return self._description
|
|
460
|
+
|
|
461
|
+
def _set_description(self, value):
|
|
462
|
+
if isinstance(value, str):
|
|
463
|
+
try:
|
|
464
|
+
self._description = str(value)
|
|
465
|
+
except ValueError as exp:
|
|
466
|
+
logger.warning("data.name: %s" % exp)
|
|
467
|
+
else:
|
|
468
|
+
self._description = str(value)
|
|
469
|
+
|
|
470
|
+
description = property(fget=_get_description, fset=_set_description, doc="description of the object")
|
|
471
|
+
|
|
472
|
+
@property
|
|
473
|
+
def filename(self):
|
|
474
|
+
|
|
475
|
+
validchars = "-_.() "
|
|
476
|
+
out = ""
|
|
477
|
+
|
|
478
|
+
name = "{}".format(self.name)
|
|
479
|
+
|
|
480
|
+
for c in name:
|
|
481
|
+
if str.isalpha(c) or str.isdigit(c) or (c in validchars):
|
|
482
|
+
out += c
|
|
483
|
+
else:
|
|
484
|
+
out += "_"
|
|
485
|
+
return out
|
|
486
|
+
|
|
487
|
+
def _get_prefix(self):
|
|
488
|
+
return self._prefix
|
|
489
|
+
|
|
490
|
+
def _set_prefix(self, value):
|
|
491
|
+
if isinstance(value, str):
|
|
492
|
+
try:
|
|
493
|
+
self._prefix = value[:256]
|
|
494
|
+
except ValueError as exp:
|
|
495
|
+
logger.warning("data.prefix: %s" % exp)
|
|
496
|
+
else:
|
|
497
|
+
self._prefix = str(value)[:256]
|
|
498
|
+
|
|
499
|
+
prefix = property(fget=_get_prefix, fset=_set_prefix, doc="prefix of the object name")
|
|
500
|
+
|
|
501
|
+
def _get_table(self):
|
|
502
|
+
return self._table
|
|
503
|
+
|
|
504
|
+
def _set_table(self, df):
|
|
505
|
+
if isinstance(df, pd.DataFrame):
|
|
506
|
+
self._table = df
|
|
507
|
+
if self._table.index.name is None:
|
|
508
|
+
self._table.index.name = "index"
|
|
509
|
+
|
|
510
|
+
table = property(fget=_get_table, fset=_set_table, doc="table object(pandas.DataFrame)")
|
|
511
|
+
df = table
|
|
512
|
+
|
|
513
|
+
def description_to_df(self):
|
|
514
|
+
"""get description as DataFrame
|
|
515
|
+
|
|
516
|
+
:return: DataFrame of description lines
|
|
517
|
+
"""
|
|
518
|
+
return pd.DataFrame(self.description.splitlines())
|
|
519
|
+
|
|
520
|
+
def description_from_df(self, df):
|
|
521
|
+
"""set description from DataFrame of lines
|
|
522
|
+
|
|
523
|
+
:return:
|
|
524
|
+
"""
|
|
525
|
+
if df is not None and isinstance(df, pd.DataFrame) and len(df)>0:
|
|
526
|
+
lines = df.iloc[:, 0]
|
|
527
|
+
lines = lines.astype(str)
|
|
528
|
+
self.description = "\n".join(lines.values)
|
|
529
|
+
|
|
530
|
+
def to_folder(self, path, dtype="csv"):
|
|
531
|
+
"""export data to folder
|
|
532
|
+
|
|
533
|
+
:param path:
|
|
534
|
+
:param dtype:
|
|
535
|
+
:return:
|
|
536
|
+
"""
|
|
537
|
+
|
|
538
|
+
if dtype not in ["csv", "xlsx"]:
|
|
539
|
+
dtype = "xlsx"
|
|
540
|
+
if not os.path.exists(path):
|
|
541
|
+
try:
|
|
542
|
+
os.makedirs(path)
|
|
543
|
+
except OSError as exp:
|
|
544
|
+
logger.error(exp)
|
|
545
|
+
else:
|
|
546
|
+
self.clear_folder(path)
|
|
547
|
+
|
|
548
|
+
self.metadata.set_attr(name="class", value=self.__class__.__name__, description="object class", unit="-",
|
|
549
|
+
dtype="str")
|
|
550
|
+
self.metadata.set_attr(name="uuid", value=self.uuid, description="object uuid", unit="-", dtype="str")
|
|
551
|
+
self.metadata.set_attr(name="name", value=self.name, description="object name", unit="-", dtype="str")
|
|
552
|
+
|
|
553
|
+
if dtype == "csv":
|
|
554
|
+
metadata_filepath = os.path.join(path, "metadata.csv")
|
|
555
|
+
logger.debug("export meta csv '{}'".format(metadata_filepath))
|
|
556
|
+
self.metadata.to_csv(metadata_filepath)
|
|
557
|
+
|
|
558
|
+
# table export
|
|
559
|
+
if isinstance(self._table, pd.DataFrame) and len(self._table) > 0:
|
|
560
|
+
exportpath = os.path.join(path, "{}.csv".format(self.osname))
|
|
561
|
+
self._table.to_csv(exportpath, index=False)
|
|
562
|
+
if dtype == "xlsx":
|
|
563
|
+
if not isinstance(self._table, pd.DataFrame):
|
|
564
|
+
self.table = pd.DataFrame()
|
|
565
|
+
exportpath = os.path.join(path, "{}.xlsx".format(self.osname))
|
|
566
|
+
self.to_xlsx(exportpath)
|
|
567
|
+
# group export
|
|
568
|
+
for data in self.group.values():
|
|
569
|
+
exportpath = os.path.join(path, "{}-{}".format(data.__class__.__name__.lower(), data.osname))
|
|
570
|
+
data.to_folder(exportpath, dtype=dtype)
|
|
571
|
+
return path
|
|
572
|
+
|
|
573
|
+
def set_column_metadata(self):
|
|
574
|
+
"""convert 'Force [N]' -> 'Force', 'N'
|
|
575
|
+
|
|
576
|
+
add Attribute for each column
|
|
577
|
+
|
|
578
|
+
sdata_column_0: value="Force", unit="N"
|
|
579
|
+
sdata_column_1
|
|
580
|
+
...
|
|
581
|
+
sdata_column_n
|
|
582
|
+
|
|
583
|
+
|
|
584
|
+
"""
|
|
585
|
+
if self.df is None:
|
|
586
|
+
return
|
|
587
|
+
for i, col in enumerate(self.df.columns):
|
|
588
|
+
name, unit = nameunit_from_colname(col)
|
|
589
|
+
scolname = f"{self.SDATA_COLUMN}_{i}"
|
|
590
|
+
self.metadata.add(scolname, value=name, unit=unit, label=col)
|
|
591
|
+
|
|
592
|
+
def set_columnnames_from_metadata(self):
|
|
593
|
+
"""Use '!sdata_column_*' to reformat columns names
|
|
594
|
+
|
|
595
|
+
'!sdata_column_0', value='Force', unit='N' -> 'Force [N]'
|
|
596
|
+
|
|
597
|
+
> f'{name} [{unit}]'
|
|
598
|
+
|
|
599
|
+
"""
|
|
600
|
+
if self.df is None:
|
|
601
|
+
return
|
|
602
|
+
|
|
603
|
+
cols = []
|
|
604
|
+
for i, col in enumerate(self.df.columns):
|
|
605
|
+
scolname = f"{self.SDATA_COLUMN}_{i}"
|
|
606
|
+
attr = self.metadata.get(scolname)
|
|
607
|
+
if attr is None:
|
|
608
|
+
name, unit = nameunit_from_colname(col)
|
|
609
|
+
else:
|
|
610
|
+
name = attr.value
|
|
611
|
+
unit = attr.unit
|
|
612
|
+
new_colname = f'{name} [{unit}]'
|
|
613
|
+
cols.append(new_colname)
|
|
614
|
+
self.df.columns = cols
|
|
615
|
+
|
|
616
|
+
@classmethod
|
|
617
|
+
def from_folder(cls, path):
|
|
618
|
+
"""sdata object instance
|
|
619
|
+
|
|
620
|
+
:param path:
|
|
621
|
+
:return:
|
|
622
|
+
"""
|
|
623
|
+
# data = Data.from_folder(path)
|
|
624
|
+
|
|
625
|
+
data = cls()
|
|
626
|
+
if not os.path.exists(path):
|
|
627
|
+
logger.error("from_folder error: path '{}' not exists.".format(path))
|
|
628
|
+
return data
|
|
629
|
+
|
|
630
|
+
data.metadata = data._load_metadata(path)
|
|
631
|
+
try:
|
|
632
|
+
data.uuid = data.metadata.get_attr("uuid").value
|
|
633
|
+
data.name = data.metadata.get_attr("name").value
|
|
634
|
+
except Exception as exp:
|
|
635
|
+
logger.error("Data.from_folder: {}".format(data.metadata.to_dict()))
|
|
636
|
+
raise
|
|
637
|
+
|
|
638
|
+
# table import
|
|
639
|
+
files = [x for x in os.listdir(path) if
|
|
640
|
+
not os.path.isdir(os.path.join(path, x)) and not x.startswith("metadata")]
|
|
641
|
+
if len(files) == 1:
|
|
642
|
+
assert len(files) == 1, "invalid number of files for Table '{}'".format(files)
|
|
643
|
+
importpath = os.path.join(path, files[0])
|
|
644
|
+
print("read table {}".format(importpath))
|
|
645
|
+
# data._table = pd.read_csv(importpath)
|
|
646
|
+
|
|
647
|
+
if not os.path.exists(path):
|
|
648
|
+
return cls()
|
|
649
|
+
metadata = cls._load_metadata(path)
|
|
650
|
+
data = cls()
|
|
651
|
+
data.metadata = metadata
|
|
652
|
+
data.uuid = data.metadata.get_attr("uuid").value
|
|
653
|
+
data.name = data.metadata.get_attr("name").value
|
|
654
|
+
|
|
655
|
+
folders = [x for x in os.listdir(path) if os.path.isdir(os.path.join(path, x))]
|
|
656
|
+
for folder in folders:
|
|
657
|
+
subfolder = os.path.join(path, folder)
|
|
658
|
+
data_ = data.from_folder(subfolder)
|
|
659
|
+
subdata = data_.from_folder(subfolder)
|
|
660
|
+
data.add_data(subdata)
|
|
661
|
+
return data
|
|
662
|
+
|
|
663
|
+
@staticmethod
|
|
664
|
+
def clear_folder(path):
|
|
665
|
+
"""delete subfolder in export folder
|
|
666
|
+
|
|
667
|
+
:param path: path
|
|
668
|
+
:return: None
|
|
669
|
+
"""
|
|
670
|
+
|
|
671
|
+
def is_valid(path):
|
|
672
|
+
prefix = path.split("-")[0]
|
|
673
|
+
if prefix in [x.lower() for x in SDATACLS.keys()]:
|
|
674
|
+
return True
|
|
675
|
+
else:
|
|
676
|
+
return False
|
|
677
|
+
|
|
678
|
+
subfolders = [x for x in os.listdir(path) if os.path.isdir(os.path.join(path, x))]
|
|
679
|
+
valid_subfolders = [x for x in subfolders if is_valid(x)]
|
|
680
|
+
for subfolder in valid_subfolders:
|
|
681
|
+
try:
|
|
682
|
+
subfolder = os.path.join(path, subfolder)
|
|
683
|
+
logger.debug("clear_folder: rm {}".format(subfolder))
|
|
684
|
+
shutil.rmtree(subfolder)
|
|
685
|
+
except OSError as exp:
|
|
686
|
+
raise
|
|
687
|
+
|
|
688
|
+
@staticmethod
|
|
689
|
+
def _load_metadata(path):
|
|
690
|
+
"""load metadata from csv
|
|
691
|
+
|
|
692
|
+
:returns: Metadata instance"""
|
|
693
|
+
metadata_filepath = os.path.join(path, "metadata.csv")
|
|
694
|
+
if os.path.exists(metadata_filepath):
|
|
695
|
+
metadata = Metadata().from_csv(metadata_filepath)
|
|
696
|
+
else:
|
|
697
|
+
metadata = Metadata()
|
|
698
|
+
return metadata
|
|
699
|
+
|
|
700
|
+
@staticmethod
|
|
701
|
+
def _get_class_from_metadata(metadata):
|
|
702
|
+
"""get class object from metadata
|
|
703
|
+
|
|
704
|
+
:returns: relevant sdata class object"""
|
|
705
|
+
classattr = metadata.get_attr("class")
|
|
706
|
+
if classattr is not None:
|
|
707
|
+
sdataclassname = classattr.value
|
|
708
|
+
sdatacls = SDATACLS.get(sdataclassname)
|
|
709
|
+
if sdataclassname not in SDATACLS:
|
|
710
|
+
logger.warning("unsupported cls '{}'".format(sdataclassname))
|
|
711
|
+
sdatacls = Data
|
|
712
|
+
else:
|
|
713
|
+
logger.warning("cls not defined '{}'".format(metadata))
|
|
714
|
+
sdatacls = None
|
|
715
|
+
return sdatacls
|
|
716
|
+
|
|
717
|
+
@property
|
|
718
|
+
def osname(self):
|
|
719
|
+
""":returns: os compatible name (ascii?)"""
|
|
720
|
+
return self.asciiname.lower()
|
|
721
|
+
|
|
722
|
+
@property
|
|
723
|
+
def asciiname(self):
|
|
724
|
+
name = copy.copy(self.name)
|
|
725
|
+
mapper = [("ä", "ae"), ("ö", "oe"), ("ü", "ue"), ("Ä", "Ae"), ("Ö", "Oe"), ("Ü", "Ue"),
|
|
726
|
+
("ß", "sz"), (" ", "_"), ("/", "_"), ("\\", "_")]
|
|
727
|
+
for k, v in mapper:
|
|
728
|
+
name = name.replace(k, v)
|
|
729
|
+
return name.encode('ascii', 'replace').decode("ascii")
|
|
730
|
+
|
|
731
|
+
def verify_attributes(self):
|
|
732
|
+
"""check mandatory attributes"""
|
|
733
|
+
invalid_attrs = []
|
|
734
|
+
# attr_defs = ["name", "value", "dtype", "unit", "description", "required"]
|
|
735
|
+
for attr_defs in self.ATTR_NAMES:
|
|
736
|
+
required = attr_defs[5]
|
|
737
|
+
if required is False:
|
|
738
|
+
continue
|
|
739
|
+
attr = self.metadata.get_attr(attr_defs[0])
|
|
740
|
+
if attr is None:
|
|
741
|
+
invalid_attrs.append(attr_defs[0])
|
|
742
|
+
elif attr.value is None:
|
|
743
|
+
invalid_attrs.append(attr.name)
|
|
744
|
+
return invalid_attrs
|
|
745
|
+
|
|
746
|
+
def __str__(self):
|
|
747
|
+
return f"({self.__class__.__name__} '{self.name}':{self.uuid})"
|
|
748
|
+
|
|
749
|
+
__repr__ = __str__
|
|
750
|
+
@property
|
|
751
|
+
def mdf(self):
|
|
752
|
+
return self.metadata.df
|
|
753
|
+
|
|
754
|
+
@property
|
|
755
|
+
def udf(self):
|
|
756
|
+
return self.metadata.udf
|
|
757
|
+
|
|
758
|
+
@property
|
|
759
|
+
def sdf(self):
|
|
760
|
+
return self.metadata.sdf
|
|
761
|
+
|
|
762
|
+
@property
|
|
763
|
+
def md(self):
|
|
764
|
+
return self.metadata
|
|
765
|
+
|
|
766
|
+
def get_group(self):
|
|
767
|
+
return self._group
|
|
768
|
+
|
|
769
|
+
group = property(get_group, doc="get group")
|
|
770
|
+
|
|
771
|
+
def keys(self):
|
|
772
|
+
"""get all child objects uuids
|
|
773
|
+
|
|
774
|
+
:return: list of uuid's
|
|
775
|
+
"""
|
|
776
|
+
return list(self.group.keys())
|
|
777
|
+
|
|
778
|
+
def values(self):
|
|
779
|
+
"""get all child objects
|
|
780
|
+
|
|
781
|
+
:return: list of child objects
|
|
782
|
+
"""
|
|
783
|
+
return list(self.group.values())
|
|
784
|
+
|
|
785
|
+
def items(self):
|
|
786
|
+
"""get all child objects
|
|
787
|
+
|
|
788
|
+
:return: [(child uuid, child objects), ]
|
|
789
|
+
"""
|
|
790
|
+
return list(self.group.items())
|
|
791
|
+
|
|
792
|
+
def clear_group(self):
|
|
793
|
+
"""clear group dict"""
|
|
794
|
+
self._group = OrderedDict()
|
|
795
|
+
|
|
796
|
+
def add_data(self, data):
|
|
797
|
+
"""add data, if data.name is unique"""
|
|
798
|
+
if hasattr(data, "metadata"):
|
|
799
|
+
names = [dat.name.lower() for uid, dat in self.group.items()]
|
|
800
|
+
if data.name.lower() in names:
|
|
801
|
+
logger.error("{}: name '{}' aready exists".format(data.__class__.__name__, data.name))
|
|
802
|
+
return
|
|
803
|
+
self.group[data.uuid] = data
|
|
804
|
+
else:
|
|
805
|
+
logger.warning(f"ignore data {data}, {data.__class__.__name__} (wrong type!)")
|
|
806
|
+
|
|
807
|
+
def get_data_by_uuid(self, uid):
|
|
808
|
+
"""get data by uuid"""
|
|
809
|
+
return self.group.get(uid)
|
|
810
|
+
|
|
811
|
+
def get_data_by_name(self, name):
|
|
812
|
+
""":return obj by name"""
|
|
813
|
+
d = dict([(obj.name, uid) for uid, obj in self.group.items()])
|
|
814
|
+
uid = d.get(name)
|
|
815
|
+
return self.get_data_by_uuid(uid)
|
|
816
|
+
|
|
817
|
+
def tree_folder(self, dir, padding=" ", print_files=True, hidden_files=False, last=True):
|
|
818
|
+
"""print tree folder structure"""
|
|
819
|
+
if last is False:
|
|
820
|
+
print(padding[:-1] + '├─' + os.path.basename(os.path.abspath(dir)))
|
|
821
|
+
else:
|
|
822
|
+
print(padding[:-1] + '└─' + os.path.basename(os.path.abspath(dir)))
|
|
823
|
+
padding = padding + ' '
|
|
824
|
+
files = []
|
|
825
|
+
if print_files:
|
|
826
|
+
files = [x for x in sorted(os.listdir(dir)) if not x.startswith(".")]
|
|
827
|
+
else:
|
|
828
|
+
files = [x for x in sorted(os.listdir(dir)) if os.path.isdir(dir + os.sep + x)]
|
|
829
|
+
|
|
830
|
+
# metadata first
|
|
831
|
+
metafiles = [f for f in files if f.startswith("metadata")]
|
|
832
|
+
files = [x for x in files if x not in metafiles]
|
|
833
|
+
files = metafiles + sorted(files)
|
|
834
|
+
|
|
835
|
+
for count, file in enumerate(sorted(files)):
|
|
836
|
+
# print(padding + '|')
|
|
837
|
+
path = dir + os.sep + file
|
|
838
|
+
if os.path.isdir(path):
|
|
839
|
+
if count == (len(files) - 1):
|
|
840
|
+
self.tree_folder(path, padding + ' ', print_files, last=True)
|
|
841
|
+
else:
|
|
842
|
+
self.tree_folder(path, padding + '|', print_files, last=False)
|
|
843
|
+
else:
|
|
844
|
+
if count == (len(files) - 1):
|
|
845
|
+
print(padding + '└─' + file)
|
|
846
|
+
else:
|
|
847
|
+
print(padding + '├─' + file)
|
|
848
|
+
|
|
849
|
+
def dir(self):
|
|
850
|
+
"""returns a nested list of all child objects
|
|
851
|
+
|
|
852
|
+
:return: list of sdata.Data objects
|
|
853
|
+
"""
|
|
854
|
+
return [(x.name, x.dir()) for x in self.group.values()]
|
|
855
|
+
|
|
856
|
+
def to_dataframe(self):
|
|
857
|
+
"""Export the DataFrame with additional metadata and description.
|
|
858
|
+
|
|
859
|
+
Returns:
|
|
860
|
+
pandas.DataFrame: A copy of the DataFrame with added attributes.
|
|
861
|
+
"""
|
|
862
|
+
df = self.df.copy()
|
|
863
|
+
df.attrs["!sdata"] = {
|
|
864
|
+
"metadata": self.metadata.to_dict(),
|
|
865
|
+
"description": self.description
|
|
866
|
+
}
|
|
867
|
+
return df
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
def to_parquet(self, filepath=None, **kwargs):
|
|
871
|
+
"""
|
|
872
|
+
|
|
873
|
+
"""
|
|
874
|
+
engine = kwargs.get("engine", "pyarrow")
|
|
875
|
+
df = self.df.copy()
|
|
876
|
+
#df["!sdata_index"] = df.index
|
|
877
|
+
#df.reset_index(inplace=True)
|
|
878
|
+
df.attrs["!sdata"] = {"metadata": self.metadata.to_dict(),
|
|
879
|
+
"description": self.description}
|
|
880
|
+
if filepath is not None:
|
|
881
|
+
df.to_parquet(filepath, engine=engine)
|
|
882
|
+
else:
|
|
883
|
+
return df.to_parquet(engine=engine)
|
|
884
|
+
|
|
885
|
+
@classmethod
|
|
886
|
+
def from_parquet(cls, filepath):
|
|
887
|
+
"""load data from parquet file
|
|
888
|
+
|
|
889
|
+
:param filepath:
|
|
890
|
+
:return: Data object
|
|
891
|
+
"""
|
|
892
|
+
try:
|
|
893
|
+
if os.path.exists(filepath):
|
|
894
|
+
df = pd.read_parquet(filepath)
|
|
895
|
+
tt = cls(name=filepath)
|
|
896
|
+
tt.table = df
|
|
897
|
+
|
|
898
|
+
attrs = df.attrs.get("!sdata")
|
|
899
|
+
try:
|
|
900
|
+
tt.metadata = tt.metadata.from_dict(attrs["metadata"])
|
|
901
|
+
except:
|
|
902
|
+
logger.warning(f"ignore metadata {filepath}")
|
|
903
|
+
try:
|
|
904
|
+
tt.description = attrs.get("description")
|
|
905
|
+
except:
|
|
906
|
+
logger.warning(f"ignore description {filepath}")
|
|
907
|
+
if attrs is not None:
|
|
908
|
+
tt.df.attrs.pop("!sdata")
|
|
909
|
+
else:
|
|
910
|
+
raise Exception("parquet file '{}' not available".format(filepath))
|
|
911
|
+
return tt
|
|
912
|
+
except Exception as exp:
|
|
913
|
+
raise
|
|
914
|
+
|
|
915
|
+
def to_xlsx_byteio(self):
|
|
916
|
+
"""get xlsx as byteio
|
|
917
|
+
|
|
918
|
+
:return: BytesIO
|
|
919
|
+
"""
|
|
920
|
+
|
|
921
|
+
def adjust_col_width(sheetname, df, writer, width=40):
|
|
922
|
+
worksheet = writer.sheets[sheetname] # pull worksheet object
|
|
923
|
+
worksheet.set_column(0, 0, width)
|
|
924
|
+
for idx, col in enumerate(df): # loop through all columns
|
|
925
|
+
# series = df[col]
|
|
926
|
+
# max_len = max((
|
|
927
|
+
# series.astype(str, raise_on_error=False).map(len).max(), # len of largest item
|
|
928
|
+
# len(str(series.name)) # len of column name/header
|
|
929
|
+
# )) + 1 # adding a little extra space
|
|
930
|
+
worksheet.set_column(idx + 1, idx + 1, width)
|
|
931
|
+
|
|
932
|
+
output = BytesIO()
|
|
933
|
+
writer = pd.ExcelWriter(output, engine='xlsxwriter')
|
|
934
|
+
self.metadata.df.to_excel(writer, sheet_name='metadata')
|
|
935
|
+
adjust_col_width('metadata', self.metadata.df, writer)
|
|
936
|
+
|
|
937
|
+
self.df.to_excel(writer, sheet_name='table')
|
|
938
|
+
adjust_col_width('table', self.table, writer, width=15)
|
|
939
|
+
|
|
940
|
+
df_description = pd.DataFrame(self.description.splitlines())
|
|
941
|
+
df_description.to_excel(writer, sheet_name='description', index=False, header=None)
|
|
942
|
+
adjust_col_width('description', df_description, writer, width=200)
|
|
943
|
+
|
|
944
|
+
writer.close()
|
|
945
|
+
processed_data = output.getvalue()
|
|
946
|
+
return processed_data
|
|
947
|
+
|
|
948
|
+
def to_xlsx_base64(self):
|
|
949
|
+
"""get xlsx as byteio base64 encoded
|
|
950
|
+
|
|
951
|
+
:return: base64
|
|
952
|
+
"""
|
|
953
|
+
val = self.to_xlsx_byteio()
|
|
954
|
+
b64 = base64.b64encode(val)
|
|
955
|
+
return b64
|
|
956
|
+
|
|
957
|
+
def get_download_link(self):
|
|
958
|
+
"""Generates a link allowing the data in a given panda dataframe to be downloaded
|
|
959
|
+
in: dataframe
|
|
960
|
+
out: href string
|
|
961
|
+
"""
|
|
962
|
+
b64 = self.to_xlsx_base64()
|
|
963
|
+
return '<a href="data:application/octet-stream;base64,{1}" download="{0}.xlsx">Download {0}.xlsx file</a>'.format(
|
|
964
|
+
self.osname, b64.decode())
|
|
965
|
+
|
|
966
|
+
def to_xlsx(self, filepath=None):
|
|
967
|
+
"""export atrributes and data to excel
|
|
968
|
+
|
|
969
|
+
:param filepath:
|
|
970
|
+
:return:
|
|
971
|
+
"""
|
|
972
|
+
|
|
973
|
+
def adjust_col_width(sheetname, df, writer, width=40):
|
|
974
|
+
worksheet = writer.sheets[sheetname] # pull worksheet object
|
|
975
|
+
worksheet.set_column(0, 0, width)
|
|
976
|
+
for idx, col in enumerate(df): # loop through all columns
|
|
977
|
+
# series = df[col]
|
|
978
|
+
# max_len = max((
|
|
979
|
+
# series.astype(str, raise_on_error=False).map(len).max(), # len of largest item
|
|
980
|
+
# len(str(series.name)) # len of column name/header
|
|
981
|
+
# )) + 1 # adding a little extra space
|
|
982
|
+
worksheet.set_column(idx + 1, idx + 1, width)
|
|
983
|
+
|
|
984
|
+
with pd.ExcelWriter(filepath) as writer:
|
|
985
|
+
|
|
986
|
+
# metadata
|
|
987
|
+
# dfm = pd.DataFrame.from_dict(self.metadata, orient="index", columns=["value"])
|
|
988
|
+
dfm = self.metadata.to_dataframe()
|
|
989
|
+
|
|
990
|
+
# dfm = dfm.sort_index()
|
|
991
|
+
dfm.index.name = "key"
|
|
992
|
+
dfm.to_excel(writer, sheet_name='metadata', index=False)
|
|
993
|
+
adjust_col_width('metadata', dfm, writer)
|
|
994
|
+
|
|
995
|
+
# data
|
|
996
|
+
if self.table is not None:
|
|
997
|
+
if self._table.index.name is None:
|
|
998
|
+
self._table.index.name = "index"
|
|
999
|
+
self.table.to_excel(writer, sheet_name='table')
|
|
1000
|
+
adjust_col_width('table', self.table, writer, width=15)
|
|
1001
|
+
else:
|
|
1002
|
+
df = pd.DataFrame()
|
|
1003
|
+
df.index.name = "index"
|
|
1004
|
+
df.to_excel(writer, sheet_name='table')
|
|
1005
|
+
adjust_col_width('table', df, writer, width=15)
|
|
1006
|
+
|
|
1007
|
+
df_description = pd.DataFrame(self.description.splitlines())
|
|
1008
|
+
df_description.to_excel(writer, sheet_name='description', index=False, header=None)
|
|
1009
|
+
adjust_col_width('description', df_description, writer, width=200)
|
|
1010
|
+
|
|
1011
|
+
# # raw data
|
|
1012
|
+
# self.df_raw.index.name = "index"
|
|
1013
|
+
# self.df_raw.to_excel(writer, sheet_name='df_raw')
|
|
1014
|
+
# adjust_col_width('df_raw', self.df_raw, writer, width=15)
|
|
1015
|
+
|
|
1016
|
+
@classmethod
|
|
1017
|
+
def from_xlsx(cls, filepath):
|
|
1018
|
+
"""save table as xlsx
|
|
1019
|
+
|
|
1020
|
+
:param filepath:
|
|
1021
|
+
:return:
|
|
1022
|
+
"""
|
|
1023
|
+
try:
|
|
1024
|
+
if os.path.exists(filepath):
|
|
1025
|
+
tt = cls(name=filepath)
|
|
1026
|
+
|
|
1027
|
+
# read df
|
|
1028
|
+
try:
|
|
1029
|
+
tt.table = pd.read_excel(filepath, sheet_name="table", index_col=0)
|
|
1030
|
+
except:
|
|
1031
|
+
logger.info("no table data in '{}'".format(filepath))
|
|
1032
|
+
dfm = pd.read_excel(filepath, sheet_name="metadata")
|
|
1033
|
+
dfm.set_index(dfm.name.values, inplace=True)
|
|
1034
|
+
# dfm["value"] = dfm["value"].replace(np.nan, None)
|
|
1035
|
+
dfm["description"] = dfm["description"].replace(np.nan, '')
|
|
1036
|
+
dfm["label"] = dfm["label"].replace(np.nan, '')
|
|
1037
|
+
|
|
1038
|
+
# read metadata
|
|
1039
|
+
tt.metadata = tt.metadata.from_dataframe(dfm)
|
|
1040
|
+
for attr in tt.metadata.sdata_attributes.values():
|
|
1041
|
+
if attr.value == "nan":
|
|
1042
|
+
attr.value = ""
|
|
1043
|
+
|
|
1044
|
+
# read description
|
|
1045
|
+
try:
|
|
1046
|
+
dfd = pd.read_excel(filepath, sheet_name="description", header=None)
|
|
1047
|
+
tt.description = "\n".join(dfd[0])
|
|
1048
|
+
except:
|
|
1049
|
+
logger.info("no description in '{}'".format(filepath))
|
|
1050
|
+
return tt
|
|
1051
|
+
else:
|
|
1052
|
+
raise Exception("excel file '{}' not available".format(filepath))
|
|
1053
|
+
except Exception as exp:
|
|
1054
|
+
raise
|
|
1055
|
+
|
|
1056
|
+
def to_json(self, filepath=None):
|
|
1057
|
+
"""export Data in json format
|
|
1058
|
+
|
|
1059
|
+
:param filepath: export file path (default:None)
|
|
1060
|
+
:return: json str
|
|
1061
|
+
"""
|
|
1062
|
+
|
|
1063
|
+
if self.table is not None:
|
|
1064
|
+
json_table = self.table.to_dict()
|
|
1065
|
+
else:
|
|
1066
|
+
json_table = {}
|
|
1067
|
+
|
|
1068
|
+
j = {"metadata": self.metadata.to_dict(),
|
|
1069
|
+
"table": json_table,
|
|
1070
|
+
"description": self.description
|
|
1071
|
+
}
|
|
1072
|
+
if filepath:
|
|
1073
|
+
with open(filepath, "w") as fh:
|
|
1074
|
+
json.dump(j, fh)
|
|
1075
|
+
else:
|
|
1076
|
+
return json.dumps(j)
|
|
1077
|
+
|
|
1078
|
+
@classmethod
|
|
1079
|
+
def from_json(cls, s=None, filepath=None):
|
|
1080
|
+
"""create Data from json str or file
|
|
1081
|
+
|
|
1082
|
+
:param s: json str
|
|
1083
|
+
:param filepath:
|
|
1084
|
+
:return: sdata.Data
|
|
1085
|
+
"""
|
|
1086
|
+
data = cls(name="N.N.")
|
|
1087
|
+
if s is None and filepath is not None:
|
|
1088
|
+
with open(filepath, "r") as fh:
|
|
1089
|
+
d = json.load(fh)
|
|
1090
|
+
elif s is None and filepath is None:
|
|
1091
|
+
logger.error("data.from_json: no json data available")
|
|
1092
|
+
return
|
|
1093
|
+
elif s is not None and filepath is None:
|
|
1094
|
+
d = json.loads(s)
|
|
1095
|
+
else:
|
|
1096
|
+
logger.error("data.from_json: unexpected error")
|
|
1097
|
+
d = None
|
|
1098
|
+
|
|
1099
|
+
if d:
|
|
1100
|
+
if "metadata" in d.keys():
|
|
1101
|
+
data.metadata.update_from_dict(d["metadata"])
|
|
1102
|
+
else:
|
|
1103
|
+
logger.error("Data.from_json: table not available")
|
|
1104
|
+
|
|
1105
|
+
if "table" in d.keys():
|
|
1106
|
+
data.table = pd.DataFrame.from_dict(d["table"])
|
|
1107
|
+
# data.table = pd.read_json(json.dumps(d["table"]))
|
|
1108
|
+
# data.table = pd.read_json(d["table"])
|
|
1109
|
+
else:
|
|
1110
|
+
logger.error("Data.from_json: metadata not available")
|
|
1111
|
+
|
|
1112
|
+
if "description" in d.keys():
|
|
1113
|
+
data.description = d["description"]
|
|
1114
|
+
else:
|
|
1115
|
+
logger.error("Data.from_json: description not available")
|
|
1116
|
+
|
|
1117
|
+
return data
|
|
1118
|
+
|
|
1119
|
+
@classmethod
|
|
1120
|
+
def from_url(cls, url=None, stype=None):
|
|
1121
|
+
"""create Data from json str or file
|
|
1122
|
+
|
|
1123
|
+
:param url: url
|
|
1124
|
+
:param stype: "json" ("xlsx", "csv")
|
|
1125
|
+
:return: sdata.Data
|
|
1126
|
+
"""
|
|
1127
|
+
|
|
1128
|
+
supported_stypes = ["json"]
|
|
1129
|
+
|
|
1130
|
+
if stype not in supported_stypes:
|
|
1131
|
+
raise NotADirectoryError("stype '{}' is not supported".format(stype))
|
|
1132
|
+
return
|
|
1133
|
+
|
|
1134
|
+
# 'requests' ist optional; Fallback auf die Standardbibliothek (urllib).
|
|
1135
|
+
try:
|
|
1136
|
+
import requests
|
|
1137
|
+
raw = requests.get(url).text
|
|
1138
|
+
except ImportError:
|
|
1139
|
+
import urllib.request
|
|
1140
|
+
with urllib.request.urlopen(url) as _resp:
|
|
1141
|
+
raw = _resp.read().decode("utf-8", "replace")
|
|
1142
|
+
if stype=="json":
|
|
1143
|
+
data = cls.from_json(raw)
|
|
1144
|
+
return data
|
|
1145
|
+
|
|
1146
|
+
|
|
1147
|
+
def to_csv(self, filepath=None):
|
|
1148
|
+
"""export sdata.Data to csv
|
|
1149
|
+
|
|
1150
|
+
:param filepath:
|
|
1151
|
+
:return:
|
|
1152
|
+
"""
|
|
1153
|
+
|
|
1154
|
+
exportlines = []
|
|
1155
|
+
exportlines.append(self.metadata.to_csv_header(prefix="#;", sep=";", filepath=None))
|
|
1156
|
+
if self.df is not None:
|
|
1157
|
+
exportlines.append(self.df.to_csv(sep=";"))
|
|
1158
|
+
exportstr = "".join(exportlines)
|
|
1159
|
+
|
|
1160
|
+
if filepath is None:
|
|
1161
|
+
return exportstr
|
|
1162
|
+
else:
|
|
1163
|
+
with open(filepath, "w") as fh:
|
|
1164
|
+
fh.write(exportstr)
|
|
1165
|
+
|
|
1166
|
+
@classmethod
|
|
1167
|
+
def from_csv(cls, filepath, sep=";"):
|
|
1168
|
+
"""import sdata.Data from csv
|
|
1169
|
+
|
|
1170
|
+
:param filepath:
|
|
1171
|
+
:param sep: separator (default=";")
|
|
1172
|
+
:return: sdata.Data
|
|
1173
|
+
"""
|
|
1174
|
+
data = cls()
|
|
1175
|
+
df = pd.read_csv(filepath, sep=";", comment="#", index_col=0)
|
|
1176
|
+
|
|
1177
|
+
|
|
1178
|
+
#attr
|
|
1179
|
+
sio = open(filepath, "r")
|
|
1180
|
+
sio.seek(0)
|
|
1181
|
+
attritute_list = []
|
|
1182
|
+
for line in sio:
|
|
1183
|
+
if line.startswith("#;"):
|
|
1184
|
+
line = line.rstrip("\n")
|
|
1185
|
+
line = line.split(sep)
|
|
1186
|
+
attritute_list.append(line[1:8])
|
|
1187
|
+
data.metadata = Metadata.from_list(attritute_list)
|
|
1188
|
+
data.table = df
|
|
1189
|
+
return data
|
|
1190
|
+
|
|
1191
|
+
|
|
1192
|
+
@classmethod
|
|
1193
|
+
def from_csv_string(cls, s=None, sep=";"):
|
|
1194
|
+
"""import sdata.Data from csv
|
|
1195
|
+
|
|
1196
|
+
:param s: csv str
|
|
1197
|
+
:param filepath:
|
|
1198
|
+
:param sep: separator (default=";")
|
|
1199
|
+
:return: sdata.Data
|
|
1200
|
+
"""
|
|
1201
|
+
if s is None and filepath is None:
|
|
1202
|
+
logging.error("specify string if filepath")
|
|
1203
|
+
return
|
|
1204
|
+
data = cls()
|
|
1205
|
+
df = None
|
|
1206
|
+
try:
|
|
1207
|
+
sio = StringIO(s)
|
|
1208
|
+
df = pd.read_csv(sio, sep=";", comment="#")
|
|
1209
|
+
sio.seek(0)
|
|
1210
|
+
except Exception as exp:
|
|
1211
|
+
logging.error(f"{exp}")
|
|
1212
|
+
raise
|
|
1213
|
+
|
|
1214
|
+
attritute_list = []
|
|
1215
|
+
for line in sio:
|
|
1216
|
+
if line.startswith("#;"):
|
|
1217
|
+
line = line.rstrip("\n")
|
|
1218
|
+
line = line.split(sep)
|
|
1219
|
+
attritute_list.append(line[1:8])
|
|
1220
|
+
data.metadata = Metadata.from_list(attritute_list)
|
|
1221
|
+
data.table = df
|
|
1222
|
+
return data
|
|
1223
|
+
|
|
1224
|
+
def to_hdf5(self, filepath, **kwargs):
|
|
1225
|
+
"""export sdata.Data to hdf5
|
|
1226
|
+
|
|
1227
|
+
:param filepath:
|
|
1228
|
+
:param complib: default='zlib' ['zlib', 'lzo', 'bzip2', 'blosc', 'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy', 'blosc:zlib', 'blosc:zstd']
|
|
1229
|
+
:param complevel: default=9 [0-9]
|
|
1230
|
+
|
|
1231
|
+
:return:
|
|
1232
|
+
"""
|
|
1233
|
+
if not isinstance(self.df, pd.DataFrame):
|
|
1234
|
+
df = pd.DataFrame()
|
|
1235
|
+
else:
|
|
1236
|
+
df = self.df
|
|
1237
|
+
kwargs["mode"] = "w"
|
|
1238
|
+
if kwargs.get("complib") is None:
|
|
1239
|
+
kwargs["complib"] = "zlib"
|
|
1240
|
+
|
|
1241
|
+
if kwargs.get("complevel") is None:
|
|
1242
|
+
kwargs["complevel"] = 9
|
|
1243
|
+
|
|
1244
|
+
with pd.HDFStore(filepath, **kwargs) as hdf:
|
|
1245
|
+
hdf.put('metadata'.format(self.uuid), self.metadata.df, format='fixed', data_columns=True)
|
|
1246
|
+
hdf.put('table'.format(self.uuid), df, format='fixed', data_columns=True)
|
|
1247
|
+
hdf.put('description'.format(self.uuid), self.description_to_df(), format='fixed', data_columns=True)
|
|
1248
|
+
|
|
1249
|
+
def to_sqlite(self, filepath, **kwargs):
|
|
1250
|
+
"""export sdata.Data to sqlite
|
|
1251
|
+
|
|
1252
|
+
:param filepath:
|
|
1253
|
+
:param kwargs:
|
|
1254
|
+
:return:
|
|
1255
|
+
"""
|
|
1256
|
+
|
|
1257
|
+
if not isinstance(self.df, pd.DataFrame):
|
|
1258
|
+
df = pd.DataFrame()
|
|
1259
|
+
else:
|
|
1260
|
+
df = self.df
|
|
1261
|
+
df.columns = df.columns.astype(str)
|
|
1262
|
+
|
|
1263
|
+
with SqliteDict(filepath, autocommit=True) as sqdict:
|
|
1264
|
+
sqdict['metadata'] = self.metadata.df.to_parquet()
|
|
1265
|
+
sqdict['table'] = df.to_parquet()
|
|
1266
|
+
|
|
1267
|
+
description = self.description_to_df().copy()
|
|
1268
|
+
description.columns = description.columns.astype(str)
|
|
1269
|
+
sqdict['description'] = description.to_parquet()
|
|
1270
|
+
|
|
1271
|
+
@classmethod
|
|
1272
|
+
def from_sqlite(cls, filepath, **kwargs):
|
|
1273
|
+
"""import sdata.Data from sqlite
|
|
1274
|
+
|
|
1275
|
+
:param filepath:
|
|
1276
|
+
:param kwargs:
|
|
1277
|
+
:return: sdata.Data
|
|
1278
|
+
"""
|
|
1279
|
+
|
|
1280
|
+
with SqliteDict(filepath) as sqdict:
|
|
1281
|
+
metadata = sdata.Metadata.from_dataframe(pd.read_parquet(BytesIO(sqdict['metadata'])))
|
|
1282
|
+
df_table = pd.read_parquet(BytesIO(sqdict['table']))
|
|
1283
|
+
df_description = pd.read_parquet(BytesIO(sqdict['description']))
|
|
1284
|
+
data = Data(metadata=metadata, table=df_table)
|
|
1285
|
+
data.description_from_df(df_description)
|
|
1286
|
+
return data
|
|
1287
|
+
|
|
1288
|
+
@classmethod
|
|
1289
|
+
def metadata_from_hdf5(cls, filepath, **kwargs):
|
|
1290
|
+
"""import sdata.Data.Metadata from hdf5
|
|
1291
|
+
|
|
1292
|
+
:param filepath:
|
|
1293
|
+
:return: sdata.Data
|
|
1294
|
+
"""
|
|
1295
|
+
if not os.path.exists:
|
|
1296
|
+
logger.error("hdf5 file '{}' not available".format(filepath))
|
|
1297
|
+
return
|
|
1298
|
+
|
|
1299
|
+
with pd.HDFStore(filepath, mode="r+") as hdf:
|
|
1300
|
+
metadata_path = "/metadata".format(uuid)
|
|
1301
|
+
df_metadata = hdf.get(metadata_path)
|
|
1302
|
+
metadata = Metadata.from_dataframe(df_metadata)
|
|
1303
|
+
return metadata
|
|
1304
|
+
|
|
1305
|
+
@classmethod
|
|
1306
|
+
def from_hdf5(cls, filepath, **kwargs):
|
|
1307
|
+
"""import sdata.Data from hdf5
|
|
1308
|
+
|
|
1309
|
+
:param filepath:
|
|
1310
|
+
:return: sdata.Data
|
|
1311
|
+
"""
|
|
1312
|
+
if not os.path.exists:
|
|
1313
|
+
logger.error("hdf5 file '{}' not available".format(filepath))
|
|
1314
|
+
return
|
|
1315
|
+
|
|
1316
|
+
with pd.HDFStore(filepath, mode="r+") as hdf:
|
|
1317
|
+
metadata_path = "/metadata".format(uuid)
|
|
1318
|
+
table_path = "/table".format(uuid)
|
|
1319
|
+
description_path = "/description".format(uuid)
|
|
1320
|
+
df_metadata = hdf.get(metadata_path)
|
|
1321
|
+
df_table = hdf.get(table_path)
|
|
1322
|
+
df_description = hdf.get(description_path)
|
|
1323
|
+
metadata = Metadata.from_dataframe(df_metadata)
|
|
1324
|
+
# logger.debug("hdf {}".format(metadata.get("!sdata_uuid").value))
|
|
1325
|
+
data = Data(metadata=metadata, table=df_table)
|
|
1326
|
+
data.description_from_df(df_description)
|
|
1327
|
+
|
|
1328
|
+
return data
|
|
1329
|
+
|
|
1330
|
+
def to_html(self, filepath, xlsx=True, style=None):
|
|
1331
|
+
"""export Data to html
|
|
1332
|
+
|
|
1333
|
+
:param filepath:
|
|
1334
|
+
:param xlsx:
|
|
1335
|
+
:param style:
|
|
1336
|
+
:return:
|
|
1337
|
+
"""
|
|
1338
|
+
|
|
1339
|
+
table_values = self.df.head()
|
|
1340
|
+
table_headers = self.df.columns
|
|
1341
|
+
|
|
1342
|
+
table_description_values = self.df.describe()
|
|
1343
|
+
table_description_headers = self.df.describe().columns
|
|
1344
|
+
|
|
1345
|
+
metadata_values = self.metadata.df.head().values
|
|
1346
|
+
metadata_headers = self.metadata.df.columns
|
|
1347
|
+
|
|
1348
|
+
if xlsx is True:
|
|
1349
|
+
xlsx_tag = self.get_download_link()
|
|
1350
|
+
else:
|
|
1351
|
+
xlsx_tag = ""
|
|
1352
|
+
|
|
1353
|
+
from tabulate import tabulate # optional; nur für HTML-Repr benötigt
|
|
1354
|
+
param = {"title":"{0} [{1}]".format(self.osname,
|
|
1355
|
+
self.uuid),
|
|
1356
|
+
"description":self.description,
|
|
1357
|
+
"metadata": tabulate(metadata_values, metadata_headers, tablefmt="html"),
|
|
1358
|
+
"table": tabulate(table_values, table_headers, tablefmt="html"),
|
|
1359
|
+
"table_description": tabulate(table_description_values, table_description_headers, tablefmt="html"),
|
|
1360
|
+
"xlsx_tag":xlsx_tag,
|
|
1361
|
+
"sdata":"created with sdata v{}.".format(__version__),
|
|
1362
|
+
"now":"{}".format(now_utc_str()),
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
tmpl = """<!DOCTYPE html>
|
|
1366
|
+
<html lang="de">
|
|
1367
|
+
<head>
|
|
1368
|
+
<meta charset="utf-8">
|
|
1369
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
1370
|
+
<title>{title}</title>
|
|
1371
|
+
<style>
|
|
1372
|
+
h1 {{
|
|
1373
|
+
background-color: #00FFFF77;
|
|
1374
|
+
color: black;
|
|
1375
|
+
}}
|
|
1376
|
+
h2 {{
|
|
1377
|
+
background-color: #00FFFF44;
|
|
1378
|
+
color: black;
|
|
1379
|
+
}}
|
|
1380
|
+
h3 {{
|
|
1381
|
+
background-color: #00FFFF11;
|
|
1382
|
+
color: black;
|
|
1383
|
+
}}
|
|
1384
|
+
p {{
|
|
1385
|
+
color: black;
|
|
1386
|
+
}}
|
|
1387
|
+
table, th, td, caption {{
|
|
1388
|
+
border: 1px solid #a0a0a0;
|
|
1389
|
+
}}
|
|
1390
|
+
|
|
1391
|
+
table {{
|
|
1392
|
+
border-collapse: collapse;
|
|
1393
|
+
border-spacing: 1em;
|
|
1394
|
+
border-width: thin 0 0 thin;
|
|
1395
|
+
margin: 0 0 1em;
|
|
1396
|
+
table-layout: auto;
|
|
1397
|
+
max-width: 100%;
|
|
1398
|
+
text-align: right;
|
|
1399
|
+
}}
|
|
1400
|
+
th, td {{
|
|
1401
|
+
font-weight: normal;
|
|
1402
|
+
text-align: left;
|
|
1403
|
+
border-spacing: 1em;
|
|
1404
|
+
padding: .1em .3em;
|
|
1405
|
+
}}
|
|
1406
|
+
th, caption {{
|
|
1407
|
+
background-color: #f1f3f4;
|
|
1408
|
+
font-weight: 700;
|
|
1409
|
+
}}
|
|
1410
|
+
</style>
|
|
1411
|
+
</head>
|
|
1412
|
+
<body>
|
|
1413
|
+
<h1>{title}</h1>
|
|
1414
|
+
<h2>Download</h2>
|
|
1415
|
+
<p">{xlsx_tag}</p>
|
|
1416
|
+
<h2>Description</h2>
|
|
1417
|
+
<p>{description}</p>
|
|
1418
|
+
<h2>Metadata</h2>
|
|
1419
|
+
{metadata}
|
|
1420
|
+
<h2>Table</h2>
|
|
1421
|
+
{table}
|
|
1422
|
+
<h3>Table Description</h3>
|
|
1423
|
+
{table_description}
|
|
1424
|
+
<p>{sdata}</p>
|
|
1425
|
+
<p>{now}</p>
|
|
1426
|
+
</body>
|
|
1427
|
+
</html>""".format(**param)
|
|
1428
|
+
try:
|
|
1429
|
+
with open(filepath, "w") as fh:
|
|
1430
|
+
fh.write(tmpl)
|
|
1431
|
+
except Exception as exp:
|
|
1432
|
+
raise
|
|
1433
|
+
|
|
1434
|
+
def copy(self, **kwargs):
|
|
1435
|
+
"""create a copy of the Data object
|
|
1436
|
+
|
|
1437
|
+
.. code-block:: python
|
|
1438
|
+
|
|
1439
|
+
data = sdata.Data(name="data", uuid="38b26864e7794f5182d38459bab85842", description="this is remarkable")
|
|
1440
|
+
datac = data.copy()
|
|
1441
|
+
print("data {0.uuid}".format(data))
|
|
1442
|
+
print("datac {0.uuid}".format(datac))
|
|
1443
|
+
print("datac.metadata['!sdata_parent'] {0.value}".format(datac.metadata["sdata_parent"]))
|
|
1444
|
+
|
|
1445
|
+
.. code-block::
|
|
1446
|
+
|
|
1447
|
+
data 38b26864e7794f5182d38459bab85842
|
|
1448
|
+
datac 2c4eb15900af435d8cd9c8573ca777e2
|
|
1449
|
+
datac.metadata['!sdata_parent'] 38b26864e7794f5182d38459bab85842
|
|
1450
|
+
|
|
1451
|
+
:return: Data
|
|
1452
|
+
"""
|
|
1453
|
+
data = copy.deepcopy(self)
|
|
1454
|
+
data.metadata.add(self.SDATA_PARENT, self.suuid)
|
|
1455
|
+
data.metadata.add(self.SDATA_UUID, self.gen_uuid())
|
|
1456
|
+
data.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str")
|
|
1457
|
+
if "uuid" in kwargs:
|
|
1458
|
+
data.uuid = kwargs.get("uuid")
|
|
1459
|
+
if "name" in kwargs:
|
|
1460
|
+
data.name = kwargs.get("name")
|
|
1461
|
+
logger.debug(f"make copy of {self.uuid} -> {data.uuid}")
|
|
1462
|
+
data.gen_suuid()
|
|
1463
|
+
|
|
1464
|
+
return data
|
|
1465
|
+
|
|
1466
|
+
def gen_uuid(self):
|
|
1467
|
+
"""generate new uuid string
|
|
1468
|
+
|
|
1469
|
+
:return: str, e.g. '5fa04a3738e4431dbc34eccea5e795c4'
|
|
1470
|
+
"""
|
|
1471
|
+
return uuid.uuid4().hex
|
|
1472
|
+
|
|
1473
|
+
def refactor(self, fix_columns=True, add_table_metadata=True):
|
|
1474
|
+
"""helper function
|
|
1475
|
+
|
|
1476
|
+
* to cleanup dataframe column name
|
|
1477
|
+
* to define Attributes for all dataframe columns
|
|
1478
|
+
"""
|
|
1479
|
+
if isinstance(self.table, pd.DataFrame):
|
|
1480
|
+
mapper = {}
|
|
1481
|
+
for old_colname in self.table.columns:
|
|
1482
|
+
name, unit = extract_name_unit(old_colname)
|
|
1483
|
+
if fix_columns:
|
|
1484
|
+
mapper[old_colname] = name
|
|
1485
|
+
if add_table_metadata:
|
|
1486
|
+
old_attr = self.metadata.get(old_colname)
|
|
1487
|
+
if old_attr:
|
|
1488
|
+
logger.info("skip: {}".format(old_attr))
|
|
1489
|
+
self.metadata.relabel(old_colname, name)
|
|
1490
|
+
else:
|
|
1491
|
+
self.metadata.add(name=name, description=old_colname, unit=unit, dtype="float")
|
|
1492
|
+
self.table.rename(columns=mapper, inplace=True)
|
|
1493
|
+
|
|
1494
|
+
SDATACLS = {"Data": Data,
|
|
1495
|
+
}
|
|
1496
|
+
#
|
|
1497
|
+
# __all__ = ["Data", Data]
|
|
1498
|
+
|
|
1499
|
+
|
|
1500
|
+
def print_classes():
|
|
1501
|
+
for name, obj in inspect.getmembers(sys.modules[__name__]):
|
|
1502
|
+
if inspect.isclass(obj):
|
|
1503
|
+
print(obj)
|