sdata 0.22.7__tar.gz → 0.25.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. {sdata-0.22.7/sdata.egg-info → sdata-0.25.0}/PKG-INFO +10 -4
  2. {sdata-0.22.7 → sdata-0.25.0}/sdata/__init__.py +3 -3
  3. {sdata-0.22.7 → sdata-0.25.0}/sdata/data.py +165 -24
  4. {sdata-0.22.7 → sdata-0.25.0}/sdata/experiments/__init__.py +43 -7
  5. {sdata-0.22.7 → sdata-0.25.0}/sdata/image.py +112 -8
  6. sdata-0.25.0/sdata/iolib/hashtree.py +264 -0
  7. {sdata-0.22.7 → sdata-0.25.0}/sdata/iolib/vault.py +17 -4
  8. {sdata-0.22.7 → sdata-0.25.0}/sdata/metadata.py +70 -8
  9. sdata-0.25.0/sdata/node.py +145 -0
  10. sdata-0.25.0/sdata/parameter.py +411 -0
  11. sdata-0.25.0/sdata/parameters_deprecated.py +394 -0
  12. {sdata-0.22.7 → sdata-0.25.0}/sdata/suuid.py +30 -3
  13. {sdata-0.22.7 → sdata-0.25.0/sdata.egg-info}/PKG-INFO +10 -4
  14. {sdata-0.22.7 → sdata-0.25.0}/sdata.egg-info/SOURCES.txt +6 -0
  15. sdata-0.25.0/tests/iolib/test_hasttree.py +67 -0
  16. {sdata-0.22.7 → sdata-0.25.0}/tests/test_data.py +22 -0
  17. sdata-0.25.0/tests/test_parameter.py +118 -0
  18. {sdata-0.22.7 → sdata-0.25.0}/LICENSE-MIT +0 -0
  19. {sdata-0.22.7 → sdata-0.25.0}/MANIFEST.in +0 -0
  20. {sdata-0.22.7 → sdata-0.25.0}/README.md +0 -0
  21. {sdata-0.22.7 → sdata-0.25.0}/sdata/blob.py +0 -0
  22. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/__init__.py +0 -0
  23. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/attrdict/__init__.py +0 -0
  24. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/attrdict/default.py +0 -0
  25. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/attrdict/dictionary.py +0 -0
  26. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/attrdict/mapping.py +0 -0
  27. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/attrdict/merge.py +0 -0
  28. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/attrdict/mixins.py +0 -0
  29. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/__init__.py +0 -0
  30. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/_common.py +0 -0
  31. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/_dump.py +0 -0
  32. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/_exceptions.py +0 -0
  33. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/_exif.py +0 -0
  34. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/_insert.py +0 -0
  35. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/_load.py +0 -0
  36. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/_remove.py +0 -0
  37. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/_transplant.py +0 -0
  38. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/_webp.py +0 -0
  39. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/piexif/helper.py +0 -0
  40. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/ranger/__init__.py +0 -0
  41. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/ranger/collections/__init__.py +0 -0
  42. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/ranger/collections/rangebucketmap.py +0 -0
  43. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/ranger/collections/rangemap.py +0 -0
  44. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/ranger/collections/rangeset.py +0 -0
  45. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/ranger/range/__init__.py +0 -0
  46. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/ranger/range/cut.py +0 -0
  47. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/ranger/range/range.py +0 -0
  48. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/semver.py +0 -0
  49. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/simple_graph_db/LICENSE +0 -0
  50. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/simple_graph_db/__init__.py +0 -0
  51. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/simple_graph_db/database.py +0 -0
  52. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/simple_graph_db/schema.sql +0 -0
  53. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/sobol_seq.py +0 -0
  54. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/sortedcontainers/LICENSE +0 -0
  55. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/sortedcontainers/__init__.py +0 -0
  56. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/sortedcontainers/sorteddict.py +0 -0
  57. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/sortedcontainers/sortedlist.py +0 -0
  58. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/sortedcontainers/sortedset.py +0 -0
  59. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/sqlitedict.py +0 -0
  60. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/timeflake/__init__.py +0 -0
  61. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/timeflake/flake.py +0 -0
  62. {sdata-0.22.7 → sdata-0.25.0}/sdata/contrib/timeflake/utils.py +0 -0
  63. {sdata-0.22.7 → sdata-0.25.0}/sdata/doe.py +0 -0
  64. {sdata-0.22.7 → sdata-0.25.0}/sdata/experiments/ks2.py +0 -0
  65. {sdata-0.22.7 → sdata-0.25.0}/sdata/experiments/lapshear.py +0 -0
  66. {sdata-0.22.7 → sdata-0.25.0}/sdata/experiments/material.py +0 -0
  67. {sdata-0.22.7 → sdata-0.25.0}/sdata/iolib/__init__.py +0 -0
  68. {sdata-0.22.7 → sdata-0.25.0}/sdata/iolib/db.py +0 -0
  69. {sdata-0.22.7 → sdata-0.25.0}/sdata/iolib/hdf.py +0 -0
  70. {sdata-0.22.7 → sdata-0.25.0}/sdata/iolib/isomme.py +0 -0
  71. {sdata-0.22.7 → sdata-0.25.0}/sdata/iolib/pgp.py +0 -0
  72. {sdata-0.22.7 → sdata-0.25.0}/sdata/iolib/pud.py +0 -0
  73. {sdata-0.22.7 → sdata-0.25.0}/sdata/iolib/simple_signature.py +0 -0
  74. {sdata-0.22.7 → sdata-0.25.0}/sdata/timestamp.py +0 -0
  75. {sdata-0.22.7 → sdata-0.25.0}/sdata/tools.py +0 -0
  76. {sdata-0.22.7 → sdata-0.25.0}/sdata/workbook.py +0 -0
  77. {sdata-0.22.7 → sdata-0.25.0}/sdata.egg-info/dependency_links.txt +0 -0
  78. {sdata-0.22.7 → sdata-0.25.0}/sdata.egg-info/requires.txt +0 -0
  79. {sdata-0.22.7 → sdata-0.25.0}/sdata.egg-info/top_level.txt +0 -0
  80. {sdata-0.22.7 → sdata-0.25.0}/setup.cfg +0 -0
  81. {sdata-0.22.7 → sdata-0.25.0}/setup.py +0 -0
  82. {sdata-0.22.7 → sdata-0.25.0}/tests/__init__.py +0 -0
  83. {sdata-0.22.7 → sdata-0.25.0}/tests/iolib/__init__.py +0 -0
  84. {sdata-0.22.7 → sdata-0.25.0}/tests/iolib/pud/__init__.py +0 -0
  85. {sdata-0.22.7 → sdata-0.25.0}/tests/iolib/pud/test_pud.py +0 -0
  86. {sdata-0.22.7 → sdata-0.25.0}/tests/iolib/test_hdf.py +0 -0
  87. {sdata-0.22.7 → sdata-0.25.0}/tests/test_attribute.py +0 -0
  88. {sdata-0.22.7 → sdata-0.25.0}/tests/test_blob.py +0 -0
  89. {sdata-0.22.7 → sdata-0.25.0}/tests/test_image.py +0 -0
  90. {sdata-0.22.7 → sdata-0.25.0}/tests/test_io.py +0 -0
  91. {sdata-0.22.7 → sdata-0.25.0}/tests/test_ks2.py +0 -0
  92. {sdata-0.22.7 → sdata-0.25.0}/tests/test_metadata.py +0 -0
  93. {sdata-0.22.7 → sdata-0.25.0}/tests/test_sqlite.py +0 -0
  94. {sdata-0.22.7 → sdata-0.25.0}/tests/test_suuid.py +0 -0
  95. {sdata-0.22.7 → sdata-0.25.0}/tests/test_table.py +0 -0
  96. {sdata-0.22.7 → sdata-0.25.0}/tests/test_testprogram.py +0 -0
  97. {sdata-0.22.7 → sdata-0.25.0}/tests/test_timeflake.py +0 -0
  98. {sdata-0.22.7 → sdata-0.25.0}/tests/test_timestamp.py +0 -0
  99. {sdata-0.22.7 → sdata-0.25.0}/tests/test_tools.py +0 -0
  100. {sdata-0.22.7 → sdata-0.25.0}/tests/test_vault.py +0 -0
  101. {sdata-0.22.7 → sdata-0.25.0}/tests/test_workbook.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: sdata
3
- Version: 0.22.7
3
+ Version: 0.25.0
4
4
  Summary: a structured data format
5
5
  Home-page: https://github.com/lepy/sdata
6
6
  Author: Lepy
@@ -9,7 +9,6 @@ Maintainer: Lepy
9
9
  Maintainer-email: lepy@tuta.io
10
10
  License: MIT/Apache-2.0
11
11
  Keywords: open data
12
- Platform: UNKNOWN
13
12
  Classifier: Development Status :: 3 - Alpha
14
13
  Classifier: Intended Audience :: Developers
15
14
  Classifier: License :: OSI Approved :: MIT License
@@ -26,6 +25,15 @@ Classifier: Programming Language :: Python :: Implementation :: CPython
26
25
  Classifier: Programming Language :: Python :: Implementation :: PyPy
27
26
  Description-Content-Type: text/markdown
28
27
  License-File: LICENSE-MIT
28
+ Requires-Dist: numpy
29
+ Requires-Dist: pandas
30
+ Requires-Dist: tabulate
31
+ Requires-Dist: xlrd
32
+ Requires-Dist: openpyxl
33
+ Requires-Dist: xlsxwriter
34
+ Requires-Dist: pytz
35
+ Requires-Dist: requests
36
+ Requires-Dist: Pillow
29
37
 
30
38
 
31
39
  [![PyPI version](https://badge.fury.io/py/sdata.svg)](https://badge.fury.io/py/sdata)
@@ -128,5 +136,3 @@ Try to paste some Excel-Data in the forms ...
128
136
  * [sdata](https://doi.org/10.5281/zenodo.4311396)
129
137
  * Ingolf Lepenies. (2020, December 8). sdata - a structured data format (Version 0.8.4). Zenodo. http://doi.org/10.5281/zenodo.4311397
130
138
 
131
-
132
-
@@ -1,7 +1,7 @@
1
1
  # -*-coding: utf-8-*-
2
2
  from __future__ import division
3
3
 
4
- __version__ = '0.22.7'
4
+ __version__ = '0.25.0'
5
5
  __revision__ = None
6
6
  __version_info__ = tuple([int(num) for num in __version__.split('.')])
7
7
 
@@ -14,8 +14,8 @@ import uuid
14
14
  import logging
15
15
 
16
16
  # fix "ValueError: unsupported pickle protocol: 5"
17
- import pickle
18
- pickle.HIGHEST_PROTOCOL = 4
17
+ #import pickle
18
+ #pickle.HIGHEST_PROTOCOL = 4
19
19
  import pandas
20
20
 
21
21
  from sdata.metadata import Metadata, Attribute
@@ -24,6 +24,7 @@ import json
24
24
  import hashlib
25
25
  import base64
26
26
  import requests
27
+ import re
27
28
  from tabulate import tabulate
28
29
  from sdata.contrib.sqlitedict import SqliteDict
29
30
 
@@ -46,12 +47,31 @@ except:
46
47
  def uuid_from_str(name):
47
48
  return uuid.uuid3(uuid.NAMESPACE_DNS, name)
48
49
 
50
+ def nameunit_from_colname(column_name):
51
+ """
52
+
53
+ """
54
+ #pattern = r"^(.*?)\s*\[(.+)\]$"
55
+ pattern = r"^(.*?)\s*\[([^\]]+)\]$"
56
+
57
+ # Anwenden des Regex
58
+ match = re.match(pattern, column_name)
59
+
60
+ if match:
61
+ name = match.group(1) # Der Name der Spalte
62
+ unit = match.group(2) # Die Einheit
63
+ else:
64
+ name = column_name
65
+ unit = "-"
66
+ return name.strip(), unit.strip()
67
+
49
68
  class Data(object):
50
69
  """Base sdata object"""
51
70
  ATTR_NAMES = []
52
71
 
53
72
  SDATA_VERSION = "!sdata_version"
54
73
  SDATA_NAME = "!sdata_name"
74
+ SDATA_SNAME = "!sdata_sname"
55
75
  SDATA_UUID = "!sdata_uuid"
56
76
  SDATA_SUUID = "!sdata_suuid"
57
77
  SDATA_CTIME = "!sdata_ctime"
@@ -60,6 +80,7 @@ class Data(object):
60
80
  SDATA_CLASS = "!sdata_class"
61
81
  SDATA_PROJECT = "!sdata_project"
62
82
  SDATA_URL = "!sdata_url"
83
+ SDATA_COLUMN = "!sdata_column"
63
84
 
64
85
  SDATA_ATTRIBUTES = [SDATA_VERSION, SDATA_NAME, SDATA_UUID, SDATA_SUUID, SDATA_CLASS, SDATA_PARENT, SDATA_PROJECT,
65
86
  SDATA_CTIME, SDATA_MTIME, SDATA_URL]
@@ -95,12 +116,15 @@ class Data(object):
95
116
  # set default sdata attributes
96
117
  self.metadata.add(self.SDATA_VERSION, __version__, dtype="str", description="sdata package version")
97
118
  self.metadata.add(self.SDATA_NAME, "N.N.", dtype="str", description="name of the data object")
119
+ self.metadata.add(self.SDATA_SNAME, "", dtype="str", description="sname of the data object")
98
120
  self.metadata.add(self.SDATA_UUID, "", dtype="str", description="Universally Unique Identifier")
99
121
  self.metadata.add(self.SDATA_SUUID, "", dtype="str", description="Super Universally Unique Identifier")
100
122
  self.metadata.add(self.SDATA_PARENT, "", dtype="str", description="uuid of the parent sdata object")
101
123
  self.metadata.add(self.SDATA_CLASS, self.__class__.__name__, dtype="str", description="sdata class")
102
124
  self.metadata.add(self.SDATA_CTIME, now_utc_str(), dtype="str", description="creation date")
103
125
  self.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str", description="modification date")
126
+ self.metadata.add(self.SDATA_URL, "", dtype="str", description="url of the data object")
127
+ self.metadata.add(self.SDATA_PROJECT, "", dtype="str", description="project name")
104
128
 
105
129
  metadata = kwargs.get("metadata")
106
130
  if metadata is not None:
@@ -127,7 +151,8 @@ class Data(object):
127
151
  self.table = kwargs.get("table", None)
128
152
  self._description = ""
129
153
  self.description = kwargs.get("description", "")
130
- self.project = kwargs.get("project", "")
154
+ self.project = kwargs.get("project", "N.N.")
155
+ self.metadata.add(self.SDATA_PROJECT, self.project, dtype="str", description="project name")
131
156
 
132
157
  if (kwargs.get("uuid")=="" or kwargs.get("uuid") is not None) and not self.metadata.get(self.SDATA_UUID).value and kwargs.get("uuid")!="hash":
133
158
  # logger.info("uuid in kwargs")
@@ -151,10 +176,23 @@ class Data(object):
151
176
  # logger.info("uuid new")
152
177
  self._set_uuid(uuid.uuid4())
153
178
 
154
- self.gen_suuid()
179
+ if "suuid" not in kwargs:
180
+ self.gen_suuid()
181
+ else:
182
+ suuid = sdata.SUUID.from_suuid_str(kwargs.get("suuid"))
183
+ self.suuid = kwargs.get("suuid")
184
+ self.sname = suuid.sname
185
+
186
+ if "sname" in kwargs:
187
+ self.sname = kwargs.get("sname")
188
+
189
+
190
+ if "parent" in kwargs:
191
+ self.metadata.add(self.SDATA_PARENT, kwargs.get("parent"))
155
192
 
156
193
  def gen_suuid(self):
157
194
  self.suuid = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).idstr
195
+ self.sname = sdata.SUUID(self.__class__.__name__, self.name, huuid=self.uuid).sname
158
196
 
159
197
  def gen_uuid_from_state(self):
160
198
  """generate the same uuid for the same data
@@ -177,7 +215,7 @@ class Data(object):
177
215
  def __eq__(self, other):
178
216
  """compare Data checksum
179
217
 
180
- :param other: sdata.Data objecet
218
+ :param other: sdata.Data object
181
219
  :return: True or False
182
220
  """
183
221
  if not isinstance(other, self.__class__):
@@ -198,6 +236,10 @@ class Data(object):
198
236
  self._set_uuid(suuid.huuid)
199
237
  return self
200
238
 
239
+ @property
240
+ def m(self):
241
+ return self.metadata
242
+
201
243
  def update_mtime(self):
202
244
  """update modification time
203
245
 
@@ -223,6 +265,13 @@ class Data(object):
223
265
  s.update(tablestr)
224
266
  return s.hexdigest()
225
267
 
268
+ @property
269
+ def sha3_256_metadata(self):
270
+ s = hashlib.sha3_256()
271
+ metadatastr = self.metadata.to_json().encode(errors="replace")
272
+ s.update(metadatastr)
273
+ return s.hexdigest()
274
+
226
275
  @property
227
276
  def sha3_256(self):
228
277
  """Return a SHA3 hash of the sData object with a hashbit length of 32 bytes.
@@ -239,8 +288,8 @@ class Data(object):
239
288
  metadatastr = self.metadata.to_json().encode(errors="replace")
240
289
  s.update(metadatastr)
241
290
  if self.table is not None:
242
- tablestr = self.table.to_json().encode(errors="replace")
243
- s.update(tablestr)
291
+ # tablestr = self.table.to_json().encode(errors="replace")
292
+ s.update(self.sha3_256_table.encode(errors="replace"))
244
293
  s.update(self.description.encode(errors="replace"))
245
294
  return s.hexdigest()
246
295
 
@@ -331,6 +380,15 @@ class Data(object):
331
380
 
332
381
  suuid = property(fget=_get_suuid, fset=_set_suuid, doc="suuid of the object")
333
382
 
383
+
384
+ def _get_sname(self):
385
+ return self.metadata.get(self.SDATA_SNAME).value
386
+
387
+ def _set_sname(self, value):
388
+ self.metadata.set_attr(self.SDATA_SNAME, value)
389
+
390
+ sname = property(fget=_get_sname, fset=_set_sname, doc="sname of the object")
391
+
334
392
  def _get_uuid(self):
335
393
  return self.metadata.get(self.SDATA_UUID).value
336
394
  # return self._uuid
@@ -497,6 +555,49 @@ class Data(object):
497
555
  data.to_folder(exportpath, dtype=dtype)
498
556
  return path
499
557
 
558
+ def set_column_metadata(self):
559
+ """convert 'Force [N]' -> 'Force', 'N'
560
+
561
+ add Attribute for each column
562
+
563
+ sdata_column_0: value="Force", unit="N"
564
+ sdata_column_1
565
+ ...
566
+ sdata_column_n
567
+
568
+
569
+ """
570
+ if self.df is None:
571
+ return
572
+ for i, col in enumerate(self.df.columns):
573
+ name, unit = nameunit_from_colname(col)
574
+ scolname = f"{self.SDATA_COLUMN}_{i}"
575
+ self.metadata.add(scolname, value=name, unit=unit, label=col)
576
+
577
+ def set_columnnames_from_metadata(self):
578
+ """Use '!sdata_column_*' to reformat columns names
579
+
580
+ '!sdata_column_0', value='Force', unit='N' -> 'Force [N]'
581
+
582
+ > f'{name} [{unit}]'
583
+
584
+ """
585
+ if self.df is None:
586
+ return
587
+
588
+ cols = []
589
+ for i, col in enumerate(self.df.columns):
590
+ scolname = f"{self.SDATA_COLUMN}_{i}"
591
+ attr = self.metadata.get(scolname)
592
+ if attr is None:
593
+ name, unit = nameunit_from_colname(col)
594
+ else:
595
+ name = attr.value
596
+ unit = attr.unit
597
+ new_colname = f'{name} [{unit}]'
598
+ cols.append(new_colname)
599
+ self.df.columns = cols
600
+
500
601
  @classmethod
501
602
  def from_folder(cls, path):
502
603
  """sdata object instance
@@ -635,6 +736,14 @@ class Data(object):
635
736
  def mdf(self):
636
737
  return self.metadata.df
637
738
 
739
+ @property
740
+ def udf(self):
741
+ return self.metadata.udf
742
+
743
+ @property
744
+ def sdf(self):
745
+ return self.metadata.sdf
746
+
638
747
  @property
639
748
  def md(self):
640
749
  return self.metadata
@@ -729,6 +838,39 @@ class Data(object):
729
838
  """
730
839
  return [(x.name, x.dir()) for x in self.group.values()]
731
840
 
841
+ def to_parquet(self, filepath=None, **kwargs):
842
+ """
843
+
844
+ """
845
+ engine = kwargs.get("engine", "pyarrow")
846
+ df = self.df.copy()
847
+ #df["!sdata_index"] = df.index
848
+ #df.reset_index(inplace=True)
849
+ df.attrs = {"metadata": self.metadata.to_dict(),
850
+ "description": self.description}
851
+ df.to_parquet(filepath, engine=engine)
852
+
853
+ @classmethod
854
+ def from_parquet(cls, filepath):
855
+ """load data from parquet file
856
+
857
+ :param filepath:
858
+ :return: Data object
859
+ """
860
+ try:
861
+ if os.path.exists(filepath):
862
+ df = pd.read_parquet(filepath)
863
+ tt = cls(name=filepath)
864
+ tt.table = df
865
+ attrs = df.attrs
866
+ tt.metadata = tt.metadata.from_dict(attrs["metadata"])
867
+ tt.description = attrs.get("description")
868
+ return tt
869
+ else:
870
+ raise Exception("parquet file '{}' not available".format(filepath))
871
+ except Exception as exp:
872
+ raise
873
+
732
874
  def to_xlsx_byteio(self):
733
875
  """get xlsx as byteio
734
876
 
@@ -839,36 +981,31 @@ class Data(object):
839
981
  """
840
982
  try:
841
983
  if os.path.exists(filepath):
842
- wb = openpyxl.load_workbook(filename=filepath)
843
- sheetnames = wb.sheetnames
844
-
845
984
  tt = cls(name=filepath)
846
985
 
847
986
  # read df
848
- if "table" in sheetnames:
987
+ try:
849
988
  tt.table = pd.read_excel(filepath, sheet_name="table", index_col=0)
850
- else:
989
+ except:
851
990
  logger.info("no table data in '{}'".format(filepath))
852
991
  dfm = pd.read_excel(filepath, sheet_name="metadata")
853
- dfm = dfm.set_index(dfm.name.values)
992
+ dfm.set_index(dfm.name.values, inplace=True)
854
993
  # dfm["value"] = dfm["value"].replace(np.nan, None)
855
994
  dfm["description"] = dfm["description"].replace(np.nan, '')
856
995
  dfm["label"] = dfm["label"].replace(np.nan, '')
857
- # print("!data.from_xlsx", dfm)
996
+
997
+ # read metadata
858
998
  tt.metadata = tt.metadata.from_dataframe(dfm)
999
+ for attr in tt.metadata.sdata_attributes.values():
1000
+ if attr.value == "nan":
1001
+ attr.value = ""
859
1002
 
860
1003
  # read description
861
- if "description" in sheetnames:
862
- cells = []
863
- for cell in wb["description"]["A"]:
864
- if cell.value is not None:
865
- cells.append(cell.value)
866
- else:
867
- cells.append("")
868
- tt.description = "\n".join(cells)
869
- else:
1004
+ try:
1005
+ dfd = pd.read_excel(filepath, sheet_name="description", header=None)
1006
+ tt.description = "\n".join(dfd[0])
1007
+ except:
870
1008
  logger.info("no description in '{}'".format(filepath))
871
-
872
1009
  return tt
873
1010
  else:
874
1011
  raise Exception("excel file '{}' not available".format(filepath))
@@ -987,15 +1124,19 @@ class Data(object):
987
1124
  :param sep: separator (default=";")
988
1125
  :return: sdata.Data
989
1126
  """
1127
+ if s is None and filepath is None:
1128
+ logging.error("specify string if filepath")
1129
+ return
990
1130
  data = cls()
991
1131
  df = None
992
1132
  try:
993
1133
  if filepath:
994
1134
  df = pd.read_csv(filepath, sep=";", comment="#", index_col=0)
995
1135
  sio = open(filepath, "r")
1136
+ sio.seek(0)
996
1137
  elif s is not None:
997
1138
  sio = StringIO(s)
998
- pd.read_csv(sio, sep=";", comment="#")
1139
+ df = pd.read_csv(sio, sep=";", comment="#")
999
1140
  sio.seek(0)
1000
1141
  else:
1001
1142
  logger.error("data.from_csv: no csv data available")
@@ -1243,7 +1384,7 @@ class Data(object):
1243
1384
  :return: Data
1244
1385
  """
1245
1386
  data = copy.deepcopy(self)
1246
- data.metadata.add(self.SDATA_PARENT, self.uuid)
1387
+ data.metadata.add(self.SDATA_PARENT, self.suuid)
1247
1388
  data.metadata.add(self.SDATA_UUID, self.gen_uuid())
1248
1389
  data.metadata.add(self.SDATA_MTIME, now_utc_str(), dtype="str")
1249
1390
  if "uuid" in kwargs:
@@ -21,6 +21,21 @@ class TestProgram(Data):
21
21
  """A sdata Testprogram
22
22
 
23
23
  """
24
+ ATTRIBUTES = [
25
+ ['material_name', '', 'str', '-', 'Der Name des Materials.', '', False],
26
+ ['material_norm_name', '', 'str', '-', 'Der normative Name des Materials.', '', False],
27
+ ['material_number_norm', '', 'str', '-', 'Die normative Nummer des Materials.', '', False],
28
+ ['material_manifacturer', '', 'str', '-', 'Die Hersteller des Materials.', '', False],
29
+ ['material_grade', '', 'str', '-', 'Der Zustand des Materials.', '', False],
30
+ ['material_coating', '', 'str', '-', 'Die Beschichtung des Materials.', '', False],
31
+ ['material_density', '', 'float', 'g/cm^3', 'Die Dichte des Materials.', '', False],
32
+ ['material_production_date', '', 'str', '-', 'Das Produktionsdatum des Materials.', '', False],
33
+ ['responsible_person', '', 'str', '-', 'Verantwortlicher des Testprogramms.', '', False],
34
+ ['responsible_person_phone', '', 'str', '-', 'Telefonnummer des Verantwortlichen vom Testprogramms.', '', False],
35
+ ['responsible_person_email', '', 'str', '-', 'Emailadressen des Verantwortlichen vom Testprogramms.', '', False],
36
+ ['responsible_person_organisation', '', 'str', '-', 'Organisation des Verantwortlichen vom Testprogramms.', '', False],
37
+ ['notes', '', 'str', '-', 'Notizen zum Testprogramm.', '', False],
38
+ ]
24
39
 
25
40
  def __init__(self, **kwargs):
26
41
  """
@@ -48,13 +63,26 @@ class TestProgram(Data):
48
63
  if kwargs.get("uuid_testprogram") is not None:
49
64
  self._set_uuid(kwargs.get("uuid_testprogram"))
50
65
 
66
+ for attrlist in self.ATTRIBUTES:
67
+ self.metadata.set_attr(attrlist[0], attrlist[1],
68
+ dtype=attrlist[2],
69
+ unit=attrlist[3],
70
+ description=attrlist[4],
71
+ label=attrlist[5],
72
+ required=attrlist[6])
73
+
74
+
51
75
  def gen_testseries(self, **kwargs):
52
76
  """generate TestSeries instance
53
77
 
54
78
  :param kwargs:
55
79
  :return: TestSeries()
56
80
  """
57
- ts = TestSeries(**kwargs)
81
+ class_name = kwargs.get("class_name", TestSeries)
82
+ if "class_name" in kwargs:
83
+ kwargs.pop("class_name")
84
+
85
+ ts = class_name(**kwargs)
58
86
  ts.project = self.project
59
87
  ts.name_testprogram = self.name_testprogram
60
88
  ts.uuid_testprogram = self.uuid_testprogram
@@ -63,7 +91,7 @@ class TestProgram(Data):
63
91
 
64
92
  @property
65
93
  def longname(self):
66
- return "_".join([self.name_testprogram])
94
+ return "|".join([self.name_testprogram])
67
95
 
68
96
  class TestSeries(TestProgram):
69
97
  """A sdata TestSeries
@@ -75,6 +103,8 @@ class TestSeries(TestProgram):
75
103
 
76
104
  SDATA_TESTTYPE = "!sdata_testtype"
77
105
 
106
+
107
+
78
108
  def __init__(self, **kwargs):
79
109
  """Test Series
80
110
 
@@ -185,7 +215,11 @@ class TestSeries(TestProgram):
185
215
  :param kwargs:
186
216
  :return: Test()
187
217
  """
188
- t = Test(**kwargs)
218
+ class_name = kwargs.get("class_name", Test)
219
+ if "class_name" in kwargs:
220
+ kwargs.pop("class_name")
221
+
222
+ t = class_name(**kwargs)
189
223
  t.name_testprogram = self.name_testprogram
190
224
  t.uuid_testprogram = self.uuid_testprogram
191
225
  t.name_testseries = self.name_testseries
@@ -198,7 +232,7 @@ class TestSeries(TestProgram):
198
232
 
199
233
  @property
200
234
  def longname(self):
201
- return "_".join([self.name_testprogram, self.name])
235
+ return "|".join([self.name_testprogram, self.name])
202
236
 
203
237
 
204
238
  class Test(TestSeries):
@@ -209,14 +243,15 @@ class Test(TestSeries):
209
243
  SDATA_TESTSERIES_NAME = "!sdata_testseries_name"
210
244
  SDATA_TESTSERIES_UUID = "!sdata_testseries_uuid"
211
245
 
246
+
212
247
  def __init__(self, **kwargs):
213
248
  """Test
214
249
 
215
250
  :param kwargs:
216
251
  """
217
252
  TestSeries.__init__(self, **kwargs)
218
- self.metadata.add(name=self.SDATA_TESTSERIES_NAME, value="N.N.", dtype="str")
219
- self.metadata.add(name=self.SDATA_TESTSERIES_UUID, value="", dtype="str")
253
+ self.metadata.add(name=self.SDATA_TESTSERIES_NAME, value="N.N.", dtype="str", description="name of the testseries")
254
+ self.metadata.add(name=self.SDATA_TESTSERIES_UUID, value="", dtype="str", description="uuid of the testseries")
220
255
 
221
256
  if kwargs.get("name") is not None:
222
257
  self._set_name(kwargs.get("name"))
@@ -235,6 +270,7 @@ class Test(TestSeries):
235
270
  # print("uuid_testprogram", kwargs.get("uuid_testprogram"), self._set_uuid_testprogram)
236
271
  self._set_uuid_testprogram(kwargs.get("uuid_testprogram"))
237
272
 
273
+
238
274
  def _get_uuid_testseries(self):
239
275
  return self.metadata.get(self.SDATA_TESTSERIES_UUID).value
240
276
 
@@ -284,7 +320,7 @@ class Test(TestSeries):
284
320
 
285
321
  @property
286
322
  def longname(self):
287
- return "_".join([self.name_testprogram, self.name_testseries, self.name])
323
+ return "|".join([self.name_testprogram, self.name_testseries, self.name])
288
324
 
289
325
 
290
326
 
@@ -12,6 +12,8 @@ import sdata.contrib.piexif
12
12
  import sdata.contrib.piexif.helper
13
13
  import json
14
14
  import os
15
+ from io import BytesIO
16
+ import base64
15
17
  import hashlib
16
18
  try:
17
19
  import PIL
@@ -47,25 +49,89 @@ class Image(Data):
47
49
  def _from_filepath(cls, filepath, **kwargs):
48
50
  """read image from file
49
51
 
52
+ :param fielpath: image file path
53
+ :return: sdata.Image
50
54
  """
51
55
  project = kwargs.get("project")
52
56
  suuid = SUUID.from_file(cls.__class__.__name__, filepath, ns_name=project)
53
- data = cls(name=os.path.basename(filepath), uuid=suuid.huuid, url=filepath, **kwargs)
57
+ kwargs["uuid"] = suuid.huuid
58
+ kwargs["url"] = filepath
59
+ data = cls(name=os.path.basename(filepath), **kwargs)
54
60
  return data
55
61
 
62
+ @classmethod
63
+ def from_image(cls, name, img, **kwargs):
64
+ """read image from PIL.Image
65
+
66
+ """
67
+ project = kwargs.get("project", "")
68
+ parent = kwargs.get("parent", "")
69
+ class_name = "Image" #cls.__class__.__name__
70
+
71
+ sh = hashlib.sha3_256()
72
+ sh.update(img.tobytes())
73
+ suuid = SUUID.from_name(class_name=class_name, name=sh.hexdigest(), ns_name=project)
74
+ suuid.project = project
75
+ kwargs["parent"] = parent
76
+ kwargs["project"] = suuid.project
77
+ kwargs["sname"] = suuid.sname
78
+ kwargs["uuid"] = suuid.huuid
79
+ kwargs["suuid"] = suuid.idstr
80
+ kwargs["url"] = sdata.osname(name)
81
+ # name = os.path.basename(filepath)
82
+ data = cls(name=name, **kwargs)
83
+ data.img = img
84
+ return data
85
+
86
+
87
+ @classmethod
88
+ def from_bytes(cls, name, image_data, **kwargs):
89
+ """read image from file
90
+
91
+ """
92
+ project = kwargs.get("project")
93
+ class_name = "Image" #cls.__class__.__name__
94
+ suuid = SUUID.from_str(class_name, name, ns_name=project)
95
+ kwargs["sname"] = suuid.sname
96
+ kwargs["uuid"] = suuid.huuid
97
+ kwargs["suuid"] = suuid.idstr
98
+ kwargs["url"] = name
99
+ data = cls(name=name, **kwargs)
100
+ data.img = PIL.Image.open(BytesIO(image_data))
101
+ try:
102
+ d = None
103
+ if name.lower().endswith(".png"):
104
+ d = cls._get_png_metadata(image_data)
105
+ elif name.lower().endswith(("jpg", "jpeg")):
106
+ d = cls._get_jpg_metadata(image_data)
107
+ if d is not None:
108
+ data.metadata = data.metadata.from_json(d)
109
+ except:
110
+ pass
111
+ return data
112
+
113
+
56
114
  @classmethod
57
115
  def from_file(cls, filepath, **kwargs):
58
116
  """read image from file
59
117
 
60
118
  """
61
119
  project = kwargs.get("project")
62
- suuid = SUUID.from_file(cls.__class__.__name__, filepath, ns_name=project)
120
+ class_name = "Image" #cls.__class__.__name__
121
+ suuid = SUUID.from_file(class_name, filepath, ns_name=project)
122
+ kwargs["sname"] = suuid.sname
123
+ kwargs["uuid"] = suuid.huuid
124
+ kwargs["suuid"] = suuid.idstr
125
+ kwargs["url"] = filepath
126
+ name = os.path.basename(filepath)
127
+
128
+
63
129
  if filepath.lower().endswith(".png"):
64
130
  data = cls.from_png(filepath, **kwargs)
65
- elif filepath.lower().endswith(".jpg"):
131
+ elif filepath.lower().endswith(("jpg", "jpeg")):
66
132
  data = cls.from_jpg(filepath, **kwargs)
67
133
  else:
68
- data = cls(name=os.path.basename(filepath), uuid=suuid.huuid, url=filepath, **kwargs)
134
+ data = cls(name=name, **kwargs)
69
135
  return data
70
136
 
71
137
  def get_sha3_256(self, filepath):
@@ -90,6 +156,15 @@ class Image(Data):
90
156
  self.img = PIL.Image.open(self.url)
91
157
  return self.img
92
158
 
159
+ def to_numpy(self):
160
+ """returns a NumPy array created from the color channels of the given Image object
161
+
162
+ array([[[192, 190, 187],
163
+ [193, 190, 187],
164
+ [193, 190, 185],
165
+ """
166
+ return np.array(self.img)
167
+
93
168
  @classmethod
94
169
  def from_png(cls, filepath, **kwargs):
95
170
  """load png image and metadata
@@ -139,9 +214,12 @@ class Image(Data):
139
214
  return
140
215
  data = cls._from_filepath(filepath, **kwargs)
141
216
  # data.load()
142
- d = cls._get_jpg_metadata(filepath)
143
- if d is not None:
144
- data.metadata = data.metadata.from_json(d)
217
+ try:
218
+ d = cls._get_jpg_metadata(filepath)
219
+ if d is not None:
220
+ data.metadata = data.metadata.from_json(d)
221
+ except Exception as exp:
222
+ logging.debug(exp)
145
223
  return data
146
224
 
147
225
  @staticmethod
@@ -158,6 +236,10 @@ class Image(Data):
158
236
  d = json.loads(json_string)
159
237
  return d
160
238
 
239
+ @property
240
+ def basename(self):
241
+ return os.path.basename(self.url)
242
+
161
243
  def saveas(self, filepath, rename=True, random=True, **kwargs):
162
244
  """save image
163
245
 
@@ -205,4 +287,26 @@ class Image(Data):
205
287
  json_string = json.dumps(self.metadata.to_json())
206
288
  exif_dict = {"Exif": {sdata.contrib.piexif.ExifIFD.UserComment: sdata.contrib.piexif.helper.UserComment.dump(json_string)}}
207
289
  exif_bytes = sdata.contrib.piexif.dump(exif_dict)
208
- self.img.save(filepath, "jpeg", exif=exif_bytes)
290
+ self.img.save(filepath, "jpeg", exif=exif_bytes)
291
+
292
+ def get_image_download_link(self, img, filename, text):
293
+
294
+ json_string = json.dumps(self.metadata.to_json())
295
+ buffered = BytesIO()
296
+
297
+ if filename.endswith(".png"):
298
+ metadaten = PIL.PngImagePlugin.PngInfo()
299
+ metadaten.add_text("sdata", json_string)
300
+ img.save(buffered, "PNG", pnginfo=metadaten)
301
+ elif filename.lower().endswith(".jpg"):
302
+ exif_dict = {"Exif": {
303
+ sdata.contrib.piexif.ExifIFD.UserComment: sdata.contrib.piexif.helper.UserComment.dump(json_string)}}
304
+ exif_bytes = sdata.contrib.piexif.dump(exif_dict)
305
+ img.save(buffered, format="JPEG", exif=exif_bytes)
306
+ else:
307
+ logging.warning(f"metadata not supported for {filepath}")
308
+ img.save(buffered)
309
+
310
+ img_str = base64.b64encode(buffered.getvalue()).decode()
311
+ href = f'<a href="data:file/txt;base64,{img_str}" download="{filename}">{text}</a>'
312
+ return href