sdata 1.5.0__tar.gz → 1.6.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdata-1.6.0/MANIFEST.in +2 -0
- {sdata-1.5.0/sdata.egg-info → sdata-1.6.0}/PKG-INFO +2 -3
- {sdata-1.5.0 → sdata-1.6.0}/pyproject.toml +6 -3
- {sdata-1.5.0 → sdata-1.6.0}/sdata/__init__.py +1 -1
- {sdata-1.5.0 → sdata-1.6.0}/sdata/base.py +25 -1
- {sdata-1.5.0 → sdata-1.6.0}/sdata/deprecated/data.py +48 -15
- {sdata-1.5.0 → sdata-1.6.0}/sdata/doe.py +64 -7
- {sdata-1.5.0 → sdata-1.6.0}/sdata/imagemeta.py +85 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/iolib/json1sqlitestore.py +1 -1
- {sdata-1.5.0 → sdata-1.6.0}/sdata/metadata.py +41 -6
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/blob.py +68 -13
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/content.py +18 -5
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/dataframe.py +12 -1
- sdata-1.6.0/sdata/sclass/lca.py +490 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/units.py +108 -68
- {sdata-1.5.0 → sdata-1.6.0/sdata.egg-info}/PKG-INFO +2 -3
- {sdata-1.5.0 → sdata-1.6.0}/sdata.egg-info/SOURCES.txt +6 -30
- sdata-1.6.0/tests/test_base_from_dict_subclass.py +52 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_data.py +32 -9
- sdata-1.6.0/tests/test_doe_coverage.py +98 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_imagemeta.py +71 -0
- sdata-1.6.0/tests/test_metadata_sorteddict.py +51 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_blob_coverage.py +5 -3
- sdata-1.6.0/tests/test_sclass_blob_errors.py +195 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_column_dtype.py +23 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_integrity.py +27 -0
- sdata-1.6.0/tests/test_sclass_lca.py +375 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_testprogram.py +1 -2
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_units_angle.py +9 -9
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_units_convert.py +8 -8
- sdata-1.6.0/tests/test_units_electrical.py +118 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_units_frequency.py +3 -3
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_units_pint_interop.py +16 -9
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_units_qudt_coverage.py +3 -3
- sdata-1.5.0/MANIFEST.in +0 -4
- sdata-1.5.0/sdata/contrib/piexif/__init__.py +0 -12
- sdata-1.5.0/sdata/contrib/piexif/_common.py +0 -94
- sdata-1.5.0/sdata/contrib/piexif/_dump.py +0 -346
- sdata-1.5.0/sdata/contrib/piexif/_exceptions.py +0 -2
- sdata-1.5.0/sdata/contrib/piexif/_exif.py +0 -638
- sdata-1.5.0/sdata/contrib/piexif/_insert.py +0 -60
- sdata-1.5.0/sdata/contrib/piexif/_load.py +0 -270
- sdata-1.5.0/sdata/contrib/piexif/_remove.py +0 -56
- sdata-1.5.0/sdata/contrib/piexif/_transplant.py +0 -45
- sdata-1.5.0/sdata/contrib/piexif/_webp.py +0 -240
- sdata-1.5.0/sdata/contrib/piexif/helper.py +0 -66
- sdata-1.5.0/sdata/contrib/ranger/__init__.py +0 -8
- sdata-1.5.0/sdata/contrib/ranger/collections/__init__.py +0 -0
- sdata-1.5.0/sdata/contrib/ranger/collections/rangebucketmap.py +0 -346
- sdata-1.5.0/sdata/contrib/ranger/collections/rangemap.py +0 -396
- sdata-1.5.0/sdata/contrib/ranger/collections/rangeset.py +0 -483
- sdata-1.5.0/sdata/contrib/ranger/range/__init__.py +0 -0
- sdata-1.5.0/sdata/contrib/ranger/range/cut.py +0 -285
- sdata-1.5.0/sdata/contrib/ranger/range/range.py +0 -644
- sdata-1.5.0/sdata/contrib/sortedcontainers/LICENSE +0 -13
- sdata-1.5.0/sdata/contrib/sortedcontainers/__init__.py +0 -74
- sdata-1.5.0/sdata/contrib/sortedcontainers/sorteddict.py +0 -789
- sdata-1.5.0/sdata/contrib/sortedcontainers/sortedlist.py +0 -2646
- sdata-1.5.0/sdata/contrib/sortedcontainers/sortedset.py +0 -733
- sdata-1.5.0/sdata/contrib/sqlitedict.py +0 -598
- sdata-1.5.0/sdata/contrib/timeflake/__init__.py +0 -40
- sdata-1.5.0/sdata/contrib/timeflake/flake.py +0 -69
- sdata-1.5.0/sdata/contrib/timeflake/utils.py +0 -47
- sdata-1.5.0/sdata/experiments/image_old.py +0 -313
- sdata-1.5.0/tests/test_doe_coverage.py +0 -48
- sdata-1.5.0/tests/test_timeflake.py +0 -124
- {sdata-1.5.0 → sdata-1.6.0}/LICENSE-MIT +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/README.md +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/contrib/__init__.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/contrib/sobol_seq.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/deprecated/__init__.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/__init__.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/_http.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/base58btc.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/did_github.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/did_peer4.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/did_web.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/diddoc.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/ed25519.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/eddsa.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/errors.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/issue_vc.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/jose.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/keys.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/utils_didkey.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/did/verify_vp.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/dtypes.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/experiments/__init__.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/experiments/blob0.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/experiments/file.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/experiments/filegroup.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/experiments/ks2.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/experiments/lapshear.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/experiments/material.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/experiments/parameters_deprecated.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/experiments/workbook.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/format.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/interactive.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/iolib/__init__.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/iolib/hashtree.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/iolib/jsonsqlitestore.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/iolib/pud.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/iolib/reader.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/iolib/writer.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/parameter.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/schema.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/__init__.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/bfo_classes.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/dataframegroup.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/dependency_graph.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/filereference.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/image.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/lazy_namespace.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/process.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/process_graph.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/sclass/zipfile.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/semantic.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/suuid.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/timestamp.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/tools.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata/vocab.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata.egg-info/dependency_links.txt +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata.egg-info/requires.txt +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/sdata.egg-info/top_level.txt +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/setup.cfg +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_attribute.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_attribute_symbol.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_base.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_base_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_blob.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_csv_sidecar_ontology.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_data_deprecation.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_did.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_did_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_dtypes.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_eddsa.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_format_versioning.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_http_backend.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_image.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_interactive.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_io.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_ks2.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_metadata.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_metadata_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_parameter.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_parameter_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_schema.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_blob_hardening.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_blob_io.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_as_blob.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_columns.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_correctness.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_datapackage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_field_metadata.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_hdf.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_interop.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_parquet_filename.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_tableschema.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframe_units.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_dataframegroup_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_filereference_blob.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_filereference_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_lazy_namespace_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_misc_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sclass_zipfile_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sdata_init_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_semantic_columns.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_semantic_jsonld.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_semantic_rdf.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_semantic_vc.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sidecar_wiring.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_sqlite.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_suuid.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_suuid_class.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_table.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_timestamp.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_timestamp_coverage.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_tools.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_unit_system_persistence.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_units.py +0 -0
- {sdata-1.5.0 → sdata-1.6.0}/tests/test_vocab.py +0 -0
sdata-1.6.0/MANIFEST.in
ADDED
|
@@ -1,16 +1,15 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: sdata
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.6.0
|
|
4
4
|
Summary: a structured data format
|
|
5
5
|
Author-email: Ingolf Lepenies <lepy@tuta.io>
|
|
6
6
|
Maintainer-email: Ingolf Lepenies <lepy@tuta.io>
|
|
7
|
-
License: MIT
|
|
7
|
+
License-Expression: MIT
|
|
8
8
|
Project-URL: Homepage, https://github.com/lepy/sdata
|
|
9
9
|
Project-URL: Documentation, https://lepy.github.io/sdata/
|
|
10
10
|
Keywords: open data
|
|
11
11
|
Classifier: Development Status :: 3 - Alpha
|
|
12
12
|
Classifier: Intended Audience :: Developers
|
|
13
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
14
13
|
Classifier: Natural Language :: English
|
|
15
14
|
Classifier: Operating System :: OS Independent
|
|
16
15
|
Classifier: Programming Language :: Python :: 3.9
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["setuptools>=
|
|
2
|
+
requires = ["setuptools>=77.0.0", "wheel"] # >=77: PEP-639-Lizenzfelder
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
@@ -7,7 +7,11 @@ name = "sdata"
|
|
|
7
7
|
dynamic = ["version"] # Single Source: sdata/__init__.py:__version__
|
|
8
8
|
description = "a structured data format"
|
|
9
9
|
readme = "README.md"
|
|
10
|
-
license =
|
|
10
|
+
license = "MIT" # SPDX (PEP 639); RFC 0008 A7 / RFC 0013
|
|
11
|
+
# Nur die eigene Lizenz: das einzige vendored Modul (sdata/contrib/sobol_seq.py,
|
|
12
|
+
# MIT) trägt seinen Lizenztext im Dateikopf. Kein Muster ohne Treffer — setuptools
|
|
13
|
+
# warnt sonst und lehnt solche Builds ab 2027-02-18 ab.
|
|
14
|
+
license-files = ["LICENSE-MIT"]
|
|
11
15
|
requires-python = ">=3.9"
|
|
12
16
|
authors = [{ name = "Ingolf Lepenies", email = "lepy@tuta.io" }]
|
|
13
17
|
maintainers = [{ name = "Ingolf Lepenies", email = "lepy@tuta.io" }]
|
|
@@ -18,7 +22,6 @@ dependencies = ["numpy", "pandas", "suuid>=0.2.0"]
|
|
|
18
22
|
classifiers = [
|
|
19
23
|
"Development Status :: 3 - Alpha",
|
|
20
24
|
"Intended Audience :: Developers",
|
|
21
|
-
"License :: OSI Approved :: MIT License",
|
|
22
25
|
"Natural Language :: English",
|
|
23
26
|
"Operating System :: OS Independent",
|
|
24
27
|
"Programming Language :: Python :: 3.9",
|
|
@@ -403,6 +403,26 @@ class Base:
|
|
|
403
403
|
else:
|
|
404
404
|
return spec
|
|
405
405
|
|
|
406
|
+
@classmethod
|
|
407
|
+
def _class_for_spec(cls, class_spec: str) -> Optional[Type['Base']]:
|
|
408
|
+
"""Konkrete Klasse für ``from_dict`` (``None`` = dynamische Klasse wie bisher).
|
|
409
|
+
|
|
410
|
+
Auf ``Base`` selbst aufgerufen bleibt das generische Verhalten. Auf einer
|
|
411
|
+
Unterklasse: die gespeicherte Klasse, wenn sie ``cls`` oder eine bereits
|
|
412
|
+
geladene Unterklasse davon ist (Abgleich über ``modul:Name``), sonst
|
|
413
|
+
``cls``. Es wird nichts aus den Daten heraus importiert.
|
|
414
|
+
"""
|
|
415
|
+
if cls is Base:
|
|
416
|
+
return None
|
|
417
|
+
module, _, name = class_spec.rpartition(":")
|
|
418
|
+
stack = [cls]
|
|
419
|
+
while stack:
|
|
420
|
+
k = stack.pop()
|
|
421
|
+
if k.__name__ == name and (not module or k.__module__ == module):
|
|
422
|
+
return k
|
|
423
|
+
stack.extend(k.__subclasses__())
|
|
424
|
+
return cls
|
|
425
|
+
|
|
406
426
|
@classmethod
|
|
407
427
|
def from_dict(cls, d: Dict[str, Any], *, strict: bool = False) -> 'Base':
|
|
408
428
|
"""
|
|
@@ -417,7 +437,11 @@ class Base:
|
|
|
417
437
|
metadata = Metadata.from_dict(d.get("metadata", {}))
|
|
418
438
|
class_spec = metadata.get(cls.SDATA_CLASS).value or "sdata.base:Base"
|
|
419
439
|
class_name = cls.classname_from_classspec(class_spec)
|
|
420
|
-
|
|
440
|
+
target = cls._class_for_spec(class_spec)
|
|
441
|
+
if target is None:
|
|
442
|
+
b = sdata_factory(class_name)
|
|
443
|
+
else:
|
|
444
|
+
b = target(name=metadata.get(cls.SDATA_NAME).value or class_name)
|
|
421
445
|
b.metadata = metadata
|
|
422
446
|
b.data = d.get("data", {})
|
|
423
447
|
b.description = d.get("description", "")
|
|
@@ -24,7 +24,40 @@ import json
|
|
|
24
24
|
import hashlib
|
|
25
25
|
import base64
|
|
26
26
|
import re
|
|
27
|
-
|
|
27
|
+
import pickle
|
|
28
|
+
import sqlite3
|
|
29
|
+
|
|
30
|
+
# Schlüssel/Wert-Ablage im Dateiformat des früher vendorten ``sqlitedict``
|
|
31
|
+
# (Apache-2.0): Tabelle "unnamed" (key TEXT PRIMARY KEY, value BLOB), Werte
|
|
32
|
+
# gepickelt. So bleiben mit älteren sdata-Versionen geschriebene Dateien lesbar
|
|
33
|
+
# und neu geschriebene für ältere Versionen lesbar.
|
|
34
|
+
_SQLITEDICT_TABLE = "unnamed"
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _sqlitedict_write(filepath, items):
|
|
38
|
+
con = sqlite3.connect(filepath)
|
|
39
|
+
try:
|
|
40
|
+
with con:
|
|
41
|
+
con.execute(f'CREATE TABLE IF NOT EXISTS "{_SQLITEDICT_TABLE}" '
|
|
42
|
+
'(key TEXT PRIMARY KEY, value BLOB)')
|
|
43
|
+
con.executemany(
|
|
44
|
+
f'REPLACE INTO "{_SQLITEDICT_TABLE}" (key, value) VALUES (?, ?)',
|
|
45
|
+
[(k, sqlite3.Binary(pickle.dumps(v, protocol=pickle.HIGHEST_PROTOCOL)))
|
|
46
|
+
for k, v in items.items()])
|
|
47
|
+
finally:
|
|
48
|
+
con.close()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _sqlitedict_read(filepath):
|
|
52
|
+
"""Alle Einträge lesen. Achtung: Werte werden ent-pickelt — nur vertrauenswürdige Dateien öffnen."""
|
|
53
|
+
if not os.path.exists(filepath):
|
|
54
|
+
raise FileNotFoundError(filepath)
|
|
55
|
+
con = sqlite3.connect(filepath)
|
|
56
|
+
try:
|
|
57
|
+
rows = con.execute(f'SELECT key, value FROM "{_SQLITEDICT_TABLE}"').fetchall()
|
|
58
|
+
finally:
|
|
59
|
+
con.close()
|
|
60
|
+
return {k: pickle.loads(bytes(v)) for k, v in rows}
|
|
28
61
|
|
|
29
62
|
if sys.version_info < (3, 0):
|
|
30
63
|
from StringIO import StringIO
|
|
@@ -1260,13 +1293,13 @@ class Data(object):
|
|
|
1260
1293
|
df = self.df
|
|
1261
1294
|
df.columns = df.columns.astype(str)
|
|
1262
1295
|
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
description
|
|
1269
|
-
|
|
1296
|
+
description = self.description_to_df().copy()
|
|
1297
|
+
description.columns = description.columns.astype(str)
|
|
1298
|
+
_sqlitedict_write(filepath, {
|
|
1299
|
+
'metadata': self.metadata.df.to_parquet(),
|
|
1300
|
+
'table': df.to_parquet(),
|
|
1301
|
+
'description': description.to_parquet(),
|
|
1302
|
+
})
|
|
1270
1303
|
|
|
1271
1304
|
@classmethod
|
|
1272
1305
|
def from_sqlite(cls, filepath, **kwargs):
|
|
@@ -1277,13 +1310,13 @@ class Data(object):
|
|
|
1277
1310
|
:return: sdata.Data
|
|
1278
1311
|
"""
|
|
1279
1312
|
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1313
|
+
sqdict = _sqlitedict_read(filepath)
|
|
1314
|
+
metadata = sdata.Metadata.from_dataframe(pd.read_parquet(BytesIO(sqdict['metadata'])))
|
|
1315
|
+
df_table = pd.read_parquet(BytesIO(sqdict['table']))
|
|
1316
|
+
df_description = pd.read_parquet(BytesIO(sqdict['description']))
|
|
1317
|
+
data = Data(metadata=metadata, table=df_table)
|
|
1318
|
+
data.description_from_df(df_description)
|
|
1319
|
+
return data
|
|
1287
1320
|
|
|
1288
1321
|
@classmethod
|
|
1289
1322
|
def metadata_from_hdf5(cls, filepath, **kwargs):
|
|
@@ -3,10 +3,50 @@
|
|
|
3
3
|
|
|
4
4
|
import collections
|
|
5
5
|
import sdata
|
|
6
|
-
from sdata.contrib.ranger import Range
|
|
7
6
|
import sdata.contrib.sobol_seq
|
|
8
7
|
from sdata.contrib.sobol_seq import Sobol
|
|
9
8
|
import pandas as pd
|
|
9
|
+
from sdata.sclass.dataframe import DataFrame
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Range:
|
|
13
|
+
"""Geschlossenes Intervall ``[min, max]`` eines DoE-Parameters.
|
|
14
|
+
|
|
15
|
+
Ersetzt das früher vendorte ``ranger.Range`` (BSD); sdata nutzt davon nur
|
|
16
|
+
:meth:`closed`, :attr:`min`, :attr:`max`, :attr:`length` und die Textform
|
|
17
|
+
``"[min , max]"``, die so auch in den Metadaten landet.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
__slots__ = ("min", "max")
|
|
21
|
+
|
|
22
|
+
def __init__(self, lower, upper):
|
|
23
|
+
if lower > upper:
|
|
24
|
+
raise ValueError("Lower bound cannot be greater than upper bound")
|
|
25
|
+
self.min = lower
|
|
26
|
+
self.max = upper
|
|
27
|
+
|
|
28
|
+
@classmethod
|
|
29
|
+
def closed(cls, lower, upper):
|
|
30
|
+
return cls(lower, upper)
|
|
31
|
+
|
|
32
|
+
@property
|
|
33
|
+
def length(self):
|
|
34
|
+
return self.max - self.min
|
|
35
|
+
|
|
36
|
+
def __eq__(self, other):
|
|
37
|
+
if not isinstance(other, Range):
|
|
38
|
+
return NotImplemented
|
|
39
|
+
return (self.min, self.max) == (other.min, other.max)
|
|
40
|
+
|
|
41
|
+
def __hash__(self):
|
|
42
|
+
return hash((self.min, self.max))
|
|
43
|
+
|
|
44
|
+
def __repr__(self):
|
|
45
|
+
return f"[{self.min!r} , {self.max!r}]"
|
|
46
|
+
|
|
47
|
+
__str__ = __repr__
|
|
48
|
+
|
|
49
|
+
|
|
10
50
|
class DOE():
|
|
11
51
|
"""
|
|
12
52
|
DoE Generator
|
|
@@ -56,7 +96,7 @@ class DOE():
|
|
|
56
96
|
X = S.i4_sobol_generate(dim_num=self.dim_num, n=n, skip=1)
|
|
57
97
|
df = pd.DataFrame(X, columns=self._ranges.keys())
|
|
58
98
|
df["doe_id"] = range(1, len(X)+1)
|
|
59
|
-
df.set_index("doe_id"
|
|
99
|
+
df = df.set_index("doe_id")
|
|
60
100
|
return df
|
|
61
101
|
|
|
62
102
|
def gen_sobol(self, n=1):
|
|
@@ -64,19 +104,36 @@ class DOE():
|
|
|
64
104
|
X = S.i4_sobol_generate(dim_num=self.dim_num, n=n, skip=1)
|
|
65
105
|
df = pd.DataFrame(X, columns=self._ranges.keys())
|
|
66
106
|
df["doe_id"] = range(1, len(X)+1)
|
|
67
|
-
df.set_index("doe_id"
|
|
107
|
+
df = df.set_index("doe_id")
|
|
68
108
|
for param_name, r in self._ranges.items():
|
|
69
109
|
df[param_name] *= r.length
|
|
70
110
|
df[param_name] += r.min
|
|
71
111
|
return df
|
|
72
112
|
|
|
73
113
|
def to_data(self, name, df_doe, uid=None):
|
|
74
|
-
"""
|
|
114
|
+
"""Design-Tabelle als :class:`~sdata.sclass.dataframe.DataFrame` verpacken.
|
|
115
|
+
|
|
116
|
+
Die Parameterbereiche landen in den Objekt-Metadaten (je Parameter ein
|
|
117
|
+
Attribut mit dem Bereich als Wert).
|
|
75
118
|
|
|
76
|
-
:param
|
|
77
|
-
:
|
|
119
|
+
:param name: Name des Datenobjekts.
|
|
120
|
+
:param df_doe: Design-Tabelle (z. B. aus :meth:`gen_sobol`).
|
|
121
|
+
:param uid: optionale Identität — eine :class:`~sdata.suuid.SUUID` oder
|
|
122
|
+
ein UUID-Hex-String; ohne Angabe wird die SUUID deterministisch aus
|
|
123
|
+
Klasse und ``name`` abgeleitet.
|
|
124
|
+
:return: sdata.sclass.dataframe.DataFrame
|
|
78
125
|
"""
|
|
79
|
-
|
|
126
|
+
kwargs = {}
|
|
127
|
+
if isinstance(uid, sdata.SUUID):
|
|
128
|
+
kwargs["suuid"] = uid
|
|
129
|
+
elif uid is not None:
|
|
130
|
+
# Fremd vorgegebene UUID (Hex, mit oder ohne Bindestriche) übernehmen.
|
|
131
|
+
kwargs["suuid"] = sdata.SUUID(DataFrame.__name__, name, huuid=str(uid).replace("-", ""))
|
|
132
|
+
# Benannten Index (z. B. "doe_id") zur Spalte machen: DataFrame
|
|
133
|
+
# serialisiert ohne Index, die Versuchs-ID ginge sonst verloren.
|
|
134
|
+
if df_doe.index.name is not None and df_doe.index.name not in df_doe.columns:
|
|
135
|
+
df_doe = df_doe.reset_index()
|
|
136
|
+
data = DataFrame(df=df_doe, name=name, description="my DoE", **kwargs)
|
|
80
137
|
for parameter_name, r in self.ranges.items():
|
|
81
138
|
data.metadata.add(parameter_name, r, unit="?", description=f"parameter range {parameter_name}",
|
|
82
139
|
label=rf"${parameter_name}$")
|
|
@@ -57,6 +57,10 @@ class UnsupportedImageFormatError(ImageMetadataError):
|
|
|
57
57
|
"""Das Bildformat wird (zum Schreiben) nicht unterstützt."""
|
|
58
58
|
|
|
59
59
|
|
|
60
|
+
class StripNotSupportedError(ImageMetadataError):
|
|
61
|
+
"""Die Nutzlast lässt sich für dieses Format nicht verlustfrei entfernen (TIFF)."""
|
|
62
|
+
|
|
63
|
+
|
|
60
64
|
class PayloadTooLargeError(ImageMetadataError):
|
|
61
65
|
"""Die Nutzlast passt nicht in ein einzelnes Format-Segment (z. B. JPEG ``APP1``)."""
|
|
62
66
|
|
|
@@ -104,6 +108,16 @@ def _png_embed(data: bytes, payload: str) -> bytes:
|
|
|
104
108
|
return bytes(out)
|
|
105
109
|
|
|
106
110
|
|
|
111
|
+
def _png_strip(data: bytes) -> bytes:
|
|
112
|
+
"""Entferne sdata-Text-Chunks (``iTXt``/``tEXt`` mit Schlüsselwort ``sdata``)."""
|
|
113
|
+
out = bytearray(_PNG_MAGIC)
|
|
114
|
+
for ctype, cdata in _png_chunks(data):
|
|
115
|
+
if ctype in (b"iTXt", b"tEXt") and cdata.split(b"\x00", 1)[0] == _SID:
|
|
116
|
+
continue
|
|
117
|
+
out += _png_make_chunk(ctype, cdata)
|
|
118
|
+
return bytes(out)
|
|
119
|
+
|
|
120
|
+
|
|
107
121
|
def _png_extract(data: bytes) -> Optional[str]:
|
|
108
122
|
"""Lies die sdata-Nutzlast aus ``iTXt``/``tEXt`` mit Schlüsselwort ``sdata``."""
|
|
109
123
|
for ctype, cdata in _png_chunks(data):
|
|
@@ -228,6 +242,16 @@ def _jp2_embed(data: bytes, payload: str) -> bytes:
|
|
|
228
242
|
return bytes(out)
|
|
229
243
|
|
|
230
244
|
|
|
245
|
+
def _jp2_strip(data: bytes) -> bytes:
|
|
246
|
+
"""Entferne die ``uuid``-Box mit der sdata-UUID; alle übrigen Boxen bleiben verbatim."""
|
|
247
|
+
out = bytearray()
|
|
248
|
+
for tbox, start, cstart, end in _jp2_boxes(data):
|
|
249
|
+
if tbox == b"uuid" and data[cstart:cstart + 16] == SDATA_JP2_UUID:
|
|
250
|
+
continue
|
|
251
|
+
out += data[start:end]
|
|
252
|
+
return bytes(out)
|
|
253
|
+
|
|
254
|
+
|
|
231
255
|
def _jp2_extract(data: bytes) -> Optional[str]:
|
|
232
256
|
"""Lies die sdata-Nutzlast aus der ``uuid``-Box mit der sdata-UUID."""
|
|
233
257
|
for tbox, _start, cstart, end in _jp2_boxes(data):
|
|
@@ -373,6 +397,18 @@ def _webp_embed(data: bytes, payload: str) -> bytes:
|
|
|
373
397
|
return b"RIFF" + struct.pack("<I", len(out) + 4) + b"WEBP" + bytes(out)
|
|
374
398
|
|
|
375
399
|
|
|
400
|
+
def _webp_strip(data: bytes) -> bytes:
|
|
401
|
+
"""Entferne den ``sdAT``-Chunk und setze die RIFF-Länge neu."""
|
|
402
|
+
out = bytearray()
|
|
403
|
+
for fourcc, body in _webp_chunks(data):
|
|
404
|
+
if fourcc == _WEBP_FOURCC:
|
|
405
|
+
continue
|
|
406
|
+
out += fourcc + struct.pack("<I", len(body)) + body
|
|
407
|
+
if len(body) & 1:
|
|
408
|
+
out += b"\x00"
|
|
409
|
+
return b"RIFF" + struct.pack("<I", len(out) + 4) + b"WEBP" + bytes(out)
|
|
410
|
+
|
|
411
|
+
|
|
376
412
|
def _webp_extract(data: bytes) -> Optional[str]:
|
|
377
413
|
"""Lies die sdata-Nutzlast aus dem ``sdAT``-Chunk."""
|
|
378
414
|
for fourcc, body in _webp_chunks(data):
|
|
@@ -445,6 +481,14 @@ def _tiff_embed(data: bytes, payload: str) -> bytes:
|
|
|
445
481
|
return bytes(blob)
|
|
446
482
|
|
|
447
483
|
|
|
484
|
+
def _tiff_strip(data: bytes) -> bytes:
|
|
485
|
+
"""Für TIFF nicht umkehrbar: ``_tiff_embed`` lenkt den Header auf eine angehängte
|
|
486
|
+
IFD-Kopie um und bewahrt den ursprünglichen IFD-Offset nicht auf."""
|
|
487
|
+
raise StripNotSupportedError(
|
|
488
|
+
"TIFF: the original IFD offset is not preserved by embed(); "
|
|
489
|
+
"the original bytes cannot be restored")
|
|
490
|
+
|
|
491
|
+
|
|
448
492
|
def _tiff_extract(data: bytes) -> Optional[str]:
|
|
449
493
|
"""Lies die sdata-Nutzlast aus dem privaten TIFF-Tag (inline oder per Offset)."""
|
|
450
494
|
e = _tiff_endian(data)
|
|
@@ -470,6 +514,16 @@ _HANDLERS = {
|
|
|
470
514
|
"tiff": (_tiff_embed, _tiff_extract),
|
|
471
515
|
}
|
|
472
516
|
|
|
517
|
+
#: ``fmt -> strip``: Umkehrung von :func:`embed`, ``strip(embed(x, p)) == x``.
|
|
518
|
+
_STRIPPERS = {
|
|
519
|
+
"png": _png_strip,
|
|
520
|
+
"jpeg": _jpeg_strip,
|
|
521
|
+
"jp2": _jp2_strip,
|
|
522
|
+
"gif": _gif_strip,
|
|
523
|
+
"webp": _webp_strip,
|
|
524
|
+
"tiff": _tiff_strip,
|
|
525
|
+
}
|
|
526
|
+
|
|
473
527
|
|
|
474
528
|
def detect_format(data: bytes) -> Optional[str]:
|
|
475
529
|
"""Erkenne das Bildformat an den Magic-Bytes.
|
|
@@ -531,3 +585,34 @@ def extract(data: bytes, fmt: Optional[str] = None) -> Optional[str]:
|
|
|
531
585
|
if fmt not in _HANDLERS:
|
|
532
586
|
return None
|
|
533
587
|
return _HANDLERS[fmt][1](data)
|
|
588
|
+
|
|
589
|
+
|
|
590
|
+
def strippable_formats() -> Tuple[str, ...]:
|
|
591
|
+
"""Formate, für die :func:`strip` die Original-Bytes bitgleich herstellt."""
|
|
592
|
+
return tuple(f for f, fn in _STRIPPERS.items() if fn is not _tiff_strip)
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
def strip(data: bytes, fmt: Optional[str] = None) -> bytes:
|
|
596
|
+
"""Entferne eine eingebettete sdata-Nutzlast (Umkehrung von :func:`embed`).
|
|
597
|
+
|
|
598
|
+
Für die Formate aus :func:`strippable_formats` gilt ``strip(embed(x, p)) == x``,
|
|
599
|
+
sofern ``x`` selbst keine sdata-Nutzlast trug. So lässt sich ein
|
|
600
|
+
inhaltsbasierter Hash über die Bytes *ohne* Nutzlast bilden, der sich durch
|
|
601
|
+
das Einbetten nicht ändert. Bilder ohne Nutzlast werden unverändert (dasselbe
|
|
602
|
+
Objekt) zurückgegeben.
|
|
603
|
+
|
|
604
|
+
:param data: die Bild-Bytes.
|
|
605
|
+
:param fmt: Format-Schlüssel; ``None`` → automatische Erkennung.
|
|
606
|
+
:return: die Bild-Bytes ohne sdata-Nutzlast.
|
|
607
|
+
:raises UnsupportedImageFormatError: wenn das Format unbekannt/nicht unterstützt ist.
|
|
608
|
+
:raises StripNotSupportedError: wenn eine Nutzlast vorliegt, das Format sie aber
|
|
609
|
+
nicht verlustfrei entfernen kann (TIFF).
|
|
610
|
+
"""
|
|
611
|
+
fmt = fmt or detect_format(data)
|
|
612
|
+
if fmt not in _STRIPPERS:
|
|
613
|
+
raise UnsupportedImageFormatError(
|
|
614
|
+
f"unsupported image format: {fmt!r} "
|
|
615
|
+
f"(supported: {', '.join(_STRIPPERS)})")
|
|
616
|
+
if _HANDLERS[fmt][1](data) is None:
|
|
617
|
+
return data
|
|
618
|
+
return _STRIPPERS[fmt](data)
|
|
@@ -14,7 +14,41 @@ import os
|
|
|
14
14
|
import hashlib
|
|
15
15
|
import re
|
|
16
16
|
import copy
|
|
17
|
-
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SortedDict(dict):
|
|
20
|
+
"""``dict``, das in Schlüssel-Reihenfolge iteriert.
|
|
21
|
+
|
|
22
|
+
Ersetzt das früher vendorte ``sortedcontainers.SortedDict``; sdata braucht
|
|
23
|
+
davon nur die sortierte Iteration. Sortiert wird bei jedem Durchlauf —
|
|
24
|
+
für Metadaten-Größen (wenige hundert Attribute) unerheblich.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
__slots__ = ()
|
|
28
|
+
|
|
29
|
+
def __iter__(self):
|
|
30
|
+
return iter(sorted(dict.keys(self)))
|
|
31
|
+
|
|
32
|
+
def __reversed__(self):
|
|
33
|
+
return iter(sorted(dict.keys(self), reverse=True))
|
|
34
|
+
|
|
35
|
+
def keys(self):
|
|
36
|
+
return list(self)
|
|
37
|
+
|
|
38
|
+
def values(self):
|
|
39
|
+
return [self[k] for k in self]
|
|
40
|
+
|
|
41
|
+
def items(self):
|
|
42
|
+
return [(k, self[k]) for k in self]
|
|
43
|
+
|
|
44
|
+
def copy(self):
|
|
45
|
+
return type(self)(self)
|
|
46
|
+
|
|
47
|
+
__copy__ = copy
|
|
48
|
+
|
|
49
|
+
def __repr__(self):
|
|
50
|
+
return f"{type(self).__name__}({dict(self.items())!r})"
|
|
51
|
+
|
|
18
52
|
|
|
19
53
|
def extract_name_unit(value):
|
|
20
54
|
"""extract name and unit from a combined string
|
|
@@ -527,10 +561,11 @@ class Metadata(object):
|
|
|
527
561
|
@classmethod
|
|
528
562
|
def from_dataframe(cls, df):
|
|
529
563
|
"""create metadata from dataframe"""
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
564
|
+
# astype(object): leere Spalten liest pandas als float64 (NaN); ein
|
|
565
|
+
# String-Fill auf float64 ist deprecated (pandas ≥ 3.1).
|
|
566
|
+
for key in ("unit", "label", "description"):
|
|
567
|
+
df[key] = df[key].astype(object).fillna("")
|
|
568
|
+
df["symbol"] = df["symbol"].astype(object).fillna("") if "symbol" in df.columns else ""
|
|
534
569
|
d = df.to_dict(orient='index')
|
|
535
570
|
metadata = cls.from_dict(d)
|
|
536
571
|
return metadata
|
|
@@ -574,7 +609,7 @@ class Metadata(object):
|
|
|
574
609
|
# Index nach Spalte "name" (Spalte behalten). Kein df.name.values, da das
|
|
575
610
|
# in neueren pandas-Versionen ein StringArray ist, den set_index als
|
|
576
611
|
# unhashbaren Key behandelt (TypeError).
|
|
577
|
-
df.set_index("name", drop=False
|
|
612
|
+
df = df.set_index("name", drop=False)
|
|
578
613
|
metadata = cls.from_dataframe(df)
|
|
579
614
|
return metadata
|
|
580
615
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import base64
|
|
2
|
+
import errno as _errno
|
|
2
3
|
import logging
|
|
3
4
|
import hashlib
|
|
4
5
|
import os
|
|
@@ -18,6 +19,44 @@ except ImportError: # pragma: no cover - optionales Backend (sdata[blob])
|
|
|
18
19
|
logger.warning('fsspec not installed')
|
|
19
20
|
fsspec = None
|
|
20
21
|
|
|
22
|
+
|
|
23
|
+
class ContentUnavailableError(ValueError):
|
|
24
|
+
"""Der Inhalt eines Blob konnte nicht gelesen oder geprüft werden (RFC 0019).
|
|
25
|
+
|
|
26
|
+
Unterklasse von ``ValueError``, damit ältere ``except ValueError`` um
|
|
27
|
+
:attr:`Blob.content_bytes` weiter greifen. ``uri`` nennt die Quelle,
|
|
28
|
+
``__cause__`` den ursprünglichen Fehler des Backends.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
uri = None # Quelle; von _unavailable gesetzt
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class ContentNotFoundError(FileNotFoundError, ContentUnavailableError):
|
|
35
|
+
"""Die Quelle existiert nicht (zugleich ``FileNotFoundError``)."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class ContentPermissionError(PermissionError, ContentUnavailableError):
|
|
39
|
+
"""Keine Berechtigung für die Quelle (zugleich ``PermissionError``)."""
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _unavailable(uri: str, exc: BaseException) -> ContentUnavailableError:
|
|
43
|
+
"""Übersetzt einen Backend-Fehler; der Aufrufer schreibt ``raise … from exc``.
|
|
44
|
+
|
|
45
|
+
Der OS-Fehlertyp steht in der Basisliste vorne, damit ``errno`` und
|
|
46
|
+
``filename`` gesetzt werden; ``uri`` wird nachträglich gesetzt, weil
|
|
47
|
+
``OSError.__new__`` keine Schlüsselwort-Argumente annimmt.
|
|
48
|
+
"""
|
|
49
|
+
msg = f"Failed to access URI '{uri}': {exc}"
|
|
50
|
+
code = getattr(exc, "errno", None)
|
|
51
|
+
if isinstance(exc, FileNotFoundError):
|
|
52
|
+
err = ContentNotFoundError(code or _errno.ENOENT, msg, uri)
|
|
53
|
+
elif isinstance(exc, PermissionError):
|
|
54
|
+
err = ContentPermissionError(code or _errno.EACCES, msg, uri)
|
|
55
|
+
else:
|
|
56
|
+
err = ContentUnavailableError(msg)
|
|
57
|
+
err.uri = uri
|
|
58
|
+
return err
|
|
59
|
+
|
|
21
60
|
class Blob(ContentIntegrityMixin, Base):
|
|
22
61
|
"""
|
|
23
62
|
A derived class from Base that represents a generic binary large object (Blob).
|
|
@@ -221,8 +260,12 @@ class Blob(ContentIntegrityMixin, Base):
|
|
|
221
260
|
Caches the result for subsequent accesses.
|
|
222
261
|
|
|
223
262
|
:return: The content as bytes.
|
|
224
|
-
:raises
|
|
225
|
-
:raises
|
|
263
|
+
:raises ContentNotFoundError: the URI does not exist (also ``FileNotFoundError``).
|
|
264
|
+
:raises ContentPermissionError: no permission (also ``PermissionError``).
|
|
265
|
+
:raises ContentUnavailableError: any other backend error, e.g. a directory,
|
|
266
|
+
a missing backend such as s3fs or an unknown protocol (RFC 0019).
|
|
267
|
+
:raises ValueError: no content/value set or unknown content_type.
|
|
268
|
+
:raises ImportError: fsspec is not installed.
|
|
226
269
|
"""
|
|
227
270
|
if getattr(self, '_content_cache', None) is not None:
|
|
228
271
|
return self._content_cache
|
|
@@ -245,7 +288,7 @@ class Blob(ContentIntegrityMixin, Base):
|
|
|
245
288
|
with fsspec.open(val, 'rb') as f:
|
|
246
289
|
loaded_bytes = f.read()
|
|
247
290
|
except Exception as e:
|
|
248
|
-
raise
|
|
291
|
+
raise _unavailable(val, e) from e
|
|
249
292
|
else:
|
|
250
293
|
raise ValueError(f"Unknown content_type: {ctype}")
|
|
251
294
|
|
|
@@ -264,7 +307,15 @@ class Blob(ContentIntegrityMixin, Base):
|
|
|
264
307
|
def exists(self) -> bool:
|
|
265
308
|
"""
|
|
266
309
|
Test whether the blob content exists.
|
|
267
|
-
|
|
310
|
+
|
|
311
|
+
For 'uri', asks the fsspec backend whether the path exists — an existing
|
|
312
|
+
but unreadable path (permissions, directory) counts as existing. For
|
|
313
|
+
'bytes', True if a value is set. False also without content or for an
|
|
314
|
+
unknown content_type.
|
|
315
|
+
|
|
316
|
+
:raises ContentUnavailableError: the existence cannot be checked, e.g.
|
|
317
|
+
missing backend such as s3fs or unknown protocol (RFC 0019).
|
|
318
|
+
:raises ImportError: fsspec is not installed.
|
|
268
319
|
"""
|
|
269
320
|
content = self.data.get('content')
|
|
270
321
|
if content is None:
|
|
@@ -279,14 +330,12 @@ class Blob(ContentIntegrityMixin, Base):
|
|
|
279
330
|
return True # In-memory, assumes exists if set
|
|
280
331
|
elif ctype == 'uri':
|
|
281
332
|
if fsspec is None:
|
|
282
|
-
|
|
283
|
-
return False
|
|
333
|
+
raise ImportError("fsspec is required for URI exists() (pip install sdata[blob]).")
|
|
284
334
|
try:
|
|
285
335
|
fs, _, paths = fsspec.core.get_fs_token_paths(val)
|
|
286
|
-
return fs.exists(paths[0])
|
|
336
|
+
return bool(fs.exists(paths[0]))
|
|
287
337
|
except Exception as e:
|
|
288
|
-
|
|
289
|
-
return False
|
|
338
|
+
raise _unavailable(val, e) from e
|
|
290
339
|
else:
|
|
291
340
|
return False
|
|
292
341
|
|
|
@@ -309,11 +358,13 @@ class Blob(ContentIntegrityMixin, Base):
|
|
|
309
358
|
def open(self, mode: str = "rb"):
|
|
310
359
|
"""Return a file-like handle to the content; use as a context manager.
|
|
311
360
|
|
|
312
|
-
For ``uri`` content
|
|
313
|
-
in-memory load);
|
|
314
|
-
|
|
361
|
+
For ``uri`` content the source is opened **immediately** and a
|
|
362
|
+
streaming file object is returned (no full in-memory load); errors
|
|
363
|
+
surface at the call, translated like :attr:`content_bytes` (RFC 0019).
|
|
364
|
+
For ``bytes`` content an :class:`io.BytesIO` over the decoded bytes.
|
|
315
365
|
|
|
316
366
|
:param mode: open mode (default ``"rb"``).
|
|
367
|
+
:raises ContentUnavailableError: (or its subclasses) the URI cannot be opened.
|
|
317
368
|
:raises ValueError: if no content/value is set or the content_type is unknown.
|
|
318
369
|
:raises ImportError: if fsspec is required (uri) but not installed.
|
|
319
370
|
"""
|
|
@@ -324,7 +375,11 @@ class Blob(ContentIntegrityMixin, Base):
|
|
|
324
375
|
if ctype == 'uri':
|
|
325
376
|
if fsspec is None:
|
|
326
377
|
raise ImportError("fsspec is required for uri open() (pip install sdata[blob]).")
|
|
327
|
-
|
|
378
|
+
uri = content['value']
|
|
379
|
+
try:
|
|
380
|
+
return fsspec.open(uri, mode).open()
|
|
381
|
+
except Exception as e:
|
|
382
|
+
raise _unavailable(uri, e) from e
|
|
328
383
|
elif ctype == 'bytes':
|
|
329
384
|
return io.BytesIO(self.content_bytes)
|
|
330
385
|
else:
|