sdata 0.25.4__tar.gz → 1.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdata-1.5.0/MANIFEST.in +4 -0
- sdata-1.5.0/PKG-INFO +264 -0
- sdata-1.5.0/README.md +214 -0
- sdata-1.5.0/pyproject.toml +89 -0
- sdata-1.5.0/sdata/__init__.py +140 -0
- sdata-1.5.0/sdata/base.py +684 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/ranger/collections/rangebucketmap.py +5 -1
- sdata-1.5.0/sdata/deprecated/__init__.py +10 -0
- {sdata-0.25.4/sdata → sdata-1.5.0/sdata/deprecated}/data.py +62 -18
- sdata-1.5.0/sdata/did/__init__.py +129 -0
- sdata-1.5.0/sdata/did/_http.py +67 -0
- sdata-1.5.0/sdata/did/base58btc.py +49 -0
- sdata-1.5.0/sdata/did/did_github.py +152 -0
- sdata-1.5.0/sdata/did/did_peer4.py +155 -0
- sdata-1.5.0/sdata/did/did_web.py +222 -0
- sdata-1.5.0/sdata/did/diddoc.py +75 -0
- sdata-1.5.0/sdata/did/ed25519.py +225 -0
- sdata-1.5.0/sdata/did/eddsa.py +107 -0
- sdata-1.5.0/sdata/did/errors.py +40 -0
- sdata-1.5.0/sdata/did/issue_vc.py +193 -0
- sdata-1.5.0/sdata/did/jose.py +126 -0
- sdata-1.5.0/sdata/did/keys.py +104 -0
- sdata-1.5.0/sdata/did/utils_didkey.py +161 -0
- sdata-1.5.0/sdata/did/verify_vp.py +158 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/doe.py +1 -0
- sdata-1.5.0/sdata/dtypes.py +510 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/experiments/__init__.py +5 -1
- sdata-0.25.4/sdata/blob.py → sdata-1.5.0/sdata/experiments/blob0.py +3 -3
- sdata-1.5.0/sdata/experiments/file.py +42 -0
- sdata-1.5.0/sdata/experiments/filegroup.py +144 -0
- sdata-0.25.4/sdata/image.py → sdata-1.5.0/sdata/experiments/image_old.py +2 -1
- {sdata-0.25.4/sdata → sdata-1.5.0/sdata/experiments}/workbook.py +1 -1
- sdata-1.5.0/sdata/format.py +125 -0
- sdata-1.5.0/sdata/imagemeta.py +533 -0
- sdata-1.5.0/sdata/interactive.py +118 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/iolib/hashtree.py +1 -1
- sdata-1.5.0/sdata/iolib/json1sqlitestore.py +554 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/iolib/jsonsqlitestore.py +17 -4
- {sdata-0.25.4 → sdata-1.5.0}/sdata/iolib/pud.py +3 -3
- sdata-1.5.0/sdata/iolib/reader.py +262 -0
- sdata-1.5.0/sdata/iolib/writer.py +419 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/metadata.py +218 -92
- {sdata-0.25.4 → sdata-1.5.0}/sdata/parameter.py +4 -1
- sdata-1.5.0/sdata/schema.py +273 -0
- sdata-1.5.0/sdata/sclass/__init__.py +111 -0
- sdata-1.5.0/sdata/sclass/bfo_classes.py +362 -0
- sdata-1.5.0/sdata/sclass/blob.py +362 -0
- sdata-1.5.0/sdata/sclass/content.py +87 -0
- sdata-1.5.0/sdata/sclass/dataframe.py +1251 -0
- sdata-1.5.0/sdata/sclass/dataframegroup.py +187 -0
- sdata-1.5.0/sdata/sclass/dependency_graph.py +85 -0
- sdata-1.5.0/sdata/sclass/filereference.py +124 -0
- sdata-1.5.0/sdata/sclass/image.py +218 -0
- sdata-1.5.0/sdata/sclass/lazy_namespace.py +155 -0
- sdata-1.5.0/sdata/sclass/process.py +383 -0
- sdata-1.5.0/sdata/sclass/process_graph.py +221 -0
- sdata-1.5.0/sdata/sclass/zipfile.py +135 -0
- sdata-1.5.0/sdata/semantic.py +346 -0
- sdata-1.5.0/sdata/suuid.py +136 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/timestamp.py +14 -10
- sdata-1.5.0/sdata/units.py +602 -0
- sdata-1.5.0/sdata/vocab.py +149 -0
- sdata-1.5.0/sdata.egg-info/PKG-INFO +264 -0
- sdata-1.5.0/sdata.egg-info/SOURCES.txt +172 -0
- sdata-1.5.0/sdata.egg-info/requires.txt +34 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata.egg-info/top_level.txt +0 -1
- {sdata-0.25.4 → sdata-1.5.0}/setup.cfg +0 -3
- sdata-1.5.0/tests/test_attribute_symbol.py +137 -0
- sdata-1.5.0/tests/test_base.py +211 -0
- sdata-1.5.0/tests/test_base_coverage.py +142 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_blob.py +0 -1
- sdata-1.5.0/tests/test_csv_sidecar_ontology.py +33 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_data.py +2 -0
- sdata-1.5.0/tests/test_data_deprecation.py +50 -0
- sdata-1.5.0/tests/test_did.py +383 -0
- sdata-1.5.0/tests/test_did_coverage.py +352 -0
- sdata-1.5.0/tests/test_doe_coverage.py +48 -0
- sdata-1.5.0/tests/test_dtypes.py +520 -0
- sdata-1.5.0/tests/test_eddsa.py +124 -0
- sdata-1.5.0/tests/test_format_versioning.py +171 -0
- sdata-1.5.0/tests/test_http_backend.py +88 -0
- sdata-1.5.0/tests/test_image.py +142 -0
- sdata-1.5.0/tests/test_imagemeta.py +364 -0
- sdata-1.5.0/tests/test_interactive.py +60 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_io.py +3 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_metadata.py +19 -1
- sdata-1.5.0/tests/test_metadata_coverage.py +249 -0
- sdata-1.5.0/tests/test_parameter_coverage.py +171 -0
- sdata-1.5.0/tests/test_schema.py +146 -0
- sdata-1.5.0/tests/test_sclass_blob_coverage.py +118 -0
- sdata-1.5.0/tests/test_sclass_blob_hardening.py +51 -0
- sdata-1.5.0/tests/test_sclass_blob_io.py +55 -0
- sdata-1.5.0/tests/test_sclass_dataframe_as_blob.py +76 -0
- sdata-1.5.0/tests/test_sclass_dataframe_column_dtype.py +61 -0
- sdata-1.5.0/tests/test_sclass_dataframe_columns.py +101 -0
- sdata-1.5.0/tests/test_sclass_dataframe_correctness.py +126 -0
- sdata-1.5.0/tests/test_sclass_dataframe_coverage.py +86 -0
- sdata-1.5.0/tests/test_sclass_dataframe_datapackage.py +105 -0
- sdata-1.5.0/tests/test_sclass_dataframe_field_metadata.py +64 -0
- sdata-1.5.0/tests/test_sclass_dataframe_hdf.py +146 -0
- sdata-1.5.0/tests/test_sclass_dataframe_integrity.py +77 -0
- sdata-1.5.0/tests/test_sclass_dataframe_interop.py +167 -0
- sdata-1.5.0/tests/test_sclass_dataframe_parquet_filename.py +49 -0
- sdata-1.5.0/tests/test_sclass_dataframe_tableschema.py +112 -0
- sdata-1.5.0/tests/test_sclass_dataframe_units.py +224 -0
- sdata-1.5.0/tests/test_sclass_dataframegroup_coverage.py +110 -0
- sdata-1.5.0/tests/test_sclass_filereference_blob.py +34 -0
- sdata-1.5.0/tests/test_sclass_filereference_coverage.py +32 -0
- sdata-1.5.0/tests/test_sclass_lazy_namespace_coverage.py +91 -0
- sdata-1.5.0/tests/test_sclass_misc_coverage.py +56 -0
- sdata-1.5.0/tests/test_sclass_zipfile_coverage.py +59 -0
- sdata-1.5.0/tests/test_sdata_init_coverage.py +30 -0
- sdata-1.5.0/tests/test_semantic_columns.py +60 -0
- sdata-1.5.0/tests/test_semantic_jsonld.py +110 -0
- sdata-1.5.0/tests/test_semantic_rdf.py +49 -0
- sdata-1.5.0/tests/test_semantic_vc.py +53 -0
- sdata-1.5.0/tests/test_sidecar_wiring.py +42 -0
- sdata-1.5.0/tests/test_suuid.py +24 -0
- sdata-1.5.0/tests/test_suuid_class.py +121 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_table.py +2 -0
- sdata-1.5.0/tests/test_timestamp.py +63 -0
- sdata-1.5.0/tests/test_timestamp_coverage.py +70 -0
- sdata-1.5.0/tests/test_unit_system_persistence.py +102 -0
- sdata-1.5.0/tests/test_units.py +50 -0
- sdata-1.5.0/tests/test_units_angle.py +95 -0
- sdata-1.5.0/tests/test_units_convert.py +186 -0
- sdata-1.5.0/tests/test_units_frequency.py +64 -0
- sdata-1.5.0/tests/test_units_pint_interop.py +141 -0
- sdata-1.5.0/tests/test_units_qudt_coverage.py +54 -0
- sdata-1.5.0/tests/test_vocab.py +65 -0
- sdata-0.25.4/MANIFEST.in +0 -6
- sdata-0.25.4/PKG-INFO +0 -151
- sdata-0.25.4/README.md +0 -101
- sdata-0.25.4/pyproject.toml +0 -3
- sdata-0.25.4/sdata/__init__.py +0 -65
- sdata-0.25.4/sdata/base.py +0 -155
- sdata-0.25.4/sdata/contrib/attrdict/__init__.py +0 -10
- sdata-0.25.4/sdata/contrib/attrdict/default.py +0 -133
- sdata-0.25.4/sdata/contrib/attrdict/dictionary.py +0 -60
- sdata-0.25.4/sdata/contrib/attrdict/mapping.py +0 -100
- sdata-0.25.4/sdata/contrib/attrdict/merge.py +0 -47
- sdata-0.25.4/sdata/contrib/attrdict/mixins.py +0 -213
- sdata-0.25.4/sdata/contrib/semver.py +0 -1259
- sdata-0.25.4/sdata/contrib/simple_graph_db/LICENSE +0 -20
- sdata-0.25.4/sdata/contrib/simple_graph_db/__init__.py +0 -603
- sdata-0.25.4/sdata/contrib/simple_graph_db/database.py +0 -234
- sdata-0.25.4/sdata/contrib/simple_graph_db/schema.sql +0 -17
- sdata-0.25.4/sdata/iolib/db.py +0 -86
- sdata-0.25.4/sdata/iolib/hdf.py +0 -95
- sdata-0.25.4/sdata/iolib/isomme.py +0 -410
- sdata-0.25.4/sdata/iolib/json1sqlitestore.py +0 -596
- sdata-0.25.4/sdata/iolib/owncloudfs.py +0 -105
- sdata-0.25.4/sdata/iolib/pgp.py +0 -80
- sdata-0.25.4/sdata/iolib/simple_signature.py +0 -446
- sdata-0.25.4/sdata/iolib/vault.py +0 -475
- sdata-0.25.4/sdata/node.py +0 -145
- sdata-0.25.4/sdata/suuid.py +0 -287
- sdata-0.25.4/sdata.egg-info/PKG-INFO +0 -151
- sdata-0.25.4/sdata.egg-info/SOURCES.txt +0 -106
- sdata-0.25.4/sdata.egg-info/requires.txt +0 -9
- sdata-0.25.4/setup.py +0 -54
- sdata-0.25.4/tests/__init__.py +0 -0
- sdata-0.25.4/tests/iolib/__init__.py +0 -0
- sdata-0.25.4/tests/iolib/pud/__init__.py +0 -0
- sdata-0.25.4/tests/iolib/pud/test_pud.py +0 -18
- sdata-0.25.4/tests/iolib/test_hasttree.py +0 -67
- sdata-0.25.4/tests/iolib/test_hdf.py +0 -59
- sdata-0.25.4/tests/iolib/test_jsonsqlitestore.py +0 -156
- sdata-0.25.4/tests/test_image.py +0 -28
- sdata-0.25.4/tests/test_suuid.py +0 -45
- sdata-0.25.4/tests/test_timestamp.py +0 -106
- sdata-0.25.4/tests/test_vault.py +0 -146
- sdata-0.25.4/tests/test_workbook.py +0 -43
- {sdata-0.25.4 → sdata-1.5.0}/LICENSE-MIT +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/__init__.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/__init__.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/_common.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/_dump.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/_exceptions.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/_exif.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/_insert.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/_load.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/_remove.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/_transplant.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/_webp.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/piexif/helper.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/ranger/__init__.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/ranger/collections/__init__.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/ranger/collections/rangemap.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/ranger/collections/rangeset.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/ranger/range/__init__.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/ranger/range/cut.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/ranger/range/range.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/sobol_seq.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/sortedcontainers/LICENSE +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/sortedcontainers/__init__.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/sortedcontainers/sorteddict.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/sortedcontainers/sortedlist.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/sortedcontainers/sortedset.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/sqlitedict.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/timeflake/__init__.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/timeflake/flake.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/contrib/timeflake/utils.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/experiments/ks2.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/experiments/lapshear.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/experiments/material.py +0 -0
- {sdata-0.25.4/sdata → sdata-1.5.0/sdata/experiments}/parameters_deprecated.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/iolib/__init__.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata/tools.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/sdata.egg-info/dependency_links.txt +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_attribute.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_ks2.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_parameter.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_sqlite.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_testprogram.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_timeflake.py +0 -0
- {sdata-0.25.4 → sdata-1.5.0}/tests/test_tools.py +0 -0
sdata-1.5.0/MANIFEST.in
ADDED
sdata-1.5.0/PKG-INFO
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sdata
|
|
3
|
+
Version: 1.5.0
|
|
4
|
+
Summary: a structured data format
|
|
5
|
+
Author-email: Ingolf Lepenies <lepy@tuta.io>
|
|
6
|
+
Maintainer-email: Ingolf Lepenies <lepy@tuta.io>
|
|
7
|
+
License: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/lepy/sdata
|
|
9
|
+
Project-URL: Documentation, https://lepy.github.io/sdata/
|
|
10
|
+
Keywords: open data
|
|
11
|
+
Classifier: Development Status :: 3 - Alpha
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Natural Language :: English
|
|
15
|
+
Classifier: Operating System :: OS Independent
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: Implementation :: CPython
|
|
21
|
+
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
|
22
|
+
Requires-Python: >=3.9
|
|
23
|
+
Description-Content-Type: text/markdown
|
|
24
|
+
License-File: LICENSE-MIT
|
|
25
|
+
Requires-Dist: numpy
|
|
26
|
+
Requires-Dist: pandas
|
|
27
|
+
Requires-Dist: suuid>=0.2.0
|
|
28
|
+
Provides-Extra: did
|
|
29
|
+
Provides-Extra: http
|
|
30
|
+
Requires-Dist: requests; extra == "http"
|
|
31
|
+
Provides-Extra: excel
|
|
32
|
+
Requires-Dist: openpyxl; extra == "excel"
|
|
33
|
+
Requires-Dist: xlsxwriter; extra == "excel"
|
|
34
|
+
Requires-Dist: tabulate; extra == "excel"
|
|
35
|
+
Provides-Extra: hdf
|
|
36
|
+
Requires-Dist: h5py; extra == "hdf"
|
|
37
|
+
Provides-Extra: sql
|
|
38
|
+
Requires-Dist: sqlalchemy; extra == "sql"
|
|
39
|
+
Provides-Extra: parquet
|
|
40
|
+
Requires-Dist: pyarrow; extra == "parquet"
|
|
41
|
+
Provides-Extra: blob
|
|
42
|
+
Requires-Dist: fsspec; extra == "blob"
|
|
43
|
+
Provides-Extra: units
|
|
44
|
+
Requires-Dist: pint; extra == "units"
|
|
45
|
+
Provides-Extra: rdf
|
|
46
|
+
Requires-Dist: rdflib; extra == "rdf"
|
|
47
|
+
Provides-Extra: schema
|
|
48
|
+
Requires-Dist: jsonschema; extra == "schema"
|
|
49
|
+
Dynamic: license-file
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
[](https://pypi.python.org/pypi/sdata/)
|
|
53
|
+
[](https://pypi.python.org/pypi/sdata/)
|
|
54
|
+
[](https://lepy.github.io/sdata/)
|
|
55
|
+
[](https://doi.org/10.5281/zenodo.4311323)
|
|
56
|
+
|
|
57
|
+
# Structured data format (sdata)
|
|
58
|
+
|
|
59
|
+
## Design goals & status
|
|
60
|
+
|
|
61
|
+
sdata is an open, self-describing data format for open-science projects. The
|
|
62
|
+
table below is an honest status of the original design goals (triaged in
|
|
63
|
+
[RFC 0008](docs/rfc/0008-bestandsaufnahme-roadmap.md)) — **implemented**,
|
|
64
|
+
**partial**, on the **roadmap**, or an explicit **non-goal**.
|
|
65
|
+
|
|
66
|
+
| Goal | Status | Notes |
|
|
67
|
+
|------|--------|-------|
|
|
68
|
+
| Open, self-describing data | **implemented** | metadata embedded in every format |
|
|
69
|
+
| Platform independent, simple object model | **implemented** | pure Python / numpy / pandas |
|
|
70
|
+
| Standard metadata formats (key/value, JSON-LD, …) | **implemented** | `Metadata`, QUDT/BFO/PROV/CSVW/DID/VC |
|
|
71
|
+
| Physical units + conversion | **implemented** | `sdata.units` (RFC 0006) |
|
|
72
|
+
| Standard dataset formats — HDF5, CSV, Parquet, Arrow, Data Package | **implemented** | HDF5 via `sdata[hdf]` |
|
|
73
|
+
| (De-)serialization of data + metadata | **implemented** | lossless round-trips |
|
|
74
|
+
| Data format versions | **implemented** | `_sdata_format_version` + migration (RFC 0010) |
|
|
75
|
+
| Project standards (e.g. tensile test) | **partial** | schema templates (`MetadataSchema`/`TableSchema`) |
|
|
76
|
+
| Series | **partial** | as a one-column table |
|
|
77
|
+
| Hierarchical structure / nested groups | **partial** | `DataFrameGroup` (flat) + `parent`/`project` relations |
|
|
78
|
+
| Transparent compression | **partial** | Parquet-internal (zstd); zlib is a **non-goal** for the store (speed) |
|
|
79
|
+
| Standard dataset formats — **netCDF** | **roadmap** | not implemented |
|
|
80
|
+
| Dataset types — **datacubes** | **roadmap** | not implemented |
|
|
81
|
+
| Optional encryption (gpg, …) | **non-goal** | encrypt blobs/filesystem out of band |
|
|
82
|
+
| posix-path syntax, change management, tensor-library interop, SWMR | **non-goal** | not planned; revisit on demand |
|
|
83
|
+
|
|
84
|
+
Machine-readable metadata is the backbone (see below); the optional semantic
|
|
85
|
+
backends degrade gracefully to pure Python.
|
|
86
|
+
|
|
87
|
+
## Quickstart
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
import pandas as pd
|
|
91
|
+
from sdata.sclass.dataframe import DataFrame
|
|
92
|
+
|
|
93
|
+
df = pd.DataFrame({"force": [1.0, 2.0, 3.0]})
|
|
94
|
+
data = DataFrame(df=df, name="specimen_01", description="a tension test")
|
|
95
|
+
data.metadata.add("max_force", 12.5, unit="kN", dtype="float",
|
|
96
|
+
description="max force", ontology="bfo:Quality")
|
|
97
|
+
|
|
98
|
+
print(data.metadata.df[["value", "unit", "dtype", "ontology"]])
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
value unit dtype ontology
|
|
103
|
+
key
|
|
104
|
+
_sdata_name specimen_01 - str
|
|
105
|
+
_sdata_sname DataFrame__specimen_01__3003... - str
|
|
106
|
+
...
|
|
107
|
+
max_force 12.5 kN float bfo:Quality
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
Every object has a deterministic, content-addressable identity (`SUUID`), a set
|
|
111
|
+
of reserved `_sdata_*` attributes (name, sname, suuid, class, ctime, parent,
|
|
112
|
+
project, topology class) and freely extensible, fully-described user attributes.
|
|
113
|
+
|
|
114
|
+
## Machine-readable metadata (JSON-LD / Linked Data)
|
|
115
|
+
|
|
116
|
+
The metadata is the backbone of the data description: it lives next to every
|
|
117
|
+
data blob and fully qualifies the data — units (QUDT/UCUM), ontology classes
|
|
118
|
+
(BFO), provenance (PROV/DCAT), tabular columns (CSVW) and identity (DID).
|
|
119
|
+
|
|
120
|
+
```python
|
|
121
|
+
# self-describing JSON-LD (qudt:Quantity units, BFO @type, did:suuid @id, csvw columns)
|
|
122
|
+
doc = data.to_jsonld()
|
|
123
|
+
|
|
124
|
+
# RDF/Turtle (uses rdflib if installed, otherwise returns the JSON-LD)
|
|
125
|
+
print(data.to_turtle())
|
|
126
|
+
|
|
127
|
+
# write the data + a sidecar <sname>.meta.jsonld right next to it
|
|
128
|
+
data.to_json("specimen_01.sjson", sidecar=True)
|
|
129
|
+
|
|
130
|
+
# validate / auto-complete against a schema template
|
|
131
|
+
from sdata.schema import MetadataSchema, AttrSpec
|
|
132
|
+
schema = MetadataSchema("TensileTest", [
|
|
133
|
+
AttrSpec("max_force", dtype="float", unit="kN", required=True, ontology="bfo:Quality"),
|
|
134
|
+
])
|
|
135
|
+
report = data.metadata.validate(schema) # ValidationReport (truthy if ok)
|
|
136
|
+
|
|
137
|
+
# sign the metadata as a W3C Verifiable Credential (pure-Python Ed25519)
|
|
138
|
+
from sdata.did import keys, pub_from_priv_jwk
|
|
139
|
+
priv = keys.gen_ed25519_jwk()
|
|
140
|
+
vc = data.metadata.to_verifiable_credential("did:example:issuer", priv)
|
|
141
|
+
subject = data.metadata.from_verifiable_credential(vc, pub_from_priv_jwk(priv))
|
|
142
|
+
|
|
143
|
+
# interactive: attribute autocomplete + rich Jupyter display
|
|
144
|
+
data.metadata.a.max_force # -> Attribute; tab-completion in Jupyter
|
|
145
|
+
data.metadata # -> _repr_html_ table
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
Resulting JSON-LD for `max_force` (excerpt):
|
|
149
|
+
|
|
150
|
+
```json
|
|
151
|
+
{
|
|
152
|
+
"@id": "did:suuid:DataFrame__specimen_01__3003...:sdata",
|
|
153
|
+
"@type": ["sdata:DataFrame", "bfo:BFO_0000004"],
|
|
154
|
+
"name": "specimen_01",
|
|
155
|
+
"sdata:max_force": {
|
|
156
|
+
"@type": ["qudt:Quantity", "bfo:Quality"],
|
|
157
|
+
"value": {"@value": 12.5, "@type": "xsd:double"},
|
|
158
|
+
"unitRef": "unit:KiloN", "symbol": "kN"
|
|
159
|
+
},
|
|
160
|
+
"columns": [{"name": "force", "datatype": "xsd:double"}]
|
|
161
|
+
}
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
The optional semantic backends degrade gracefully to pure Python (no hard
|
|
165
|
+
dependency): `pip install "sdata[rdf]"` (rdflib), `sdata[units]` (pint),
|
|
166
|
+
`sdata[schema]` (jsonschema). Core install needs only `numpy`, `pandas`, `suuid`.
|
|
167
|
+
|
|
168
|
+
## Tabular data (`DataFrame`)
|
|
169
|
+
|
|
170
|
+
`DataFrame` wraps a pandas frame plus per-column metadata and serializes to many
|
|
171
|
+
formats — Parquet, Arrow/Feather, CSV, dict/JSON, JSON-LD, a Frictionless **Data
|
|
172
|
+
Package** and **HDF5** — with the qualifying metadata embedded or written as a
|
|
173
|
+
sidecar.
|
|
174
|
+
|
|
175
|
+
```python
|
|
176
|
+
import pandas as pd
|
|
177
|
+
from sdata.sclass.dataframe import DataFrame
|
|
178
|
+
|
|
179
|
+
sdf = DataFrame(df=pd.DataFrame({"weight": [10, 20], "height": [1.5, 1.6]}),
|
|
180
|
+
name="specimen_01", description="a tension test")
|
|
181
|
+
|
|
182
|
+
# per-column annotations (only the fields you pass are changed)
|
|
183
|
+
sdf.set_column("weight", unit="kg", label="Gewicht", ontology="bfo:Quality")
|
|
184
|
+
sdf.col["height"].unit = "m" # mutate the column Attribute in place
|
|
185
|
+
sdf.column_units # {'weight': 'kg', 'height': 'm'}
|
|
186
|
+
|
|
187
|
+
# serialize (optional <sname>.meta.jsonld sidecar; bytes/str without a path)
|
|
188
|
+
sdf.to_parquet(path="out", sidecar=True) # out/<sname>.spq + sidecar
|
|
189
|
+
sdf.to_csv(path="out") # data-only CSV (pure pandas)
|
|
190
|
+
sdf.to_feather(path="out") # Arrow IPC + native per-column field metadata
|
|
191
|
+
sdf.to_datapackage(path="out") # Frictionless Data Package (.zip)
|
|
192
|
+
sdf.to_hdf(path="out") # HDF5 via h5py, one dataset/column (sdata[hdf])
|
|
193
|
+
DataFrame.from_parquet("out/specimen_01.spq")
|
|
194
|
+
|
|
195
|
+
# validate the table against a schema (missing/dtype/unit/extra columns)
|
|
196
|
+
from sdata.schema import TableSchema, AttrSpec
|
|
197
|
+
schema = TableSchema("TensileTable", [
|
|
198
|
+
AttrSpec("weight", dtype="int", unit="kg", required=True),
|
|
199
|
+
])
|
|
200
|
+
report = sdf.validate_table(schema) # ValidationReport (truthy if ok)
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
Arrow/Feather/Parquet need `pip install "sdata[parquet]"`, HDF5 `sdata[hdf]`; CSV,
|
|
204
|
+
dict, JSON-LD and the (CSV) Data Package work with the core install. See the
|
|
205
|
+
[Tabular data docs](https://lepy.github.io/sdata/usage/dataframe/)
|
|
206
|
+
([`docs/usage/dataframe.md`](docs/usage/dataframe.md)) for the full reference.
|
|
207
|
+
|
|
208
|
+
## Howto
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
* [Das sdata-Format - slides](https://lepy.github.io/sdata/ipynb/Das_sdata_Format.slides.html#)
|
|
212
|
+
* https://deepwiki.com/lepy/sdata
|
|
213
|
+
|
|
214
|
+
## Demo App
|
|
215
|
+
|
|
216
|
+
* [test the demo app with editor](https://share.streamlit.io/lepy/sdata_streamlit/main/sdata_editor.py)
|
|
217
|
+
|
|
218
|
+
Try to paste some Excel-Data in the forms ...
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
## Metadata
|
|
222
|
+
|
|
223
|
+
### Attributes
|
|
224
|
+
|
|
225
|
+
* name
|
|
226
|
+
* value
|
|
227
|
+
* dtype
|
|
228
|
+
* unit
|
|
229
|
+
* description
|
|
230
|
+
* label
|
|
231
|
+
* required
|
|
232
|
+
* ontology (CURIE/IRI of the value's class, e.g. `bfo:Quality`)
|
|
233
|
+
|
|
234
|
+
### dtypes for attributes
|
|
235
|
+
|
|
236
|
+
Every attribute value is coerced to a declared dtype (single source:
|
|
237
|
+
`sdata.dtypes`), each with a lossless JSON-LD / XSD mapping:
|
|
238
|
+
|
|
239
|
+
* `int`, `float`, `str`, `bool`
|
|
240
|
+
* `list` (list of strings), `floatlist` (typed list of floats)
|
|
241
|
+
* `timestamp` (ISO-8601 with timezone, stdlib `zoneinfo`)
|
|
242
|
+
* `date`, `time`, `duration` (ISO-8601 / `datetime` types)
|
|
243
|
+
* `decimal` (exact decimal), `complex` (complex number)
|
|
244
|
+
* `bytes` (base64 in JSON), `json` (dict/list), `uri`
|
|
245
|
+
* `langstring` (language-tagged string, `rdf:langString`, e.g. `"Hallo@de"`)
|
|
246
|
+
|
|
247
|
+
Coercion is **lenient** by default (invalid values are logged and left unchanged).
|
|
248
|
+
Pass `strict=True` (e.g. `metadata.add(..., strict=True)`) to raise `DtypeError`
|
|
249
|
+
instead. See the [conventions](https://lepy.github.io/sdata/conventions/) for the
|
|
250
|
+
error policy.
|
|
251
|
+
|
|
252
|
+
## paper
|
|
253
|
+
|
|
254
|
+
* [Das sdata-Format](https://zenodo.org/record/4311323#.X89yo9-YXys)
|
|
255
|
+
* Ingolf Lepenies. (2020). Das sdata-Format (Version 0.8.4). http://doi.org/10.5281/zenodo.4311323
|
|
256
|
+
* [slides](https://lepy.github.io/sdata/ipynb/Das_sdata_Format.slides.html#),
|
|
257
|
+
[html](https://lepy.github.io/sdata/paper/2020/Das_sdata-Format.html),
|
|
258
|
+
[pdf](https://lepy.github.io/sdata/paper/2020/Das_sdata-Format.pdf)
|
|
259
|
+
[temperaturmessung-001.json](https://lepy.github.io/sdata/paper/2020/temperaturmessung-001.json)
|
|
260
|
+
[temperaturmessung-001.xlsx](https://lepy.github.io/sdata/paper/2020/temperaturmessung-001.xlsx)
|
|
261
|
+
|
|
262
|
+
* [sdata](https://doi.org/10.5281/zenodo.4311396)
|
|
263
|
+
* Ingolf Lepenies. (2020, December 8). sdata - a structured data format (Version 0.8.4). Zenodo. http://doi.org/10.5281/zenodo.4311397
|
|
264
|
+
|
sdata-1.5.0/README.md
ADDED
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
|
|
2
|
+
[](https://pypi.python.org/pypi/sdata/)
|
|
3
|
+
[](https://pypi.python.org/pypi/sdata/)
|
|
4
|
+
[](https://lepy.github.io/sdata/)
|
|
5
|
+
[](https://doi.org/10.5281/zenodo.4311323)
|
|
6
|
+
|
|
7
|
+
# Structured data format (sdata)
|
|
8
|
+
|
|
9
|
+
## Design goals & status
|
|
10
|
+
|
|
11
|
+
sdata is an open, self-describing data format for open-science projects. The
|
|
12
|
+
table below is an honest status of the original design goals (triaged in
|
|
13
|
+
[RFC 0008](docs/rfc/0008-bestandsaufnahme-roadmap.md)) — **implemented**,
|
|
14
|
+
**partial**, on the **roadmap**, or an explicit **non-goal**.
|
|
15
|
+
|
|
16
|
+
| Goal | Status | Notes |
|
|
17
|
+
|------|--------|-------|
|
|
18
|
+
| Open, self-describing data | **implemented** | metadata embedded in every format |
|
|
19
|
+
| Platform independent, simple object model | **implemented** | pure Python / numpy / pandas |
|
|
20
|
+
| Standard metadata formats (key/value, JSON-LD, …) | **implemented** | `Metadata`, QUDT/BFO/PROV/CSVW/DID/VC |
|
|
21
|
+
| Physical units + conversion | **implemented** | `sdata.units` (RFC 0006) |
|
|
22
|
+
| Standard dataset formats — HDF5, CSV, Parquet, Arrow, Data Package | **implemented** | HDF5 via `sdata[hdf]` |
|
|
23
|
+
| (De-)serialization of data + metadata | **implemented** | lossless round-trips |
|
|
24
|
+
| Data format versions | **implemented** | `_sdata_format_version` + migration (RFC 0010) |
|
|
25
|
+
| Project standards (e.g. tensile test) | **partial** | schema templates (`MetadataSchema`/`TableSchema`) |
|
|
26
|
+
| Series | **partial** | as a one-column table |
|
|
27
|
+
| Hierarchical structure / nested groups | **partial** | `DataFrameGroup` (flat) + `parent`/`project` relations |
|
|
28
|
+
| Transparent compression | **partial** | Parquet-internal (zstd); zlib is a **non-goal** for the store (speed) |
|
|
29
|
+
| Standard dataset formats — **netCDF** | **roadmap** | not implemented |
|
|
30
|
+
| Dataset types — **datacubes** | **roadmap** | not implemented |
|
|
31
|
+
| Optional encryption (gpg, …) | **non-goal** | encrypt blobs/filesystem out of band |
|
|
32
|
+
| posix-path syntax, change management, tensor-library interop, SWMR | **non-goal** | not planned; revisit on demand |
|
|
33
|
+
|
|
34
|
+
Machine-readable metadata is the backbone (see below); the optional semantic
|
|
35
|
+
backends degrade gracefully to pure Python.
|
|
36
|
+
|
|
37
|
+
## Quickstart
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
import pandas as pd
|
|
41
|
+
from sdata.sclass.dataframe import DataFrame
|
|
42
|
+
|
|
43
|
+
df = pd.DataFrame({"force": [1.0, 2.0, 3.0]})
|
|
44
|
+
data = DataFrame(df=df, name="specimen_01", description="a tension test")
|
|
45
|
+
data.metadata.add("max_force", 12.5, unit="kN", dtype="float",
|
|
46
|
+
description="max force", ontology="bfo:Quality")
|
|
47
|
+
|
|
48
|
+
print(data.metadata.df[["value", "unit", "dtype", "ontology"]])
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```
|
|
52
|
+
value unit dtype ontology
|
|
53
|
+
key
|
|
54
|
+
_sdata_name specimen_01 - str
|
|
55
|
+
_sdata_sname DataFrame__specimen_01__3003... - str
|
|
56
|
+
...
|
|
57
|
+
max_force 12.5 kN float bfo:Quality
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
Every object has a deterministic, content-addressable identity (`SUUID`), a set
|
|
61
|
+
of reserved `_sdata_*` attributes (name, sname, suuid, class, ctime, parent,
|
|
62
|
+
project, topology class) and freely extensible, fully-described user attributes.
|
|
63
|
+
|
|
64
|
+
## Machine-readable metadata (JSON-LD / Linked Data)
|
|
65
|
+
|
|
66
|
+
The metadata is the backbone of the data description: it lives next to every
|
|
67
|
+
data blob and fully qualifies the data — units (QUDT/UCUM), ontology classes
|
|
68
|
+
(BFO), provenance (PROV/DCAT), tabular columns (CSVW) and identity (DID).
|
|
69
|
+
|
|
70
|
+
```python
|
|
71
|
+
# self-describing JSON-LD (qudt:Quantity units, BFO @type, did:suuid @id, csvw columns)
|
|
72
|
+
doc = data.to_jsonld()
|
|
73
|
+
|
|
74
|
+
# RDF/Turtle (uses rdflib if installed, otherwise returns the JSON-LD)
|
|
75
|
+
print(data.to_turtle())
|
|
76
|
+
|
|
77
|
+
# write the data + a sidecar <sname>.meta.jsonld right next to it
|
|
78
|
+
data.to_json("specimen_01.sjson", sidecar=True)
|
|
79
|
+
|
|
80
|
+
# validate / auto-complete against a schema template
|
|
81
|
+
from sdata.schema import MetadataSchema, AttrSpec
|
|
82
|
+
schema = MetadataSchema("TensileTest", [
|
|
83
|
+
AttrSpec("max_force", dtype="float", unit="kN", required=True, ontology="bfo:Quality"),
|
|
84
|
+
])
|
|
85
|
+
report = data.metadata.validate(schema) # ValidationReport (truthy if ok)
|
|
86
|
+
|
|
87
|
+
# sign the metadata as a W3C Verifiable Credential (pure-Python Ed25519)
|
|
88
|
+
from sdata.did import keys, pub_from_priv_jwk
|
|
89
|
+
priv = keys.gen_ed25519_jwk()
|
|
90
|
+
vc = data.metadata.to_verifiable_credential("did:example:issuer", priv)
|
|
91
|
+
subject = data.metadata.from_verifiable_credential(vc, pub_from_priv_jwk(priv))
|
|
92
|
+
|
|
93
|
+
# interactive: attribute autocomplete + rich Jupyter display
|
|
94
|
+
data.metadata.a.max_force # -> Attribute; tab-completion in Jupyter
|
|
95
|
+
data.metadata # -> _repr_html_ table
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Resulting JSON-LD for `max_force` (excerpt):
|
|
99
|
+
|
|
100
|
+
```json
|
|
101
|
+
{
|
|
102
|
+
"@id": "did:suuid:DataFrame__specimen_01__3003...:sdata",
|
|
103
|
+
"@type": ["sdata:DataFrame", "bfo:BFO_0000004"],
|
|
104
|
+
"name": "specimen_01",
|
|
105
|
+
"sdata:max_force": {
|
|
106
|
+
"@type": ["qudt:Quantity", "bfo:Quality"],
|
|
107
|
+
"value": {"@value": 12.5, "@type": "xsd:double"},
|
|
108
|
+
"unitRef": "unit:KiloN", "symbol": "kN"
|
|
109
|
+
},
|
|
110
|
+
"columns": [{"name": "force", "datatype": "xsd:double"}]
|
|
111
|
+
}
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
The optional semantic backends degrade gracefully to pure Python (no hard
|
|
115
|
+
dependency): `pip install "sdata[rdf]"` (rdflib), `sdata[units]` (pint),
|
|
116
|
+
`sdata[schema]` (jsonschema). Core install needs only `numpy`, `pandas`, `suuid`.
|
|
117
|
+
|
|
118
|
+
## Tabular data (`DataFrame`)
|
|
119
|
+
|
|
120
|
+
`DataFrame` wraps a pandas frame plus per-column metadata and serializes to many
|
|
121
|
+
formats — Parquet, Arrow/Feather, CSV, dict/JSON, JSON-LD, a Frictionless **Data
|
|
122
|
+
Package** and **HDF5** — with the qualifying metadata embedded or written as a
|
|
123
|
+
sidecar.
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
import pandas as pd
|
|
127
|
+
from sdata.sclass.dataframe import DataFrame
|
|
128
|
+
|
|
129
|
+
sdf = DataFrame(df=pd.DataFrame({"weight": [10, 20], "height": [1.5, 1.6]}),
|
|
130
|
+
name="specimen_01", description="a tension test")
|
|
131
|
+
|
|
132
|
+
# per-column annotations (only the fields you pass are changed)
|
|
133
|
+
sdf.set_column("weight", unit="kg", label="Gewicht", ontology="bfo:Quality")
|
|
134
|
+
sdf.col["height"].unit = "m" # mutate the column Attribute in place
|
|
135
|
+
sdf.column_units # {'weight': 'kg', 'height': 'm'}
|
|
136
|
+
|
|
137
|
+
# serialize (optional <sname>.meta.jsonld sidecar; bytes/str without a path)
|
|
138
|
+
sdf.to_parquet(path="out", sidecar=True) # out/<sname>.spq + sidecar
|
|
139
|
+
sdf.to_csv(path="out") # data-only CSV (pure pandas)
|
|
140
|
+
sdf.to_feather(path="out") # Arrow IPC + native per-column field metadata
|
|
141
|
+
sdf.to_datapackage(path="out") # Frictionless Data Package (.zip)
|
|
142
|
+
sdf.to_hdf(path="out") # HDF5 via h5py, one dataset/column (sdata[hdf])
|
|
143
|
+
DataFrame.from_parquet("out/specimen_01.spq")
|
|
144
|
+
|
|
145
|
+
# validate the table against a schema (missing/dtype/unit/extra columns)
|
|
146
|
+
from sdata.schema import TableSchema, AttrSpec
|
|
147
|
+
schema = TableSchema("TensileTable", [
|
|
148
|
+
AttrSpec("weight", dtype="int", unit="kg", required=True),
|
|
149
|
+
])
|
|
150
|
+
report = sdf.validate_table(schema) # ValidationReport (truthy if ok)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
Arrow/Feather/Parquet need `pip install "sdata[parquet]"`, HDF5 `sdata[hdf]`; CSV,
|
|
154
|
+
dict, JSON-LD and the (CSV) Data Package work with the core install. See the
|
|
155
|
+
[Tabular data docs](https://lepy.github.io/sdata/usage/dataframe/)
|
|
156
|
+
([`docs/usage/dataframe.md`](docs/usage/dataframe.md)) for the full reference.
|
|
157
|
+
|
|
158
|
+
## Howto
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
* [Das sdata-Format - slides](https://lepy.github.io/sdata/ipynb/Das_sdata_Format.slides.html#)
|
|
162
|
+
* https://deepwiki.com/lepy/sdata
|
|
163
|
+
|
|
164
|
+
## Demo App
|
|
165
|
+
|
|
166
|
+
* [test the demo app with editor](https://share.streamlit.io/lepy/sdata_streamlit/main/sdata_editor.py)
|
|
167
|
+
|
|
168
|
+
Try to paste some Excel-Data in the forms ...
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
## Metadata
|
|
172
|
+
|
|
173
|
+
### Attributes
|
|
174
|
+
|
|
175
|
+
* name
|
|
176
|
+
* value
|
|
177
|
+
* dtype
|
|
178
|
+
* unit
|
|
179
|
+
* description
|
|
180
|
+
* label
|
|
181
|
+
* required
|
|
182
|
+
* ontology (CURIE/IRI of the value's class, e.g. `bfo:Quality`)
|
|
183
|
+
|
|
184
|
+
### dtypes for attributes
|
|
185
|
+
|
|
186
|
+
Every attribute value is coerced to a declared dtype (single source:
|
|
187
|
+
`sdata.dtypes`), each with a lossless JSON-LD / XSD mapping:
|
|
188
|
+
|
|
189
|
+
* `int`, `float`, `str`, `bool`
|
|
190
|
+
* `list` (list of strings), `floatlist` (typed list of floats)
|
|
191
|
+
* `timestamp` (ISO-8601 with timezone, stdlib `zoneinfo`)
|
|
192
|
+
* `date`, `time`, `duration` (ISO-8601 / `datetime` types)
|
|
193
|
+
* `decimal` (exact decimal), `complex` (complex number)
|
|
194
|
+
* `bytes` (base64 in JSON), `json` (dict/list), `uri`
|
|
195
|
+
* `langstring` (language-tagged string, `rdf:langString`, e.g. `"Hallo@de"`)
|
|
196
|
+
|
|
197
|
+
Coercion is **lenient** by default (invalid values are logged and left unchanged).
|
|
198
|
+
Pass `strict=True` (e.g. `metadata.add(..., strict=True)`) to raise `DtypeError`
|
|
199
|
+
instead. See the [conventions](https://lepy.github.io/sdata/conventions/) for the
|
|
200
|
+
error policy.
|
|
201
|
+
|
|
202
|
+
## paper
|
|
203
|
+
|
|
204
|
+
* [Das sdata-Format](https://zenodo.org/record/4311323#.X89yo9-YXys)
|
|
205
|
+
* Ingolf Lepenies. (2020). Das sdata-Format (Version 0.8.4). http://doi.org/10.5281/zenodo.4311323
|
|
206
|
+
* [slides](https://lepy.github.io/sdata/ipynb/Das_sdata_Format.slides.html#),
|
|
207
|
+
[html](https://lepy.github.io/sdata/paper/2020/Das_sdata-Format.html),
|
|
208
|
+
[pdf](https://lepy.github.io/sdata/paper/2020/Das_sdata-Format.pdf)
|
|
209
|
+
[temperaturmessung-001.json](https://lepy.github.io/sdata/paper/2020/temperaturmessung-001.json)
|
|
210
|
+
[temperaturmessung-001.xlsx](https://lepy.github.io/sdata/paper/2020/temperaturmessung-001.xlsx)
|
|
211
|
+
|
|
212
|
+
* [sdata](https://doi.org/10.5281/zenodo.4311396)
|
|
213
|
+
* Ingolf Lepenies. (2020, December 8). sdata - a structured data format (Version 0.8.4). Zenodo. http://doi.org/10.5281/zenodo.4311397
|
|
214
|
+
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=64.0.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "sdata"
|
|
7
|
+
dynamic = ["version"] # Single Source: sdata/__init__.py:__version__
|
|
8
|
+
description = "a structured data format"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" } # RFC 0008 A7 / RFC 0013
|
|
11
|
+
requires-python = ">=3.9"
|
|
12
|
+
authors = [{ name = "Ingolf Lepenies", email = "lepy@tuta.io" }]
|
|
13
|
+
maintainers = [{ name = "Ingolf Lepenies", email = "lepy@tuta.io" }]
|
|
14
|
+
keywords = ["open data"]
|
|
15
|
+
# Schlanker Kern: nur numpy/pandas (Datenmodell) und suuid. Zeitzonen über das
|
|
16
|
+
# stdlib-Modul zoneinfo (Python >= 3.9).
|
|
17
|
+
dependencies = ["numpy", "pandas", "suuid>=0.2.0"]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 3 - Alpha",
|
|
20
|
+
"Intended Audience :: Developers",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Natural Language :: English",
|
|
23
|
+
"Operating System :: OS Independent",
|
|
24
|
+
"Programming Language :: Python :: 3.9",
|
|
25
|
+
"Programming Language :: Python :: 3.10",
|
|
26
|
+
"Programming Language :: Python :: 3.11",
|
|
27
|
+
"Programming Language :: Python :: 3.12",
|
|
28
|
+
"Programming Language :: Python :: Implementation :: CPython",
|
|
29
|
+
"Programming Language :: Python :: Implementation :: PyPy",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
[project.urls]
|
|
33
|
+
Homepage = "https://github.com/lepy/sdata"
|
|
34
|
+
Documentation = "https://lepy.github.io/sdata/"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
# sdata.did ist abhängigkeitsfrei (Ed25519 + base58btc als pure Python);
|
|
38
|
+
# das Extra bleibt als no-op erhalten, damit 'pip install sdata[did]' weiter geht.
|
|
39
|
+
did = []
|
|
40
|
+
http = ["requests"] # sonst urllib-Fallback (stdlib)
|
|
41
|
+
excel = ["openpyxl", "xlsxwriter", "tabulate"]
|
|
42
|
+
hdf = ["h5py"] # sdata.sclass.DataFrame (natives HDF5, RFC 0002)
|
|
43
|
+
sql = ["sqlalchemy"]
|
|
44
|
+
parquet = ["pyarrow"] # sdata.sclass.DataFrame (Parquet)
|
|
45
|
+
blob = ["fsspec"] # sdata.sclass.Blob (URI-Content: file/S3/Zip)
|
|
46
|
+
units = ["pint"] # sonst kuratierte QUDT/UCUM-Tabelle
|
|
47
|
+
rdf = ["rdflib"] # sonst JSON-LD (= gültiges RDF)
|
|
48
|
+
schema = ["jsonschema"] # sonst native MetadataSchema-Prüfung
|
|
49
|
+
|
|
50
|
+
[tool.setuptools.dynamic]
|
|
51
|
+
version = { attr = "sdata.__version__" }
|
|
52
|
+
|
|
53
|
+
[tool.setuptools.packages.find]
|
|
54
|
+
include = ["sdata*"]
|
|
55
|
+
|
|
56
|
+
[tool.pytest.ini_options]
|
|
57
|
+
# Nur die eigene Testsuite kollektieren; vendored contrib-Tests (z. B.
|
|
58
|
+
# sdata/contrib/ecdsa/test_*.py) bewusst ausschliessen.
|
|
59
|
+
testpaths = ["tests"]
|
|
60
|
+
|
|
61
|
+
[tool.coverage.run]
|
|
62
|
+
source = ["sdata"]
|
|
63
|
+
omit = [
|
|
64
|
+
# Vendored Third-Party-Libs (nicht Teil der eigenen Coverage)
|
|
65
|
+
"sdata/contrib/*",
|
|
66
|
+
# Test-/Hilfsdateien
|
|
67
|
+
"*/tests/*",
|
|
68
|
+
"sdata/test/*",
|
|
69
|
+
# Deprecated / experimentell / WIP / Netzwerk — bewusst ausgeblendet
|
|
70
|
+
"sdata/deprecated/*", # Data (ersetzt durch sdata.sclass.dataframe.DataFrame)
|
|
71
|
+
"sdata/experiments/*",
|
|
72
|
+
"*_old.py",
|
|
73
|
+
"*_deprecated.py",
|
|
74
|
+
"sdata/sclass/bfo_classes.py",
|
|
75
|
+
"sdata/sclass/process.py",
|
|
76
|
+
"sdata/sclass/process_graph.py",
|
|
77
|
+
"sdata/sclass/image.py", # Image(Blob); optionales Pillow-Backend (nicht als CI-Dep deklariert)
|
|
78
|
+
]
|
|
79
|
+
|
|
80
|
+
[tool.coverage.report]
|
|
81
|
+
show_missing = true
|
|
82
|
+
exclude_also = [
|
|
83
|
+
"pragma: no cover",
|
|
84
|
+
"if __name__ == .__main__.:",
|
|
85
|
+
"raise NotImplementedError",
|
|
86
|
+
"if TYPE_CHECKING:",
|
|
87
|
+
"@(abc\\.)?abstractmethod",
|
|
88
|
+
"\\.\\.\\.",
|
|
89
|
+
]
|