metasyn 0.6.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. metasyn-0.6.0/.github/workflows/python-package.yml +58 -0
  2. metasyn-0.6.0/.github/workflows/python-publish.yml +39 -0
  3. metasyn-0.6.0/.gitignore +144 -0
  4. metasyn-0.6.0/.readthedocs.yaml +27 -0
  5. metasyn-0.6.0/CITATION.cff +42 -0
  6. metasyn-0.6.0/Dockerfile +21 -0
  7. metasyn-0.6.0/LICENSE +21 -0
  8. metasyn-0.6.0/PKG-INFO +323 -0
  9. metasyn-0.6.0/README.md +253 -0
  10. metasyn-0.6.0/docs/Makefile +20 -0
  11. metasyn-0.6.0/docs/make.bat +35 -0
  12. metasyn-0.6.0/docs/requirements.txt +6 -0
  13. metasyn-0.6.0/docs/source/about/about.rst +12 -0
  14. metasyn-0.6.0/docs/source/about/contact.rst +40 -0
  15. metasyn-0.6.0/docs/source/about/license.rst +38 -0
  16. metasyn-0.6.0/docs/source/about/metasyn_in_detail.rst +288 -0
  17. metasyn-0.6.0/docs/source/api/metasyn.distribution.rst +70 -0
  18. metasyn-0.6.0/docs/source/api/metasyn.rst +68 -0
  19. metasyn-0.6.0/docs/source/api/metasyn.schema.rst +10 -0
  20. metasyn-0.6.0/docs/source/conf.py +62 -0
  21. metasyn-0.6.0/docs/source/developer/contributing.rst +52 -0
  22. metasyn-0.6.0/docs/source/developer/detailed_overview.rst +137 -0
  23. metasyn-0.6.0/docs/source/developer/developer.rst +12 -0
  24. metasyn-0.6.0/docs/source/faq.rst +57 -0
  25. metasyn-0.6.0/docs/source/images/flow_metadata_generation.png +0 -0
  26. metasyn-0.6.0/docs/source/images/flow_metadata_generation_code.png +0 -0
  27. metasyn-0.6.0/docs/source/images/flow_synthetic_data_generation.png +0 -0
  28. metasyn-0.6.0/docs/source/images/flow_synthetic_data_generation_code.png +0 -0
  29. metasyn-0.6.0/docs/source/images/logos/basic-grey.svg +40 -0
  30. metasyn-0.6.0/docs/source/images/logos/basic_grey-med.png +0 -0
  31. metasyn-0.6.0/docs/source/images/logos/basic_grey.png +0 -0
  32. metasyn-0.6.0/docs/source/images/logos/blue.png +0 -0
  33. metasyn-0.6.0/docs/source/images/logos/blue.svg +14 -0
  34. metasyn-0.6.0/docs/source/images/logos/blue_med.png +0 -0
  35. metasyn-0.6.0/docs/source/images/logos/grey.png +0 -0
  36. metasyn-0.6.0/docs/source/images/logos/grey.svg +27 -0
  37. metasyn-0.6.0/docs/source/images/logos/grey_med.png +0 -0
  38. metasyn-0.6.0/docs/source/images/logos/metasyn.afdesign +0 -0
  39. metasyn-0.6.0/docs/source/images/logos/soda.png +0 -0
  40. metasyn-0.6.0/docs/source/images/logos/white.png +0 -0
  41. metasyn-0.6.0/docs/source/images/logos/white.svg +14 -0
  42. metasyn-0.6.0/docs/source/images/logos/white_med.png +0 -0
  43. metasyn-0.6.0/docs/source/images/pipeline.afphoto +0 -0
  44. metasyn-0.6.0/docs/source/images/pipeline_basic.png +0 -0
  45. metasyn-0.6.0/docs/source/images/pipeline_estimation_code.png +0 -0
  46. metasyn-0.6.0/docs/source/images/pipeline_estimation_simple.png +0 -0
  47. metasyn-0.6.0/docs/source/images/pipeline_func_names.png +0 -0
  48. metasyn-0.6.0/docs/source/images/pipeline_generation_code.png +0 -0
  49. metasyn-0.6.0/docs/source/images/pipeline_generation_simple.png +0 -0
  50. metasyn-0.6.0/docs/source/images/pipeline_serialization_code.png +0 -0
  51. metasyn-0.6.0/docs/source/images/pipeline_serialization_simple.png +0 -0
  52. metasyn-0.6.0/docs/source/index.rst +103 -0
  53. metasyn-0.6.0/docs/source/usage/exporting_metaframes.rst +303 -0
  54. metasyn-0.6.0/docs/source/usage/extensions.rst +40 -0
  55. metasyn-0.6.0/docs/source/usage/generating_metaframes.rst +119 -0
  56. metasyn-0.6.0/docs/source/usage/generating_synthetic_data.rst +33 -0
  57. metasyn-0.6.0/docs/source/usage/installation.rst +78 -0
  58. metasyn-0.6.0/docs/source/usage/interactive_tutorials.rst +14 -0
  59. metasyn-0.6.0/docs/source/usage/quick_start.rst +103 -0
  60. metasyn-0.6.0/docs/source/usage/usage.rst +17 -0
  61. metasyn-0.6.0/examples/advanced_tutorial.ipynb +532 -0
  62. metasyn-0.6.0/examples/basic_example.json +99 -0
  63. metasyn-0.6.0/examples/basic_example.py +39 -0
  64. metasyn-0.6.0/examples/demonstration.csv +892 -0
  65. metasyn-0.6.0/examples/descriptions.ipynb +431 -0
  66. metasyn-0.6.0/examples/getting_started.ipynb +352 -0
  67. metasyn-0.6.0/examples/titanic.ipynb +259 -0
  68. metasyn-0.6.0/examples/titanic_example.json +216 -0
  69. metasyn-0.6.0/examples/utils.py +41 -0
  70. metasyn-0.6.0/metasyn/__init__.py +16 -0
  71. metasyn-0.6.0/metasyn/__main__.py +165 -0
  72. metasyn-0.6.0/metasyn/_version.py +8 -0
  73. metasyn-0.6.0/metasyn/demo/__init__.py +5 -0
  74. metasyn-0.6.0/metasyn/demo/dataset.py +85 -0
  75. metasyn-0.6.0/metasyn/demo/demo_titanic.csv +892 -0
  76. metasyn-0.6.0/metasyn/distribution/__init__.py +33 -0
  77. metasyn-0.6.0/metasyn/distribution/base.py +293 -0
  78. metasyn-0.6.0/metasyn/distribution/categorical.py +134 -0
  79. metasyn-0.6.0/metasyn/distribution/continuous.py +223 -0
  80. metasyn-0.6.0/metasyn/distribution/datetime.py +194 -0
  81. metasyn-0.6.0/metasyn/distribution/discrete.py +153 -0
  82. metasyn-0.6.0/metasyn/distribution/faker.py +174 -0
  83. metasyn-0.6.0/metasyn/distribution/legacy/__init__.py +7 -0
  84. metasyn-0.6.0/metasyn/distribution/legacy/regex/__init__.py +6 -0
  85. metasyn-0.6.0/metasyn/distribution/legacy/regex/base.py +220 -0
  86. metasyn-0.6.0/metasyn/distribution/legacy/regex/element.py +517 -0
  87. metasyn-0.6.0/metasyn/distribution/legacy/regex/optimizer.py +288 -0
  88. metasyn-0.6.0/metasyn/distribution/na.py +28 -0
  89. metasyn-0.6.0/metasyn/distribution/regex.py +114 -0
  90. metasyn-0.6.0/metasyn/metaframe.py +316 -0
  91. metasyn-0.6.0/metasyn/privacy.py +54 -0
  92. metasyn-0.6.0/metasyn/provider.py +423 -0
  93. metasyn-0.6.0/metasyn/py.typed +0 -0
  94. metasyn-0.6.0/metasyn/schema/__init__.py +1 -0
  95. metasyn-0.6.0/metasyn/schema/generative_metadata_format.json +283 -0
  96. metasyn-0.6.0/metasyn/testutils.py +71 -0
  97. metasyn-0.6.0/metasyn/validation.py +94 -0
  98. metasyn-0.6.0/metasyn/var.py +289 -0
  99. metasyn-0.6.0/metasyn.egg-info/PKG-INFO +323 -0
  100. metasyn-0.6.0/metasyn.egg-info/SOURCES.txt +115 -0
  101. metasyn-0.6.0/metasyn.egg-info/dependency_links.txt +1 -0
  102. metasyn-0.6.0/metasyn.egg-info/entry_points.txt +5 -0
  103. metasyn-0.6.0/metasyn.egg-info/requires.txt +33 -0
  104. metasyn-0.6.0/metasyn.egg-info/top_level.txt +1 -0
  105. metasyn-0.6.0/pyproject.toml +92 -0
  106. metasyn-0.6.0/setup.cfg +4 -0
  107. metasyn-0.6.0/tests/data/titanic.csv +892 -0
  108. metasyn-0.6.0/tests/test_builtin.py +17 -0
  109. metasyn-0.6.0/tests/test_cli.py +57 -0
  110. metasyn-0.6.0/tests/test_continuous.py +91 -0
  111. metasyn-0.6.0/tests/test_dataset.py +115 -0
  112. metasyn-0.6.0/tests/test_datetime.py +92 -0
  113. metasyn-0.6.0/tests/test_discrete.py +67 -0
  114. metasyn-0.6.0/tests/test_distribution.py +31 -0
  115. metasyn-0.6.0/tests/test_faker.py +15 -0
  116. metasyn-0.6.0/tests/test_na.py +13 -0
  117. metasyn-0.6.0/tests/test_var.py +283 -0
@@ -0,0 +1,58 @@
1
+ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
2
+ # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
3
+
4
+ name: Python package
5
+
6
+ on:
7
+ push:
8
+ branches: [ main ]
9
+ pull_request:
10
+ branches: [ main ]
11
+ schedule:
12
+ - cron: "0 0 * * 0"
13
+
14
+ jobs:
15
+ build:
16
+ strategy:
17
+ fail-fast: false
18
+ matrix:
19
+ os: [ubuntu-latest]
20
+ python-version: [3.8, 3.9, "3.10", "3.11"]
21
+ include:
22
+ - os: macos-latest
23
+ python-version: "3.11"
24
+ - os: windows-latest
25
+ python-version: "3.11"
26
+ runs-on: ${{ matrix.os }}
27
+
28
+ steps:
29
+ - uses: actions/checkout@v4
30
+ - name: Set up Python ${{ matrix.python-version }}
31
+ uses: actions/setup-python@v4
32
+ with:
33
+ python-version: ${{ matrix.python-version }}
34
+ - name: Install dependencies
35
+ run: |
36
+ python -m pip install --upgrade pip
37
+ python -m pip install ".[test]"
38
+ - name: Check pep8 with flake8
39
+ run: |
40
+ flake8 metasyn --max-line-length 100
41
+ - name: Lint with pylint
42
+ run: |
43
+ pylint metasyn
44
+ - name: Check docstrings with pydocstyle
45
+ run: |
46
+ pydocstyle metasyn --convention=numpy --add-select=D417 --add-ignore="D102,D105"
47
+ - name: Check types with MyPy
48
+ run: |
49
+ mypy metasyn
50
+ - name: Check if documentation builds.
51
+ run: |
52
+ cd docs; make html SPHINXOPTS="-W --keep-going"
53
+ - name: Test with pytest
54
+ run: |
55
+ pytest tests
56
+ - name: Check notebook output
57
+ run: |
58
+ pytest --nbval-lax examples
@@ -0,0 +1,39 @@
1
+ # This workflow will upload a Python Package using Twine when a release is created
2
+ # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
3
+
4
+ # This workflow uses actions that are not certified by GitHub.
5
+ # They are provided by a third-party and are governed by
6
+ # separate terms of service, privacy policy, and support
7
+ # documentation.
8
+
9
+ name: Upload Python Package
10
+
11
+ on:
12
+ release:
13
+ types: [published]
14
+
15
+ permissions:
16
+ contents: read
17
+
18
+ jobs:
19
+ deploy:
20
+
21
+ runs-on: ubuntu-latest
22
+
23
+ steps:
24
+ - uses: actions/checkout@v4
25
+ - name: Set up Python
26
+ uses: actions/setup-python@v4
27
+ with:
28
+ python-version: '3.x'
29
+ - name: Install dependencies
30
+ run: |
31
+ python -m pip install --upgrade pip
32
+ pip install build
33
+ - name: Build package
34
+ run: python -m build
35
+ - name: Publish package
36
+ uses: pypa/gh-action-pypi-publish@7eb3b701d11256e583f5b49899c5e7203deab573
37
+ with:
38
+ user: __token__
39
+ password: ${{ secrets.PYPI_API_TOKEN }}
@@ -0,0 +1,144 @@
1
+ # setuptools-scm generated file that can be ignored
2
+ _version.py
3
+
4
+ # Byte-compiled / optimized / DLL files
5
+ __pycache__/
6
+ *.py[cod]
7
+ *$py.class
8
+
9
+ # C extensions
10
+ *.so
11
+
12
+ # Distribution / packaging
13
+ .Python
14
+ build/
15
+ develop-eggs/
16
+ dist/
17
+ downloads/
18
+ eggs/
19
+ .eggs/
20
+ lib/
21
+ lib64/
22
+ parts/
23
+ sdist/
24
+ var/
25
+ wheels/
26
+ pip-wheel-metadata/
27
+ share/python-wheels/
28
+ *.egg-info/
29
+ .installed.cfg
30
+ *.egg
31
+ MANIFEST
32
+
33
+ # PyInstaller
34
+ # Usually these files are written by a python script from a template
35
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
36
+ *.manifest
37
+ *.spec
38
+
39
+ # Installer logs
40
+ pip-log.txt
41
+ pip-delete-this-directory.txt
42
+
43
+ # Unit test / coverage reports
44
+ htmlcov/
45
+ .tox/
46
+ .nox/
47
+ .coverage
48
+ .coverage.*
49
+ .cache
50
+ nosetests.xml
51
+ coverage.xml
52
+ *.cover
53
+ *.py,cover
54
+ .hypothesis/
55
+ .pytest_cache/
56
+
57
+ # Translations
58
+ *.mo
59
+ *.pot
60
+
61
+ # Django stuff:
62
+ *.log
63
+ local_settings.py
64
+ db.sqlite3
65
+ db.sqlite3-journal
66
+
67
+ # Flask stuff:
68
+ instance/
69
+ .webassets-cache
70
+
71
+ # Scrapy stuff:
72
+ .scrapy
73
+
74
+ # Sphinx documentation
75
+ docs/_build/
76
+
77
+ # PyBuilder
78
+ target/
79
+
80
+ # Jupyter Notebook
81
+ .ipynb_checkpoints
82
+
83
+ # IPython
84
+ profile_default/
85
+ ipython_config.py
86
+
87
+ # pyenv
88
+ .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ #Pipfile.lock
96
+
97
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
98
+ __pypackages__/
99
+
100
+ # Celery stuff
101
+ celerybeat-schedule
102
+ celerybeat.pid
103
+
104
+ # SageMath parsed files
105
+ *.sage.py
106
+
107
+ # Environments
108
+ .env
109
+ .venv
110
+ env/
111
+ venv/
112
+ ENV/
113
+ env.bak/
114
+ venv.bak/
115
+
116
+ # Eclipse project settings
117
+ .project
118
+ .pydevproject
119
+
120
+ # Spyder project settings
121
+ .spyderproject
122
+ .spyproject
123
+
124
+ # Rope project settings
125
+ .ropeproject
126
+
127
+ # mkdocs documentation
128
+ /site
129
+
130
+ # mypy
131
+ .mypy_cache/
132
+ .dmypy.json
133
+ dmypy.json
134
+
135
+ # Pyre type checker
136
+ .pyre/
137
+
138
+ # pycharm
139
+ .idea/
140
+ .vscode/
141
+
142
+ #DS_Store
143
+ .DS_Store
144
+ .DS_Store
@@ -0,0 +1,27 @@
1
+ # .readthedocs.yaml
2
+ # Read the Docs configuration file
3
+ # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
4
+
5
+ # Required
6
+ version: 2
7
+
8
+ # Set the version of Python and other tools you might need
9
+ build:
10
+ os: ubuntu-20.04
11
+ tools:
12
+ python: "3.9"
13
+ # You can also specify other tool versions:
14
+ # nodejs: "16"
15
+ # rust: "1.55"
16
+ # golang: "1.17"
17
+
18
+ # Build documentation in the docs/ directory with Sphinx
19
+ sphinx:
20
+ configuration: docs/source/conf.py
21
+
22
+ # Optionally declare the Python requirements required to build your docs
23
+ python:
24
+ install:
25
+ - requirements: docs/requirements.txt
26
+ - method: pip
27
+ path: .
@@ -0,0 +1,42 @@
1
+ cff-version: 1.2.0
2
+ title: metasyn
3
+ message: >-
4
+ If you use this software, please cite it using the
5
+ metadata from this file.
6
+ type: software
7
+ authors:
8
+ - given-names: Raoul
9
+ family-names: Schram
10
+ email: r.d.schram@uu.nl
11
+ affiliation: Utrecht University
12
+ orcid: 'https://orcid.org/0000-0001-6616-230X'
13
+ - given-names: Samuel
14
+ family-names: Spithorst
15
+ email: s.f.spithorst@uu.nl
16
+ orcid: 'https://orcid.org/0009-0000-4140-0658'
17
+ affiliation: Utrecht University
18
+ - given-names: Erik-Jan
19
+ name-particle: van
20
+ family-names: Kesteren
21
+ email: e.vankesteren1@uu.nl
22
+ affiliation: Utrecht University
23
+ orcid: 'https://orcid.org/0000-0003-1548-1663'
24
+ identifiers:
25
+ - type: doi
26
+ value: 10.5281/zenodo.7696031
27
+ description: Latest archived version of Metasynth
28
+ repository-code: 'https://github.com/sodascience/metasyn'
29
+ url: 'https://metasyn.readthedocs.io/'
30
+ abstract: >-
31
+ A Python package for generating synthetic data from
32
+ tabular datasets, with a focus on privacy and
33
+ transparency.
34
+ keywords:
35
+ - synthetic data
36
+ - metadata
37
+ - generative model
38
+ - data science
39
+ - data management
40
+ - machine learning
41
+ - statistics
42
+ license: MIT
@@ -0,0 +1,21 @@
1
+ # use the slim python image
2
+ FROM python:3.11-slim
3
+
4
+ # Install system dependencies
5
+ # git is used by versioneer to define the project version
6
+ RUN apt update && apt install -y git
7
+
8
+ # Install metasyn
9
+ COPY . metasyn/
10
+ RUN pip install metasyn/
11
+
12
+ # Remove metasyn folder
13
+ RUN rm -r metasyn/
14
+
15
+ # For excel output use optional XlsxWriter package
16
+ RUN pip install XlsxWriter
17
+
18
+ # Remove system dependencies
19
+ RUN apt remove -y git && apt autoremove -y
20
+
21
+ ENTRYPOINT [ "metasyn" ]
metasyn-0.6.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2023 SoDa
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
metasyn-0.6.0/PKG-INFO ADDED
@@ -0,0 +1,323 @@
1
+ Metadata-Version: 2.1
2
+ Name: metasyn
3
+ Version: 0.6.0
4
+ Summary: Package for creating synthetic datasets while preserving privacy.
5
+ Author-email: Raoul Schram <r.d.schram@uu.nl>, Erik-Jan van Kesteren <e.vankesteren1@uu.nl>
6
+ License: MIT License
7
+
8
+ Copyright (c) 2023 SoDa
9
+
10
+ Permission is hereby granted, free of charge, to any person obtaining a copy
11
+ of this software and associated documentation files (the "Software"), to deal
12
+ in the Software without restriction, including without limitation the rights
13
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14
+ copies of the Software, and to permit persons to whom the Software is
15
+ furnished to do so, subject to the following conditions:
16
+
17
+ The above copyright notice and this permission notice shall be included in all
18
+ copies or substantial portions of the Software.
19
+
20
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26
+ SOFTWARE.
27
+
28
+ Project-URL: GitHub, https://github.com/sodascience/metasyn
29
+ Project-URL: documentation, https://metasyn.readthedocs.io/en/latest/index.html
30
+ Keywords: metadata,open-data,privacy,synthetic-data,tabular datasets
31
+ Classifier: Programming Language :: Python :: 3
32
+ Classifier: Programming Language :: Python :: 3.8
33
+ Classifier: Programming Language :: Python :: 3.9
34
+ Classifier: Programming Language :: Python :: 3.10
35
+ Classifier: Programming Language :: Python :: 3.11
36
+ Classifier: Development Status :: 3 - Alpha
37
+ Classifier: License :: OSI Approved :: MIT License
38
+ Obsoletes: metasynth
39
+ Requires-Python: >=3.8
40
+ Description-Content-Type: text/markdown
41
+ License-File: LICENSE
42
+ Requires-Dist: pandas
43
+ Requires-Dist: polars>=0.14.17
44
+ Requires-Dist: numpy>=1.20
45
+ Requires-Dist: pyarrow
46
+ Requires-Dist: scipy
47
+ Requires-Dist: numpy>=1.20
48
+ Requires-Dist: faker
49
+ Requires-Dist: lingua-language-detector
50
+ Requires-Dist: scikit-learn
51
+ Requires-Dist: jsonschema
52
+ Requires-Dist: importlib-metadata; python_version < "3.10"
53
+ Requires-Dist: importlib-resources; python_version < "3.9"
54
+ Requires-Dist: wget
55
+ Requires-Dist: regexmodel>=0.2.1
56
+ Provides-Extra: test
57
+ Requires-Dist: pytest; extra == "test"
58
+ Requires-Dist: pylint; extra == "test"
59
+ Requires-Dist: pydocstyle; extra == "test"
60
+ Requires-Dist: mypy; extra == "test"
61
+ Requires-Dist: flake8; extra == "test"
62
+ Requires-Dist: nbval; extra == "test"
63
+ Requires-Dist: sphinx<7.0.0; extra == "test"
64
+ Requires-Dist: sphinx-rtd-theme; extra == "test"
65
+ Requires-Dist: sphinxcontrib-napoleon; extra == "test"
66
+ Requires-Dist: sphinx-autodoc-typehints; extra == "test"
67
+ Requires-Dist: sphinx_inline_tabs; extra == "test"
68
+ Requires-Dist: sphinx_copybutton; extra == "test"
69
+ Requires-Dist: XlsxWriter; extra == "test"
70
+
71
+ ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/metasyn)
72
+ [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/sodascience/metasyn/HEAD?labpath=examples%2Fgetting_started.ipynb)
73
+ [![Google Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/sodascience/metasyn/blob/main/examples/getting_started.ipynb)
74
+ [![docs](https://readthedocs.org/projects/metasyn/badge/?version=latest)](https://metasyn.readthedocs.io/en/latest/index.html)
75
+
76
+ ![Metasyn Logo](docs/source/images/logos/blue.svg)
77
+
78
+ # Metasyn
79
+ Metasyn is a Python package designed to generate tabular synthetic data for rigorous code testing and reproducibility.
80
+ Researchers and data owners can use metasyn to generate and share synthetic versions of their sensitive datasets, mitigating privacy concerns. Additionally, metasyn facilitates transparency and reproducibility, by allowing the underlying MetaFrames to be exported and shared. Other researchers can use these to regenerate consistent synthetic datasets, validating published work without requiring sensitive data.
81
+
82
+ The package has three main functionalities:
83
+
84
+ 1. **Estimation**: Metasyn can create a MetaFrame, from a dataset. A MetaFrame is essentially a fitted model that characterizes the structure of the original dataset without storing actual values. It captures individual distributions and features, enabling generation of synthetic data based on these MetaFrames and can be seen as (statistical) metadata.
85
+ 2. **Serialization**: Metasyn can export a MetaFrame into an easy to read JSON file, allowing users to audit, understand, and modify their data generation model.
86
+ 3. **Generation**: Metasyn can generate synthetic data based on a MetaFrame. The synthetic data produced solely depends on the MetaFrame, thereby maintaining a critical separation between the original sensitive data and the synthetic data generated. The generated synthetic data, emulates the original data's format and plausibility at the individual record level and attempts to reproduce marginal (univariate) distributions where possible. Generated values are based on the observed distributions while adding a degree of variance and smoothing. The generated data does **not** aim to preserve the relationships between variables. The frequency of missing values and their codes are maintained in the synthetically-augmented dataset.
87
+
88
+ ![Metasyn Pipeline](docs/source/images/pipeline_basic.png)
89
+
90
+ ### Key features
91
+ - **MetaFrame Generation**: Metasyn allows the creation of a MetaFrame from a dataset provided as a Polars or Pandas DataFrame.
92
+ MetaFrames includes key characteristics such as *variable names*, *data types*, *the percentage of missing values*, and *distribution parameters*.
93
+ - **Exporting MetaFrames**: Metasyn can export and import MetaFrames to GMF files. These are JSON files that follow the easy to read and understand [Generative Metadata Format (GMF)](https://github.com/sodascience/generative_metadata_format).
94
+
95
+ <details>
96
+ <summary> A simple example of an exported MetaFrame (following the GMF standard): </summary>
97
+
98
+ ```json
99
+ {
100
+ "n_rows": 5,
101
+ "n_columns": 5,
102
+ "provenance": {
103
+ "created by": {
104
+ "name": "Metasyn",
105
+ "version": "0.4.0"
106
+ },
107
+ "creation time": "2023-08-07T12:04:40.669740"
108
+ },
109
+ "vars": [
110
+ {
111
+ "name": "ID",
112
+ "type": "discrete",
113
+ "dtype": "Int64",
114
+ "prop_missing": 0.0,
115
+ "distribution": {
116
+ "implements": "core.unique_key",
117
+ "provenance": "builtin",
118
+ "class_name": "UniqueKeyDistribution",
119
+ "parameters": {
120
+ "low": 1,
121
+ "consecutive": 1
122
+ }
123
+ }
124
+ },
125
+ {
126
+ "name": "fruits",
127
+ "type": "categorical",
128
+ "dtype": "Categorical",
129
+ "prop_missing": 0.0,
130
+ "distribution": {
131
+ "implements": "core.multinoulli",
132
+ "provenance": "builtin",
133
+ "class_name": "MultinoulliDistribution",
134
+ "parameters": {
135
+ "labels": [
136
+ "apple",
137
+ "banana"
138
+ ],
139
+ "probs": [
140
+ 0.4,
141
+ 0.6
142
+ ]
143
+ }
144
+ }
145
+ },
146
+ {
147
+ "name": "B",
148
+ "type": "discrete",
149
+ "dtype": "Int64",
150
+ "prop_missing": 0.0,
151
+ "distribution": {
152
+ "implements": "core.poisson",
153
+ "provenance": "builtin",
154
+ "class_name": "PoissonDistribution",
155
+ "parameters": {
156
+ "mu": 3.0
157
+ }
158
+ }
159
+ },
160
+ {
161
+ "name": "cars",
162
+ "type": "categorical",
163
+ "dtype": "Categorical",
164
+ "prop_missing": 0.0,
165
+ "distribution": {
166
+ "implements": "core.multinoulli",
167
+ "provenance": "builtin",
168
+ "class_name": "MultinoulliDistribution",
169
+ "parameters": {
170
+ "labels": [
171
+ "audi",
172
+ "beetle"
173
+ ],
174
+ "probs": [
175
+ 0.2,
176
+ 0.8
177
+ ]
178
+ }
179
+ }
180
+ },
181
+ {
182
+ "name": "optional",
183
+ "type": "discrete",
184
+ "dtype": "Int64",
185
+ "prop_missing": 0.2,
186
+ "distribution": {
187
+ "implements": "core.discrete_uniform",
188
+ "provenance": "builtin",
189
+ "class_name": "DiscreteUniformDistribution",
190
+ "parameters": {
191
+ "low": -30,
192
+ "high": 301
193
+ }
194
+ }
195
+ }
196
+ ]
197
+ }
198
+ ```
199
+
200
+ A more advanced example GMF, based on the [Titanic](https://raw.githubusercontent.com/pandas-dev/pandas/main/doc/data/titanic.csv) dataset, can be found [here](examples/titanic_example.json)
201
+ </details>
202
+
203
+ - **Synthetic Data Generation**: Metasyn allows for the generation of a polars DataFrame with synthetic data that resembles the original data.
204
+ - **Distribution Fitting**: Metasyn allows for manual and automatic distribution fitting.
205
+ - **Data Type Support**: Metasyn supports generating synthetic data for a variety of common data types including `categorical`, `string`, `integer`, `float`, `date`, `time`, and `datetime`.
206
+ - **Integration with Faker**: Metasyn integrates with the [faker](https://github.com/joke2k/faker) package, a Python library for generating fake data such as names and emails. Allowing for synthetic data that is formatted realistically, while retaining privacy.
207
+ - **Structured String Detection**: Metasyn identifies structured strings within your dataset, which can include formatted text,
208
+ codes, identifiers, or any string that follows a specific pattern.
209
+ - **Handling Unique Values**: Metasyn can identify and process variables with unique values or keys in the data, preserving their uniqueness in the synthetic dataset, which is crucial for generating synthetic data that maintains the characteristics of the original dataset.
210
+
211
+ Curious and want to learn more? Check out our[documentation](https://metasyn.readthedocs.io/en/latest/index.html)!
212
+
213
+ ## Getting Started
214
+ ### Try it out online
215
+ If you're new to Python or simply want to quickly explore the basic features of metasyn, you can try it out using the online Google Colab tutorial. [Click here](https://colab.research.google.com/github/sodascience/metasyn/blob/main/examples/getting_started.ipynb) to access the tutorial. It provides a step-by-step walkthrough and example dataset to help you get started. However, please exercise caution when using sensitive data, as it will be handled through Google servers.
216
+
217
+ ### Local Installation
218
+ For more advanced users and researchers who prefer working on their local machines, you can install metasyn directly from PyPI using the following command in the terminal (not Python):
219
+
220
+ ```sh
221
+ pip install metasyn
222
+ ```
223
+
224
+ ## Usage
225
+ To learn how to use metasyn effectively, refer to the comprehensive [documentation](https://metasyn.readthedocs.io/en/latest/index.html). The documentation covers all the necessary information and provides detailed explanations, examples, and usage guidelines.
226
+
227
+ Additionally, the documentation offers a series of [tutorials](https://metasyn.readthedocs.io/en/latest/usage/interactive_tutorials.html) that delve into specific features and use cases. These tutorials can further assist you in understanding and leveraging the capabilities of metasyn.
228
+
229
+ ### Quick start
230
+ Get started quickly with metasyn using the following example. In this concise demonstration, you'll learn the basic functionality of metasyn by generating synthetic data from [titanic](https://raw.githubusercontent.com/pandas-dev/pandas/main/doc/data/titanic.csv) dataset.
231
+
232
+ It is important to start by importing the appropriate libraries:
233
+
234
+ ```python
235
+ # import libraries
236
+ import polars as pl
237
+ from metasyn import MetaFrame, demo_file
238
+ ```
239
+
240
+ #### Estimation: Generating a MetaFrame
241
+ ##### 1. Begin by creating a polars dataframe:
242
+ ```python
243
+ # import the demo csv
244
+ dataset_csv = demo_file() # This function automatically loads the Titanic dataset (as found here )
245
+
246
+ # create dataframe
247
+ data_types = {
248
+ "Sex": pl.Categorical,
249
+ "Embarked": pl.Categorical,
250
+ "Survived": pl.Categorical,
251
+ "Pclass": pl.Categorical,
252
+ "SibSp": pl.Categorical,
253
+ "Parch": pl.Categorical
254
+ }
255
+
256
+ df = pl.read_csv(dataset_csv, dtypes=data_types)
257
+ ```
258
+
259
+ <details>
260
+ <summary>
261
+ Note on using Pandas
262
+ </summary>
263
+
264
+ Internally, metasyn uses Polars (instead of Pandas) mainly because typing and the handling of non-existing data is more
265
+ consistent. It is possible to supply a Pandas DataFrame instead of a polars DataFrame to `MetaFrame.fit_dataframe`.
266
+ However, this uses the automatic polars conversion functionality, which for some edge cases result in problems. Therefore,
267
+ we advise users to create Polars DataFrames. The resulting synthetic dataset is always a polars dataframe, but this can
268
+ be easily converted back to a Pandas DataFrame by using `df_pandas = df_polars.to_pandas()`.
269
+ </details>
270
+
271
+ ##### 2. Next, we can generate a MetaFrame from the polars DataFrame.
272
+
273
+ ```python
274
+ # create a MetaFrame (mf) from the DataFrame (df)
275
+ mf = MetaFrame.fit_dataframe(df)
276
+ ```
277
+
278
+ > Note: At this point you will encounter a warning about `PassengerId` not being set as unique, you can safely ignore it and proceed. This warning occurs because `PassengerId` appears to contain unique values, but is not explicitly marked as a unique column. To remove the warning, you can set `PassengerId` to be a unique column. Our documentation explains how to do this when generating Metaframes: [Set Columns as Unique](https://metasyn.readthedocs.io/en/latest/usage/generating_metaframes.html#optional-parameters).
279
+
280
+ #### (De)serialization: Exporting and importing a MetaFrame
281
+ _Note that exporting and importing is optional. You can generate synthetic data from **any** loaded MetaFrame, whether that be through importing a GMF file or generating a MetaFrame from an original DataFrame._
282
+
283
+ ##### 3. We can export this MetaFrame to a GMF file using:
284
+
285
+ ```python
286
+ # export MetaFrame
287
+ mf.export("exported_metaframe.json")
288
+ ```
289
+
290
+ ##### 4. Similarly, we can import a MetaFrame from a GMF file using:
291
+ ```python
292
+ # load MetaFrame
293
+ mf = MetaFrame.from_json("exported_metaframe.json")
294
+ ```
295
+
296
+ #### Generation: Generating synthetic data
297
+ ##### 5. Finally, we can generate a DataFrame with synthetic data based on a MetaFrame using:
298
+
299
+ ```python
300
+ # synthesize a DataFrame with 5 rows of data based on a MetaFrame
301
+ synthetic_data = mf.synthesize(5)
302
+ ```
303
+
304
+ <!-- CONTRIBUTING -->
305
+ ## Contributing
306
+ Contributions are what make the open source community an amazing place to learn, inspire, and create.
307
+
308
+ Any contributions you make are greatly appreciated.
309
+
310
+ To contribute:
311
+ 1. Fork the Project
312
+ 2. Create your Feature Branch (`git checkout -b feature/AmazingFeature`)
313
+ 3. Commit your Changes (`git commit -m 'Add some AmazingFeature'`)
314
+ 4. Push to the Branch (`git push origin feature/AmazingFeature`)
315
+ 5. Open a Pull Request
316
+
317
+
318
+ <!-- CONTACT -->
319
+ ## Contact
320
+ **Metasyn** is a project by the [ODISSEI Social Data Science (SoDa)](https://odissei-data.nl/nl/soda/) team.
321
+ Do you have questions, suggestions, or remarks on the technical implementation? File an issue in the issue tracker or feel free to contact [Erik-Jan van Kesteren](https://github.com/vankesteren) or [Raoul Schram](https://github.com/qubixes).
322
+
323
+ <img src="docs/source/images/logos/soda.png" alt="SoDa logo" width="250px"/>