prob-conf-mat 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. prob_conf_mat-0.1.0/.github/dependabot.yaml +14 -0
  2. prob_conf_mat-0.1.0/.github/workflows/documentation.yaml +64 -0
  3. prob_conf_mat-0.1.0/.github/workflows/pypi-publish.yaml +39 -0
  4. prob_conf_mat-0.1.0/.github/workflows/test.yaml +47 -0
  5. prob_conf_mat-0.1.0/.gitignore +149 -0
  6. prob_conf_mat-0.1.0/.pre-commit-config.yaml +33 -0
  7. prob_conf_mat-0.1.0/CITATION.cff +26 -0
  8. prob_conf_mat-0.1.0/CNAME +1 -0
  9. prob_conf_mat-0.1.0/LICENSE +21 -0
  10. prob_conf_mat-0.1.0/Makefile +81 -0
  11. prob_conf_mat-0.1.0/PKG-INFO +318 -0
  12. prob_conf_mat-0.1.0/README.md +269 -0
  13. prob_conf_mat-0.1.0/documentation/Explanation/experiment_aggregation.md +71 -0
  14. prob_conf_mat-0.1.0/documentation/Explanation/generating_confusion_matrices.md +162 -0
  15. prob_conf_mat-0.1.0/documentation/Explanation/inference.md +99 -0
  16. prob_conf_mat-0.1.0/documentation/Explanation/metric_computation.md +48 -0
  17. prob_conf_mat-0.1.0/documentation/Explanation/totsch_replication.ipynb +1531 -0
  18. prob_conf_mat-0.1.0/documentation/Getting Started/01_estimating_uncertainty.ipynb +514 -0
  19. prob_conf_mat-0.1.0/documentation/Getting Started/02_comparing_experiments.ipynb +394 -0
  20. prob_conf_mat-0.1.0/documentation/Getting Started/03_aggregating_experiments.ipynb +741 -0
  21. prob_conf_mat-0.1.0/documentation/Getting Started/04_loading_and_saving_to_disk.ipynb +498 -0
  22. prob_conf_mat-0.1.0/documentation/Getting Started/index.md +54 -0
  23. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/mlp_0.csv +10 -0
  24. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/mlp_1.csv +10 -0
  25. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/mlp_2.csv +10 -0
  26. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/mlp_3.csv +10 -0
  27. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/mlp_4.csv +10 -0
  28. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/svm_0.csv +10 -0
  29. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/svm_1.csv +10 -0
  30. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/svm_2.csv +10 -0
  31. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/svm_3.csv +10 -0
  32. prob_conf_mat-0.1.0/documentation/Getting Started/mnist_digits/svm_4.csv +10 -0
  33. prob_conf_mat-0.1.0/documentation/How To Guides/choosing_a_prior.md +50 -0
  34. prob_conf_mat-0.1.0/documentation/How To Guides/comparing_many_experiments.md +71 -0
  35. prob_conf_mat-0.1.0/documentation/How To Guides/configuration.md +123 -0
  36. prob_conf_mat-0.1.0/documentation/How To Guides/extending_the_library.md +256 -0
  37. prob_conf_mat-0.1.0/documentation/How To Guides/metric_syntax.md +90 -0
  38. prob_conf_mat-0.1.0/documentation/Reference/Averaging.md +123 -0
  39. prob_conf_mat-0.1.0/documentation/Reference/Experiment Aggregation/heterogeneity.md +17 -0
  40. prob_conf_mat-0.1.0/documentation/Reference/Experiment Aggregation/index.md +64 -0
  41. prob_conf_mat-0.1.0/documentation/Reference/Experiment.md +26 -0
  42. prob_conf_mat-0.1.0/documentation/Reference/ExperimentGroup.md +19 -0
  43. prob_conf_mat-0.1.0/documentation/Reference/IO.md +41 -0
  44. prob_conf_mat-0.1.0/documentation/Reference/Metrics.md +649 -0
  45. prob_conf_mat-0.1.0/documentation/Reference/Statistics.md +108 -0
  46. prob_conf_mat-0.1.0/documentation/Reference/Study.md +194 -0
  47. prob_conf_mat-0.1.0/documentation/assets/css/main.css +26 -0
  48. prob_conf_mat-0.1.0/documentation/assets/figures/binary_confusion_matrix.png +0 -0
  49. prob_conf_mat-0.1.0/documentation/assets/figures/binary_confusion_matrix.svg +264 -0
  50. prob_conf_mat-0.1.0/documentation/assets/figures/central_tendency_and_uncertainty.svg +4228 -0
  51. prob_conf_mat-0.1.0/documentation/assets/figures/example_rope.svg +3261 -0
  52. prob_conf_mat-0.1.0/documentation/assets/figures/examples/aggregation_with_heterogeneity_fe.svg +2529 -0
  53. prob_conf_mat-0.1.0/documentation/assets/figures/examples/aggregation_with_heterogeneity_histogram.svg +2554 -0
  54. prob_conf_mat-0.1.0/documentation/assets/figures/examples/averaged_binary_metric_computation.svg +550 -0
  55. prob_conf_mat-0.1.0/documentation/assets/figures/examples/computation_schedule.svg +304 -0
  56. prob_conf_mat-0.1.0/documentation/assets/figures/examples/plot_experiment_aggregation.svg +2795 -0
  57. prob_conf_mat-0.1.0/documentation/assets/figures/examples/plot_forest_plot.svg +1677 -0
  58. prob_conf_mat-0.1.0/documentation/assets/figures/examples/plot_metric_summaries.png +0 -0
  59. prob_conf_mat-0.1.0/documentation/assets/figures/examples/plot_metric_summaries.svg +1110 -0
  60. prob_conf_mat-0.1.0/documentation/assets/figures/examples/plot_metric_summaries_annotated.svg +796 -0
  61. prob_conf_mat-0.1.0/documentation/assets/figures/examples/plot_pairwise_comparison.svg +3245 -0
  62. prob_conf_mat-0.1.0/documentation/assets/figures/examples/plot_pairwise_comparison_annotated.svg +851 -0
  63. prob_conf_mat-0.1.0/documentation/assets/figures/multiclass_confusion_matrix.png +0 -0
  64. prob_conf_mat-0.1.0/documentation/assets/figures/multiclass_confusion_matrix.svg +677 -0
  65. prob_conf_mat-0.1.0/documentation/assets/figures/overview.svg +1746 -0
  66. prob_conf_mat-0.1.0/documentation/assets/figures/priors/by_count.png +0 -0
  67. prob_conf_mat-0.1.0/documentation/assets/figures/priors/by_dimension.png +0 -0
  68. prob_conf_mat-0.1.0/documentation/assets/figures/random_vs_non_random.svg +1700 -0
  69. prob_conf_mat-0.1.0/documentation/assets/figures/readme/comparison_plot.svg +814 -0
  70. prob_conf_mat-0.1.0/documentation/assets/figures/readme/forest_plot.svg +1663 -0
  71. prob_conf_mat-0.1.0/documentation/assets/figures/readme/uncertainty_fig.svg +515 -0
  72. prob_conf_mat-0.1.0/documentation/assets/figures/replication/totsch_fig_6.svg +4193 -0
  73. prob_conf_mat-0.1.0/documentation/assets/katex.js +12 -0
  74. prob_conf_mat-0.1.0/documentation/assets/logo.svg +56 -0
  75. prob_conf_mat-0.1.0/documentation/assets/logo_rectangle_dark_text.svg +79 -0
  76. prob_conf_mat-0.1.0/documentation/assets/logo_rectangle_light_text.svg +68 -0
  77. prob_conf_mat-0.1.0/documentation/assets/logo_square.svg +68 -0
  78. prob_conf_mat-0.1.0/documentation/assets/mathjax.js +19 -0
  79. prob_conf_mat-0.1.0/documentation/assets/social-preview.png +0 -0
  80. prob_conf_mat-0.1.0/documentation/assets/social-preview.svg +144 -0
  81. prob_conf_mat-0.1.0/documentation/index.md +238 -0
  82. prob_conf_mat-0.1.0/documentation/templates/averaging.md +20 -0
  83. prob_conf_mat-0.1.0/documentation/templates/experiment_aggregation.md +45 -0
  84. prob_conf_mat-0.1.0/documentation/templates/metrics.md +19 -0
  85. prob_conf_mat-0.1.0/documentation/templates/metrics_index.md +48 -0
  86. prob_conf_mat-0.1.0/documentation/templates/mkdocs.py +357 -0
  87. prob_conf_mat-0.1.0/documentation_overrides/.icons/custom/logo-rectangle.svg +56 -0
  88. prob_conf_mat-0.1.0/documentation_overrides/main.html +23 -0
  89. prob_conf_mat-0.1.0/documentation_overrides/partials/integrations/analytics/custom.html +5 -0
  90. prob_conf_mat-0.1.0/documentation_overrides/partials/logo.html +2 -0
  91. prob_conf_mat-0.1.0/mkdocs.yaml +277 -0
  92. prob_conf_mat-0.1.0/pyproject.toml +194 -0
  93. prob_conf_mat-0.1.0/requirements.txt +417 -0
  94. prob_conf_mat-0.1.0/src/prob_conf_mat/__init__.py +9 -0
  95. prob_conf_mat-0.1.0/src/prob_conf_mat/config.py +836 -0
  96. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment.py +438 -0
  97. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment_aggregation/__init__.py +4 -0
  98. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment_aggregation/abc.py +199 -0
  99. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment_aggregation/aggregators.py +558 -0
  100. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment_aggregation/heterogeneity.py +275 -0
  101. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment_aggregation/interface.py +34 -0
  102. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment_comparison/__init__.py +2 -0
  103. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment_comparison/listwise.py +101 -0
  104. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment_comparison/pairwise.py +317 -0
  105. prob_conf_mat-0.1.0/src/prob_conf_mat/experiment_group.py +274 -0
  106. prob_conf_mat-0.1.0/src/prob_conf_mat/io.py +266 -0
  107. prob_conf_mat-0.1.0/src/prob_conf_mat/metrics/__init__.py +38 -0
  108. prob_conf_mat-0.1.0/src/prob_conf_mat/metrics/_metrics.py +1252 -0
  109. prob_conf_mat-0.1.0/src/prob_conf_mat/metrics/abc.py +501 -0
  110. prob_conf_mat-0.1.0/src/prob_conf_mat/metrics/averaging.py +126 -0
  111. prob_conf_mat-0.1.0/src/prob_conf_mat/metrics/collection.py +176 -0
  112. prob_conf_mat-0.1.0/src/prob_conf_mat/metrics/experimental_metrics/__init__.py +1 -0
  113. prob_conf_mat-0.1.0/src/prob_conf_mat/metrics/experimental_metrics/f1_gain.py +112 -0
  114. prob_conf_mat-0.1.0/src/prob_conf_mat/metrics/interface.py +147 -0
  115. prob_conf_mat-0.1.0/src/prob_conf_mat/py.typed +0 -0
  116. prob_conf_mat-0.1.0/src/prob_conf_mat/stats/__init__.py +15 -0
  117. prob_conf_mat-0.1.0/src/prob_conf_mat/stats/batched_averaging.py +48 -0
  118. prob_conf_mat-0.1.0/src/prob_conf_mat/stats/dirichlet_distribution.py +115 -0
  119. prob_conf_mat-0.1.0/src/prob_conf_mat/stats/hdi_estimation.py +51 -0
  120. prob_conf_mat-0.1.0/src/prob_conf_mat/stats/kde.py +82 -0
  121. prob_conf_mat-0.1.0/src/prob_conf_mat/stats/mode_estimation.py +40 -0
  122. prob_conf_mat-0.1.0/src/prob_conf_mat/stats/score_interval.py +25 -0
  123. prob_conf_mat-0.1.0/src/prob_conf_mat/stats/summary.py +93 -0
  124. prob_conf_mat-0.1.0/src/prob_conf_mat/stats/truncated_sampling.py +44 -0
  125. prob_conf_mat-0.1.0/src/prob_conf_mat/study.py +3726 -0
  126. prob_conf_mat-0.1.0/src/prob_conf_mat/utils/__init__.py +7 -0
  127. prob_conf_mat-0.1.0/src/prob_conf_mat/utils/cache.py +142 -0
  128. prob_conf_mat-0.1.0/src/prob_conf_mat/utils/formatting.py +18 -0
  129. prob_conf_mat-0.1.0/src/prob_conf_mat/utils/rng.py +105 -0
  130. prob_conf_mat-0.1.0/src/prob_conf_mat/utils/typing.py +9 -0
  131. prob_conf_mat-0.1.0/tests/__init__.py +0 -0
  132. prob_conf_mat-0.1.0/tests/data/confusion_matrices/sklearn_1.csv +3 -0
  133. prob_conf_mat-0.1.0/tests/data/confusion_matrices/sklearn_2.csv +3 -0
  134. prob_conf_mat-0.1.0/tests/data/confusion_matrices/sklearn_face_classification.csv +7 -0
  135. prob_conf_mat-0.1.0/tests/data/confusion_matrices/sklearn_text_documents.csv +4 -0
  136. prob_conf_mat-0.1.0/tests/data/confusion_matrices/wikipedia_fecal_occult_blood.csv +2 -0
  137. prob_conf_mat-0.1.0/tests/data/malformed_confusion_matrices/malformed_shape.csv +3 -0
  138. prob_conf_mat-0.1.0/tests/data/malformed_confusion_matrices/nan_values.csv +2 -0
  139. prob_conf_mat-0.1.0/tests/data/malformed_confusion_matrices/non_integer_data.csv +2 -0
  140. prob_conf_mat-0.1.0/tests/data/mnist_digits/mlp_0.csv +10 -0
  141. prob_conf_mat-0.1.0/tests/data/mnist_digits/mlp_1.csv +10 -0
  142. prob_conf_mat-0.1.0/tests/data/mnist_digits/mlp_2.csv +10 -0
  143. prob_conf_mat-0.1.0/tests/data/mnist_digits/mlp_3.csv +10 -0
  144. prob_conf_mat-0.1.0/tests/data/mnist_digits/mlp_4.csv +10 -0
  145. prob_conf_mat-0.1.0/tests/data/mnist_digits/svm_0.csv +10 -0
  146. prob_conf_mat-0.1.0/tests/data/mnist_digits/svm_1.csv +10 -0
  147. prob_conf_mat-0.1.0/tests/data/mnist_digits/svm_2.csv +10 -0
  148. prob_conf_mat-0.1.0/tests/data/mnist_digits/svm_3.csv +10 -0
  149. prob_conf_mat-0.1.0/tests/data/mnist_digits/svm_4.csv +10 -0
  150. prob_conf_mat-0.1.0/tests/profile/basic_example.py +61 -0
  151. prob_conf_mat-0.1.0/tests/profile/dirichlet_distribution.py +164 -0
  152. prob_conf_mat-0.1.0/tests/test_config.py +618 -0
  153. prob_conf_mat-0.1.0/tests/test_dirichlet.py +131 -0
  154. prob_conf_mat-0.1.0/tests/test_experiment_aggregation.py +289 -0
  155. prob_conf_mat-0.1.0/tests/test_extension.py +130 -0
  156. prob_conf_mat-0.1.0/tests/test_io.py +146 -0
  157. prob_conf_mat-0.1.0/tests/test_listwise_comparison.py +87 -0
  158. prob_conf_mat-0.1.0/tests/test_metric_equivalence.py +183 -0
  159. prob_conf_mat-0.1.0/tests/test_metrics.py +171 -0
  160. prob_conf_mat-0.1.0/tests/test_notebooks.py +35 -0
  161. prob_conf_mat-0.1.0/tests/test_reporting_example.py +187 -0
  162. prob_conf_mat-0.1.0/tests/test_rng.py +191 -0
  163. prob_conf_mat-0.1.0/tests/test_study.py +225 -0
  164. prob_conf_mat-0.1.0/uv.lock +2638 -0
@@ -0,0 +1,14 @@
1
+ version: 2
2
+ updates:
3
+ # Enable version updates for GitHub Actions
4
+ - package-ecosystem: "github-actions"
5
+ # Workflow files stored in the default location of `.github/workflows`
6
+ # You don't need to specify `/.github/workflows` for `directory`. You can use `directory: "/"`.
7
+ directory: "/"
8
+ schedule:
9
+ interval: "weekly"
10
+
11
+ - package-ecosystem: "uv"
12
+ directory: "/"
13
+ schedule:
14
+ interval: "weekly"
@@ -0,0 +1,64 @@
1
+ # Taken from:
2
+ # https://thomasthornton.cloud/2024/05/01/deploying-mkdocs-to-github-pages-with-github-actions/
3
+ name: Documentation
4
+
5
+ on:
6
+ push:
7
+ branches:
8
+ - main
9
+
10
+ permissions:
11
+ contents: write
12
+ id-token: write
13
+ pages: write
14
+
15
+ jobs:
16
+ build:
17
+
18
+ runs-on: ubuntu-latest
19
+
20
+ steps:
21
+ - name: Checkout
22
+ uses: actions/checkout@v4
23
+ with:
24
+ fetch-depth: 0
25
+
26
+ - name: Install uv
27
+ uses: astral-sh/setup-uv@v6
28
+ with:
29
+ python-version: "3.11"
30
+ enable-cache: true
31
+ cache-dependency-glob: "uv.lock"
32
+
33
+ - name: Install the project
34
+ run: uv sync --locked --all-extras --dev
35
+
36
+ - name: Compile documentation templates
37
+ run: uv run --dev ./documentation/templates/mkdocs.py
38
+
39
+ - name: Build documentation
40
+ run: uv run --dev mkdocs gh-deploy --clean --site-dir "./site" --force
41
+
42
+ deploy:
43
+ needs: build
44
+ environment:
45
+ name: github-pages
46
+ url: ${{ steps.deployment.outputs.page_url }}
47
+ runs-on: ubuntu-latest
48
+ steps:
49
+ - name: Checkout
50
+ uses: actions/checkout@v4
51
+ with:
52
+ ref: gh-pages
53
+
54
+ - name: Setup Pages
55
+ uses: actions/configure-pages@v5
56
+
57
+ - name: Upload artifact
58
+ uses: actions/upload-pages-artifact@v3
59
+ with:
60
+ path: '.'
61
+
62
+ - name: Deploy to GitHub Pages
63
+ id: deployment
64
+ uses: actions/deploy-pages@v4
@@ -0,0 +1,39 @@
1
+ # Github: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
2
+ # UV: https://github.com/astral-sh/trusted-publishing-examples/blob/main/.github/workflows/release.yml
3
+ name: Publish Python Package
4
+
5
+ on:
6
+ release:
7
+ types: [published]
8
+
9
+ permissions:
10
+ contents: read
11
+
12
+ jobs:
13
+ pypi:
14
+ name: Publish to PyPI
15
+ runs-on: ubuntu-latest
16
+ permissions:
17
+ # IMPORTANT: this permission is mandatory for trusted publishing
18
+ id-token: write
19
+
20
+ # Dedicated environments with protections for publishing are strongly recommended.
21
+ # For more information, see: https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#deployment-protection-rules
22
+ environment:
23
+ name: pypi
24
+ url: https://pypi.org/project/prob-conf-mat/
25
+
26
+ steps:
27
+ - name: Checkout
28
+ uses: actions/checkout@v4
29
+
30
+ - name: Install uv
31
+ uses: astral-sh/setup-uv@v6
32
+ with:
33
+ enable-cache: true
34
+
35
+ - name: Build release distributions
36
+ run: uv build
37
+
38
+ - name: Publish release distributions
39
+ run: uv publish --trusted-publishing always
@@ -0,0 +1,47 @@
1
+ name: Test
2
+
3
+ on:
4
+ push:
5
+ branches:
6
+ - main
7
+ pull_request:
8
+ branches:
9
+ - main
10
+
11
+ permissions:
12
+ contents: read
13
+
14
+ jobs:
15
+ test:
16
+
17
+ runs-on: ubuntu-latest
18
+ strategy:
19
+ matrix:
20
+ python-version:
21
+ - "3.11"
22
+ - "3.12"
23
+ - "3.13"
24
+
25
+ steps:
26
+ - uses: actions/checkout@v4
27
+ with:
28
+ fetch-depth: 2
29
+
30
+ - name: Install uv
31
+ uses: astral-sh/setup-uv@v6
32
+ with:
33
+ python-version: ${{ matrix.python-version }}
34
+ enable-cache: true
35
+ cache-dependency-glob: "uv.lock"
36
+
37
+ - name: Install the project
38
+ run: uv sync --locked --all-extras --dev
39
+
40
+ - name: Run tests
41
+ run: uv run --dev pytest --cov --cov-branch --cov-report=xml
42
+
43
+ - name: Upload results to Codecov
44
+ uses: codecov/codecov-action@v5
45
+ with:
46
+ token: ${{ secrets.CODECOV_TOKEN }}
47
+ slug: ioverho/prob_conf_mat
@@ -0,0 +1,149 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[cod]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ pip-wheel-metadata/
24
+ share/python-wheels/
25
+ *.egg-info/
26
+ .installed.cfg
27
+ *.egg
28
+ MANIFEST
29
+
30
+ # PyInstaller
31
+ # Usually these files are written by a python script from a template
32
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
33
+ *.manifest
34
+ *.spec
35
+
36
+ # Installer logs
37
+ pip-log.txt
38
+ pip-delete-this-directory.txt
39
+
40
+ # Unit test / coverage reports
41
+ htmlcov/
42
+ .tox/
43
+ .nox/
44
+ .coverage
45
+ .coverage.*
46
+ .cache
47
+ nosetests.xml
48
+ coverage.xml
49
+ *.cover
50
+ *.py,cover
51
+ .hypothesis/
52
+ .pytest_cache/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ target/
76
+
77
+ # Jupyter Notebook
78
+ .ipynb_checkpoints
79
+
80
+ # IPython
81
+ profile_default/
82
+ ipython_config.py
83
+
84
+ # pyenv
85
+ .python-version
86
+
87
+ # pipenv
88
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
89
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
90
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
91
+ # install all needed dependencies.
92
+ #Pipfile.lock
93
+
94
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow
95
+ __pypackages__/
96
+
97
+ # Celery stuff
98
+ celerybeat-schedule
99
+ celerybeat.pid
100
+
101
+ # SageMath parsed files
102
+ *.sage.py
103
+
104
+ # Environments
105
+ .env
106
+ .venv
107
+ env/
108
+ venv/
109
+ ENV/
110
+ env.bak/
111
+ venv.bak/
112
+
113
+ # Spyder project settings
114
+ .spyderproject
115
+ .spyproject
116
+
117
+ # Rope project settings
118
+ .ropeproject
119
+
120
+ # mkdocs documentation
121
+ /site
122
+
123
+ # mypy
124
+ .mypy_cache/
125
+ .dmypy.json
126
+ dmypy.json
127
+
128
+ # Pyre type checker
129
+ .pyre/
130
+
131
+ # Ivo stuff
132
+ */klad*/
133
+ klad*.*
134
+ .vscode/
135
+ logs/
136
+ misc/
137
+ results/
138
+ dev/
139
+ deprecated_src/
140
+ viz/
141
+ deprecated/
142
+ .cache
143
+ *.tuna
144
+ deprecated_report/# pixi environments
145
+ .pixi
146
+ *.egg-info
147
+ /klad/
148
+ /deprecated/
149
+ .ruff_cache
@@ -0,0 +1,33 @@
1
+ # See https://pre-commit.com for more information
2
+ # See https://pre-commit.com/hooks.html for more hooks
3
+ repos:
4
+ - repo: https://github.com/astral-sh/ruff-pre-commit
5
+ # Ruff version.
6
+ rev: v0.12.4
7
+ hooks:
8
+ # Run the linter.
9
+ - id: ruff-check
10
+ args: [ --fix ]
11
+ - repo: https://github.com/pre-commit/pre-commit-hooks
12
+ rev: v5.0.0
13
+ hooks:
14
+ - id: trailing-whitespace
15
+ - id: end-of-file-fixer
16
+ exclude_types: [html, svg]
17
+ - id: check-yaml
18
+ - id: check-added-large-files
19
+ - id: check-ast
20
+ - id: check-toml
21
+ - id: check-case-conflict
22
+ - id: check-illegal-windows-names
23
+ - id: check-merge-conflict
24
+ - id: name-tests-test
25
+ args: ["--pytest-test-first"]
26
+ - repo: https://github.com/astral-sh/uv-pre-commit
27
+ # uv version.
28
+ rev: 0.7.3
29
+ hooks:
30
+ - id: uv-lock
31
+ args: ["--resolution=highest"]
32
+ - id: uv-export
33
+ args: ["--no-dev", "--frozen", "--format=requirements.txt"]
@@ -0,0 +1,26 @@
1
+ # This CITATION.cff file was generated with cffinit.
2
+ # Visit https://bit.ly/cffinit to generate yours today!
3
+
4
+ cff-version: 1.2.0
5
+ title: prob_conf_mat
6
+ message: >-
7
+ If you use this software, please cite it using the
8
+ metadata from this file.
9
+ type: software
10
+ authors:
11
+ - given-names: Ivo
12
+ family-names: Verhoeven
13
+ email: mail@ivoverhoeven.nl
14
+ affiliation: University of Amsterdam
15
+ orcid: 'https://orcid.org/0000-0002-5163-210X'
16
+ repository-code: 'https://github.com/ioverho/prob_conf_mat'
17
+ abstract: >-
18
+ Confusion matrices with uncertainty quantification,
19
+ experiment aggregation and significance testing.
20
+ keywords:
21
+ - confusion matrices
22
+ - classification
23
+ - confusion matrix
24
+ - statistics
25
+ - probabilistic
26
+ license: MIT
@@ -0,0 +1 @@
1
+ pcm.ivoverhoeven.nl
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2025 Ivo Verhoeven
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,81 @@
1
+ .DEFAULT_GOAL := help
2
+
3
+ ##@ Utility
4
+ .PHONY: help
5
+ help: ## Display this help
6
+ @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make <target>\033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
7
+
8
+ .PHONY: hello-world
9
+ hello-world: ## Tests uv and make
10
+ @uv run python -c "import prob_conf_mat; print('Hello World!')"
11
+
12
+ .PHONY: clean
13
+ clean: ## Clean up caches and build artifacts
14
+ @rm -rf .venv/
15
+ @rm -rf .__pycache__/
16
+ @rm -rf .cache/
17
+ @rm -rf .pytest_cache/
18
+ @rm -rf .ruff_cache/
19
+ @rm -rf build/
20
+ @rm -rf site/
21
+ @rm -rf .coverage
22
+ # I don't trust this enough to actually run it
23
+ # @find . -type f -name '*.py[co]' -delete -or -type d -name __pycache__ -delete
24
+
25
+ ##@ Environment
26
+ .PHONY: install
27
+ install: ## Install default dependencies
28
+ @uv sync --no-dev --frozen
29
+
30
+ .PHONY: install-dev
31
+ install-dev: ## Install dev dependencies
32
+ @uv sync --dev --frozen
33
+
34
+ .PHONY: upgrade
35
+ upgrade: ## Upgrade installed dependencies
36
+ @uv lock --refresh --upgrade
37
+ @uv cache prune
38
+
39
+ .PHONY: export
40
+ export: ## Export uv to requirements.txt file
41
+ @uv export --no-dev --output-file ./requirements.txt --format requirements.txt
42
+
43
+ ##@ Testing, Linting, Typing & Formatting
44
+ .PHONY: test
45
+ test: ## Runs all tests
46
+ @uv run --dev pytest
47
+
48
+ .PHONY: coverage
49
+ coverage: ## Checks test coverage
50
+ @uv run --dev coverage run -m pytest
51
+ @uv run --dev coverage html
52
+
53
+ .PHONY: lint
54
+ lint: ## Run linting
55
+ @uv run --dev ruff check ./src/prob_conf_mat --fix
56
+
57
+ .PHONY: type
58
+ type: ## Run static typechecking
59
+ @mkdir --parents ./tests/logs/pyright
60
+ @uv run --dev pyright > ./tests/logs/pyright/report
61
+
62
+ .PHONY: commit
63
+ commit: ## Run pre-commit checks
64
+ @uv run --dev pre-commit run
65
+
66
+ ##@ Documentation
67
+ .PHONY: docs-build
68
+ docs-build: ## Update the docs
69
+ @uv run --dev python ./documentation/templates/mkdocs.py
70
+ @uv run --dev mkdocs build
71
+
72
+ .PHONY: docs-serve
73
+ docs-serve: ## Serve documentation site
74
+ @uv run --dev python ./documentation/templates/mkdocs.py
75
+ @uv run mkdocs serve --watch "./documentation" --watch "./src/prob_conf_mat"
76
+
77
+ #@ Profiling
78
+ #.PHONY: importtime
79
+ #importtime: ## Profile import time
80
+ # @uv run --no-dev python -X importtime -c "from prob_conf_mat import Study" 2> ./tests/logs/import.log
81
+ # @uv run --dev tuna ./tests/logs/import.log