dataspoof-mlpilot 2.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (255) hide show
  1. dataspoof_mlpilot-2.1.0/.dockerignore +11 -0
  2. dataspoof_mlpilot-2.1.0/.github/workflows/CI.yml +271 -0
  3. dataspoof_mlpilot-2.1.0/.github/workflows/docs.yml +47 -0
  4. dataspoof_mlpilot-2.1.0/.gitignore +26 -0
  5. dataspoof_mlpilot-2.1.0/BUILD.md +54 -0
  6. dataspoof_mlpilot-2.1.0/CHANGELOG.md +204 -0
  7. dataspoof_mlpilot-2.1.0/CONTRIBUTING.md +40 -0
  8. dataspoof_mlpilot-2.1.0/Cargo.lock +358 -0
  9. dataspoof_mlpilot-2.1.0/Cargo.toml +26 -0
  10. dataspoof_mlpilot-2.1.0/PKG-INFO +1822 -0
  11. dataspoof_mlpilot-2.1.0/README.md +1707 -0
  12. dataspoof_mlpilot-2.1.0/SECURITY.md +50 -0
  13. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/Chart.yaml +6 -0
  14. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/_helpers.tpl +13 -0
  15. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/deployment.yaml +80 -0
  16. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/ingress.yaml +29 -0
  17. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/pdb.yaml +11 -0
  18. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/pvc.yaml +15 -0
  19. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/secret.yaml +11 -0
  20. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/service.yaml +10 -0
  21. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/serviceaccount.yaml +11 -0
  22. dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/values.yaml +58 -0
  23. dataspoof_mlpilot-2.1.0/deploy/hub/Dockerfile +24 -0
  24. dataspoof_mlpilot-2.1.0/deploy/hub/docker-compose.yml +33 -0
  25. dataspoof_mlpilot-2.1.0/deploy/monitoring/docker-compose.yml +49 -0
  26. dataspoof_mlpilot-2.1.0/deploy/monitoring/grafana/dashboards/mlpilot-model.json +1173 -0
  27. dataspoof_mlpilot-2.1.0/deploy/monitoring/grafana/provisioning/dashboards/mlpilot.yml +9 -0
  28. dataspoof_mlpilot-2.1.0/deploy/monitoring/grafana/provisioning/datasources/prometheus.yml +10 -0
  29. dataspoof_mlpilot-2.1.0/deploy/monitoring/prometheus/alerts.yml +117 -0
  30. dataspoof_mlpilot-2.1.0/deploy/monitoring/prometheus/alerts_test.yml +110 -0
  31. dataspoof_mlpilot-2.1.0/deploy/monitoring/prometheus/prometheus.yml +23 -0
  32. dataspoof_mlpilot-2.1.0/deploy/opa/policies/mlpilot/authz.rego +39 -0
  33. dataspoof_mlpilot-2.1.0/deploy/opa/policies/mlpilot/config/data.json +8 -0
  34. dataspoof_mlpilot-2.1.0/deploy/opa/policies/mlpilot/policy_test.rego +128 -0
  35. dataspoof_mlpilot-2.1.0/deploy/opa/policies/mlpilot/promotion.rego +66 -0
  36. dataspoof_mlpilot-2.1.0/deploy/terraform/aws/main.tf +194 -0
  37. dataspoof_mlpilot-2.1.0/deploy/terraform/aws/outputs.tf +20 -0
  38. dataspoof_mlpilot-2.1.0/deploy/terraform/aws/variables.tf +109 -0
  39. dataspoof_mlpilot-2.1.0/deploy/terraform/aws-ecs/main.tf +293 -0
  40. dataspoof_mlpilot-2.1.0/deploy/terraform/aws-ecs/outputs.tf +15 -0
  41. dataspoof_mlpilot-2.1.0/deploy/terraform/aws-ecs/variables.tf +126 -0
  42. dataspoof_mlpilot-2.1.0/deploy/terraform/aws-sagemaker/main.tf +232 -0
  43. dataspoof_mlpilot-2.1.0/deploy/terraform/aws-sagemaker/outputs.tf +16 -0
  44. dataspoof_mlpilot-2.1.0/deploy/terraform/aws-sagemaker/variables.tf +128 -0
  45. dataspoof_mlpilot-2.1.0/docs/architecture.md +32 -0
  46. dataspoof_mlpilot-2.1.0/docs/assets/workflow.svg +19 -0
  47. dataspoof_mlpilot-2.1.0/docs/changelog.md +1 -0
  48. dataspoof_mlpilot-2.1.0/docs/contributing.md +1 -0
  49. dataspoof_mlpilot-2.1.0/docs/guide/anomaly.md +16 -0
  50. dataspoof_mlpilot-2.1.0/docs/guide/automl.md +78 -0
  51. dataspoof_mlpilot-2.1.0/docs/guide/aws.md +31 -0
  52. dataspoof_mlpilot-2.1.0/docs/guide/causal.md +20 -0
  53. dataspoof_mlpilot-2.1.0/docs/guide/cleaning.md +186 -0
  54. dataspoof_mlpilot-2.1.0/docs/guide/deployment.md +64 -0
  55. dataspoof_mlpilot-2.1.0/docs/guide/environments-cicd.md +28 -0
  56. dataspoof_mlpilot-2.1.0/docs/guide/evaluation.md +78 -0
  57. dataspoof_mlpilot-2.1.0/docs/guide/feature-store.md +26 -0
  58. dataspoof_mlpilot-2.1.0/docs/guide/features.md +50 -0
  59. dataspoof_mlpilot-2.1.0/docs/guide/forecasting.md +17 -0
  60. dataspoof_mlpilot-2.1.0/docs/guide/hub.md +58 -0
  61. dataspoof_mlpilot-2.1.0/docs/guide/ingestion.md +62 -0
  62. dataspoof_mlpilot-2.1.0/docs/guide/insights.md +36 -0
  63. dataspoof_mlpilot-2.1.0/docs/guide/integrations.md +73 -0
  64. dataspoof_mlpilot-2.1.0/docs/guide/lineage.md +29 -0
  65. dataspoof_mlpilot-2.1.0/docs/guide/media.md +32 -0
  66. dataspoof_mlpilot-2.1.0/docs/guide/model-security.md +31 -0
  67. dataspoof_mlpilot-2.1.0/docs/guide/monitoring.md +38 -0
  68. dataspoof_mlpilot-2.1.0/docs/guide/observability.md +33 -0
  69. dataspoof_mlpilot-2.1.0/docs/guide/quality.md +100 -0
  70. dataspoof_mlpilot-2.1.0/docs/guide/registry.md +33 -0
  71. dataspoof_mlpilot-2.1.0/docs/guide/secrets-policy.md +34 -0
  72. dataspoof_mlpilot-2.1.0/docs/guide/splitting.md +24 -0
  73. dataspoof_mlpilot-2.1.0/docs/guide/stats.md +28 -0
  74. dataspoof_mlpilot-2.1.0/docs/guide/versioning.md +23 -0
  75. dataspoof_mlpilot-2.1.0/docs/index.md +105 -0
  76. dataspoof_mlpilot-2.1.0/docs/installation.md +89 -0
  77. dataspoof_mlpilot-2.1.0/docs/policies.md +40 -0
  78. dataspoof_mlpilot-2.1.0/docs/reference/anomaly.md +5 -0
  79. dataspoof_mlpilot-2.1.0/docs/reference/automl.md +5 -0
  80. dataspoof_mlpilot-2.1.0/docs/reference/causal.md +5 -0
  81. dataspoof_mlpilot-2.1.0/docs/reference/cicd.md +3 -0
  82. dataspoof_mlpilot-2.1.0/docs/reference/cleaners.md +5 -0
  83. dataspoof_mlpilot-2.1.0/docs/reference/core.md +5 -0
  84. dataspoof_mlpilot-2.1.0/docs/reference/deploy.md +9 -0
  85. dataspoof_mlpilot-2.1.0/docs/reference/environments.md +3 -0
  86. dataspoof_mlpilot-2.1.0/docs/reference/evaluation.md +5 -0
  87. dataspoof_mlpilot-2.1.0/docs/reference/feature_store.md +5 -0
  88. dataspoof_mlpilot-2.1.0/docs/reference/features.md +5 -0
  89. dataspoof_mlpilot-2.1.0/docs/reference/forecast.md +5 -0
  90. dataspoof_mlpilot-2.1.0/docs/reference/hub.md +9 -0
  91. dataspoof_mlpilot-2.1.0/docs/reference/ingestion.md +5 -0
  92. dataspoof_mlpilot-2.1.0/docs/reference/insights.md +5 -0
  93. dataspoof_mlpilot-2.1.0/docs/reference/integrations.md +29 -0
  94. dataspoof_mlpilot-2.1.0/docs/reference/lineage.md +3 -0
  95. dataspoof_mlpilot-2.1.0/docs/reference/media.md +5 -0
  96. dataspoof_mlpilot-2.1.0/docs/reference/monitoring.md +9 -0
  97. dataspoof_mlpilot-2.1.0/docs/reference/poisoning.md +3 -0
  98. dataspoof_mlpilot-2.1.0/docs/reference/quality.md +5 -0
  99. dataspoof_mlpilot-2.1.0/docs/reference/registry.md +5 -0
  100. dataspoof_mlpilot-2.1.0/docs/reference/robustness.md +3 -0
  101. dataspoof_mlpilot-2.1.0/docs/reference/scale.md +5 -0
  102. dataspoof_mlpilot-2.1.0/docs/reference/secretstore.md +3 -0
  103. dataspoof_mlpilot-2.1.0/docs/reference/security.md +5 -0
  104. dataspoof_mlpilot-2.1.0/docs/reference/splitting.md +3 -0
  105. dataspoof_mlpilot-2.1.0/docs/reference/stats.md +5 -0
  106. dataspoof_mlpilot-2.1.0/docs/reference/versioning.md +5 -0
  107. dataspoof_mlpilot-2.1.0/docs/security.md +1 -0
  108. dataspoof_mlpilot-2.1.0/docs/workflow.md +24 -0
  109. dataspoof_mlpilot-2.1.0/mkdocs.yml +138 -0
  110. dataspoof_mlpilot-2.1.0/mlpilot/__init__.py +60 -0
  111. dataspoof_mlpilot-2.1.0/mlpilot/_compat.py +64 -0
  112. dataspoof_mlpilot-2.1.0/mlpilot/_detect.py +136 -0
  113. dataspoof_mlpilot-2.1.0/mlpilot/_rustcore.pyi +22 -0
  114. dataspoof_mlpilot-2.1.0/mlpilot/anomaly.py +137 -0
  115. dataspoof_mlpilot-2.1.0/mlpilot/automl/__init__.py +47 -0
  116. dataspoof_mlpilot-2.1.0/mlpilot/automl/_persist.py +92 -0
  117. dataspoof_mlpilot-2.1.0/mlpilot/automl/automl.py +630 -0
  118. dataspoof_mlpilot-2.1.0/mlpilot/automl/cluster.py +169 -0
  119. dataspoof_mlpilot-2.1.0/mlpilot/automl/frame.py +67 -0
  120. dataspoof_mlpilot-2.1.0/mlpilot/automl/models.py +147 -0
  121. dataspoof_mlpilot-2.1.0/mlpilot/automl/tasks.py +76 -0
  122. dataspoof_mlpilot-2.1.0/mlpilot/causal.py +227 -0
  123. dataspoof_mlpilot-2.1.0/mlpilot/cicd.py +396 -0
  124. dataspoof_mlpilot-2.1.0/mlpilot/cleaners.py +584 -0
  125. dataspoof_mlpilot-2.1.0/mlpilot/cleaning.py +1368 -0
  126. dataspoof_mlpilot-2.1.0/mlpilot/core/__init__.py +28 -0
  127. dataspoof_mlpilot-2.1.0/mlpilot/core/pipeline.py +516 -0
  128. dataspoof_mlpilot-2.1.0/mlpilot/core/registry.py +91 -0
  129. dataspoof_mlpilot-2.1.0/mlpilot/core/step.py +150 -0
  130. dataspoof_mlpilot-2.1.0/mlpilot/deploy/__init__.py +18 -0
  131. dataspoof_mlpilot-2.1.0/mlpilot/deploy/__main__.py +5 -0
  132. dataspoof_mlpilot-2.1.0/mlpilot/deploy/_names.py +7 -0
  133. dataspoof_mlpilot-2.1.0/mlpilot/deploy/asgi.py +11 -0
  134. dataspoof_mlpilot-2.1.0/mlpilot/deploy/aws.py +187 -0
  135. dataspoof_mlpilot-2.1.0/mlpilot/deploy/cli.py +284 -0
  136. dataspoof_mlpilot-2.1.0/mlpilot/deploy/export.py +285 -0
  137. dataspoof_mlpilot-2.1.0/mlpilot/deploy/scoring.py +179 -0
  138. dataspoof_mlpilot-2.1.0/mlpilot/deploy/server.py +753 -0
  139. dataspoof_mlpilot-2.1.0/mlpilot/deploy/wsgi.py +53 -0
  140. dataspoof_mlpilot-2.1.0/mlpilot/environments.py +240 -0
  141. dataspoof_mlpilot-2.1.0/mlpilot/evaluation/__init__.py +23 -0
  142. dataspoof_mlpilot-2.1.0/mlpilot/evaluation/_slicing.py +90 -0
  143. dataspoof_mlpilot-2.1.0/mlpilot/evaluation/adapter.py +127 -0
  144. dataspoof_mlpilot-2.1.0/mlpilot/evaluation/card.py +604 -0
  145. dataspoof_mlpilot-2.1.0/mlpilot/evaluation/errors.py +177 -0
  146. dataspoof_mlpilot-2.1.0/mlpilot/evaluation/explain.py +312 -0
  147. dataspoof_mlpilot-2.1.0/mlpilot/evaluation/fairness.py +377 -0
  148. dataspoof_mlpilot-2.1.0/mlpilot/evaluation/metrics.py +309 -0
  149. dataspoof_mlpilot-2.1.0/mlpilot/feature_store.py +313 -0
  150. dataspoof_mlpilot-2.1.0/mlpilot/features/__init__.py +33 -0
  151. dataspoof_mlpilot-2.1.0/mlpilot/features/dates.py +91 -0
  152. dataspoof_mlpilot-2.1.0/mlpilot/features/defaults.py +55 -0
  153. dataspoof_mlpilot-2.1.0/mlpilot/features/encoding.py +130 -0
  154. dataspoof_mlpilot-2.1.0/mlpilot/features/generators.py +184 -0
  155. dataspoof_mlpilot-2.1.0/mlpilot/features/leakage.py +221 -0
  156. dataspoof_mlpilot-2.1.0/mlpilot/features/selection.py +119 -0
  157. dataspoof_mlpilot-2.1.0/mlpilot/features/text.py +133 -0
  158. dataspoof_mlpilot-2.1.0/mlpilot/forecast.py +331 -0
  159. dataspoof_mlpilot-2.1.0/mlpilot/hub/__init__.py +22 -0
  160. dataspoof_mlpilot-2.1.0/mlpilot/hub/app.py +700 -0
  161. dataspoof_mlpilot-2.1.0/mlpilot/hub/client.py +142 -0
  162. dataspoof_mlpilot-2.1.0/mlpilot/hub/db.py +171 -0
  163. dataspoof_mlpilot-2.1.0/mlpilot/hub/policy.py +82 -0
  164. dataspoof_mlpilot-2.1.0/mlpilot/hub/security.py +180 -0
  165. dataspoof_mlpilot-2.1.0/mlpilot/hub/storage.py +103 -0
  166. dataspoof_mlpilot-2.1.0/mlpilot/hub/ui.html +71 -0
  167. dataspoof_mlpilot-2.1.0/mlpilot/hub/ui.js +150 -0
  168. dataspoof_mlpilot-2.1.0/mlpilot/ingestion/__init__.py +23 -0
  169. dataspoof_mlpilot-2.1.0/mlpilot/ingestion/_util.py +107 -0
  170. dataspoof_mlpilot-2.1.0/mlpilot/ingestion/api.py +178 -0
  171. dataspoof_mlpilot-2.1.0/mlpilot/ingestion/core.py +136 -0
  172. dataspoof_mlpilot-2.1.0/mlpilot/ingestion/files.py +144 -0
  173. dataspoof_mlpilot-2.1.0/mlpilot/ingestion/incremental.py +174 -0
  174. dataspoof_mlpilot-2.1.0/mlpilot/ingestion/schema.py +68 -0
  175. dataspoof_mlpilot-2.1.0/mlpilot/ingestion/sql.py +217 -0
  176. dataspoof_mlpilot-2.1.0/mlpilot/ingestion/warehouse.py +134 -0
  177. dataspoof_mlpilot-2.1.0/mlpilot/insights/__init__.py +15 -0
  178. dataspoof_mlpilot-2.1.0/mlpilot/insights/finder.py +340 -0
  179. dataspoof_mlpilot-2.1.0/mlpilot/insights/narrate.py +75 -0
  180. dataspoof_mlpilot-2.1.0/mlpilot/insights/report.py +147 -0
  181. dataspoof_mlpilot-2.1.0/mlpilot/integrations/__init__.py +20 -0
  182. dataspoof_mlpilot-2.1.0/mlpilot/integrations/airflow.py +63 -0
  183. dataspoof_mlpilot-2.1.0/mlpilot/integrations/aws_glue.py +211 -0
  184. dataspoof_mlpilot-2.1.0/mlpilot/integrations/azure_data_factory.py +117 -0
  185. dataspoof_mlpilot-2.1.0/mlpilot/integrations/dagster.py +35 -0
  186. dataspoof_mlpilot-2.1.0/mlpilot/integrations/dvc.py +272 -0
  187. dataspoof_mlpilot-2.1.0/mlpilot/integrations/evidently.py +140 -0
  188. dataspoof_mlpilot-2.1.0/mlpilot/integrations/feast.py +166 -0
  189. dataspoof_mlpilot-2.1.0/mlpilot/integrations/kafka.py +96 -0
  190. dataspoof_mlpilot-2.1.0/mlpilot/integrations/mlflow.py +296 -0
  191. dataspoof_mlpilot-2.1.0/mlpilot/integrations/prefect.py +29 -0
  192. dataspoof_mlpilot-2.1.0/mlpilot/integrations/tasks.py +95 -0
  193. dataspoof_mlpilot-2.1.0/mlpilot/lineage.py +312 -0
  194. dataspoof_mlpilot-2.1.0/mlpilot/media/__init__.py +19 -0
  195. dataspoof_mlpilot-2.1.0/mlpilot/media/audio.py +279 -0
  196. dataspoof_mlpilot-2.1.0/mlpilot/media/image.py +353 -0
  197. dataspoof_mlpilot-2.1.0/mlpilot/monitoring/__init__.py +22 -0
  198. dataspoof_mlpilot-2.1.0/mlpilot/monitoring/drift.py +275 -0
  199. dataspoof_mlpilot-2.1.0/mlpilot/monitoring/estimate.py +178 -0
  200. dataspoof_mlpilot-2.1.0/mlpilot/monitoring/monitor.py +490 -0
  201. dataspoof_mlpilot-2.1.0/mlpilot/monitoring/ops.py +130 -0
  202. dataspoof_mlpilot-2.1.0/mlpilot/poisoning.py +225 -0
  203. dataspoof_mlpilot-2.1.0/mlpilot/py.typed +0 -0
  204. dataspoof_mlpilot-2.1.0/mlpilot/quality/__init__.py +25 -0
  205. dataspoof_mlpilot-2.1.0/mlpilot/quality/checks.py +547 -0
  206. dataspoof_mlpilot-2.1.0/mlpilot/quality/contracts.py +458 -0
  207. dataspoof_mlpilot-2.1.0/mlpilot/quality/pipeline_steps.py +115 -0
  208. dataspoof_mlpilot-2.1.0/mlpilot/quality/profile.py +328 -0
  209. dataspoof_mlpilot-2.1.0/mlpilot/quality/schema.py +65 -0
  210. dataspoof_mlpilot-2.1.0/mlpilot/registry/__init__.py +15 -0
  211. dataspoof_mlpilot-2.1.0/mlpilot/registry/_store.py +141 -0
  212. dataspoof_mlpilot-2.1.0/mlpilot/registry/models.py +280 -0
  213. dataspoof_mlpilot-2.1.0/mlpilot/registry/tracking.py +308 -0
  214. dataspoof_mlpilot-2.1.0/mlpilot/robustness.py +259 -0
  215. dataspoof_mlpilot-2.1.0/mlpilot/scale.py +236 -0
  216. dataspoof_mlpilot-2.1.0/mlpilot/secretstore.py +232 -0
  217. dataspoof_mlpilot-2.1.0/mlpilot/security.py +142 -0
  218. dataspoof_mlpilot-2.1.0/mlpilot/splitting.py +370 -0
  219. dataspoof_mlpilot-2.1.0/mlpilot/stats.py +430 -0
  220. dataspoof_mlpilot-2.1.0/mlpilot/steps.py +926 -0
  221. dataspoof_mlpilot-2.1.0/mlpilot/versioning.py +270 -0
  222. dataspoof_mlpilot-2.1.0/pyproject.toml +222 -0
  223. dataspoof_mlpilot-2.1.0/src/clean.rs +63 -0
  224. dataspoof_mlpilot-2.1.0/src/lib.rs +195 -0
  225. dataspoof_mlpilot-2.1.0/src/stats.rs +125 -0
  226. dataspoof_mlpilot-2.1.0/src/text.rs +99 -0
  227. dataspoof_mlpilot-2.1.0/tests/test_advanced.py +302 -0
  228. dataspoof_mlpilot-2.1.0/tests/test_automl.py +173 -0
  229. dataspoof_mlpilot-2.1.0/tests/test_aws_deploy.py +97 -0
  230. dataspoof_mlpilot-2.1.0/tests/test_cicd.py +89 -0
  231. dataspoof_mlpilot-2.1.0/tests/test_cleaners.py +184 -0
  232. dataspoof_mlpilot-2.1.0/tests/test_core.py +220 -0
  233. dataspoof_mlpilot-2.1.0/tests/test_data_versions_and_feature_store.py +143 -0
  234. dataspoof_mlpilot-2.1.0/tests/test_deploy.py +180 -0
  235. dataspoof_mlpilot-2.1.0/tests/test_docker_stacks.py +409 -0
  236. dataspoof_mlpilot-2.1.0/tests/test_environments.py +93 -0
  237. dataspoof_mlpilot-2.1.0/tests/test_evaluation.py +279 -0
  238. dataspoof_mlpilot-2.1.0/tests/test_features.py +246 -0
  239. dataspoof_mlpilot-2.1.0/tests/test_hub.py +264 -0
  240. dataspoof_mlpilot-2.1.0/tests/test_ingestion.py +387 -0
  241. dataspoof_mlpilot-2.1.0/tests/test_insights.py +121 -0
  242. dataspoof_mlpilot-2.1.0/tests/test_label_free_monitoring.py +101 -0
  243. dataspoof_mlpilot-2.1.0/tests/test_lineage_and_catalogs.py +202 -0
  244. dataspoof_mlpilot-2.1.0/tests/test_media.py +177 -0
  245. dataspoof_mlpilot-2.1.0/tests/test_model_security.py +105 -0
  246. dataspoof_mlpilot-2.1.0/tests/test_monitoring.py +132 -0
  247. dataspoof_mlpilot-2.1.0/tests/test_ops.py +313 -0
  248. dataspoof_mlpilot-2.1.0/tests/test_quality.py +333 -0
  249. dataspoof_mlpilot-2.1.0/tests/test_real_integrations.py +397 -0
  250. dataspoof_mlpilot-2.1.0/tests/test_registry.py +122 -0
  251. dataspoof_mlpilot-2.1.0/tests/test_rename_compat.py +31 -0
  252. dataspoof_mlpilot-2.1.0/tests/test_secretstore.py +72 -0
  253. dataspoof_mlpilot-2.1.0/tests/test_serving_feedback_reload.py +165 -0
  254. dataspoof_mlpilot-2.1.0/tests/test_splitting.py +90 -0
  255. dataspoof_mlpilot-2.1.0/tests/test_stats.py +102 -0
@@ -0,0 +1,11 @@
1
+ # Only the source needed to build the package goes into Docker build contexts.
2
+ **
3
+ !Cargo.toml
4
+ !pyproject.toml
5
+ !README.md
6
+ !src/**
7
+ !mlpilot/**
8
+ **/__pycache__
9
+ **/*.pyc
10
+ **/*.pyd
11
+ **/*.so
@@ -0,0 +1,271 @@
1
+ # Test, type-check, audit, build wheels for Linux / macOS / Windows, and publish to PyPI.
2
+ #
3
+ # - Every push / PR: tests (OS x Python x Polars matrix), mypy, dependency + Rust audits,
4
+ # wheels and sdist, so breakage is caught before a release.
5
+ # - A pushed version tag (e.g. `v2.0.0`): also publish to PyPI with Trusted Publishing (no
6
+ # API token) and signed build-provenance attestations, plus a CycloneDX SBOM on the release.
7
+ #
8
+ # The crate uses pyo3's `abi3-py311` feature, so one wheel per platform covers CPython 3.11+.
9
+
10
+ name: CI
11
+
12
+ on:
13
+ push:
14
+ branches: [main]
15
+ tags: ["v*"]
16
+ pull_request:
17
+ workflow_dispatch:
18
+
19
+ permissions:
20
+ contents: read
21
+
22
+ concurrency:
23
+ group: ci-${{ github.ref }}
24
+ cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }}
25
+
26
+ jobs:
27
+ test:
28
+ name: test (${{ matrix.os }}, py${{ matrix.python }}, polars ${{ matrix.polars }})
29
+ runs-on: ${{ matrix.os }}
30
+ strategy:
31
+ fail-fast: false
32
+ matrix:
33
+ os: [ubuntu-latest]
34
+ python: ["3.11", "3.12", "3.13"]
35
+ polars: ["1", "2"]
36
+ include:
37
+ - { os: windows-latest, python: "3.12", polars: "2" }
38
+ - { os: macos-latest, python: "3.12", polars: "2" }
39
+ env:
40
+ AC_OFFLINE: "1" # the library must work without network access
41
+ defaults:
42
+ run:
43
+ shell: bash
44
+ steps:
45
+ - uses: actions/checkout@v4
46
+ - uses: actions/setup-python@v5
47
+ with:
48
+ python-version: ${{ matrix.python }}
49
+ cache: pip
50
+ - uses: dtolnay/rust-toolchain@stable
51
+ - uses: Swatinem/rust-cache@v2
52
+ - name: Install (builds the Rust extension)
53
+ run: |
54
+ python -m pip install --upgrade pip
55
+ python -m pip install ".[test,sql,excel,server,optuna,skops,signing]" "polars==${{ matrix.polars }}.*"
56
+ - name: Run tests
57
+ # Run against the installed package (not the source tree) so packaging bugs show up.
58
+ run: |
59
+ mkdir _run
60
+ cp -r tests _run/
61
+ if [ -f titanic.csv ]; then cp titanic.csv _run/; fi
62
+ cd _run
63
+ python -m pytest tests -q -p no:cacheprovider -o pythonpath=
64
+
65
+ core-only:
66
+ name: test (core install, no extras)
67
+ runs-on: ubuntu-latest
68
+ env:
69
+ AC_OFFLINE: "1"
70
+ steps:
71
+ - uses: actions/checkout@v4
72
+ - uses: actions/setup-python@v5
73
+ with:
74
+ python-version: "3.12"
75
+ - uses: dtolnay/rust-toolchain@stable
76
+ - run: python -m pip install . pytest
77
+ - name: Import must not load heavy optional packages or touch the network
78
+ run: |
79
+ cd /tmp && python -c "
80
+ import sys, mlpilot
81
+ heavy = [m for m in ('nltk','plotly','matplotlib','presidio_analyzer','langchain_anthropic') if m in sys.modules]
82
+ assert not heavy, heavy
83
+ print('ok')"
84
+ - run: mkdir _run && cp -r tests _run/ && cd _run && python -m pytest tests -q -p no:cacheprovider -o pythonpath=
85
+
86
+ integrations:
87
+ # Real MLflow server + `mlflow models serve`, real git/DVC with a remote, real Feast.
88
+ name: test (real MLflow, DVC, Feast)
89
+ runs-on: ubuntu-latest
90
+ steps:
91
+ - uses: actions/checkout@v4
92
+ - uses: actions/setup-python@v5
93
+ with:
94
+ python-version: "3.12"
95
+ cache: pip
96
+ - uses: dtolnay/rust-toolchain@stable
97
+ - uses: Swatinem/rust-cache@v2
98
+ - run: python -m pip install ".[test,mlflow,dvc,feast,evidently,aws,server,signing]" "moto[server]"
99
+ - run: |
100
+ git config --global user.email ci@example.com
101
+ git config --global user.name ci
102
+ mkdir _run
103
+ cp -r tests _run/
104
+ cd _run
105
+ python -m pytest -q -p no:cacheprovider -o pythonpath= -rs tests/test_real_integrations.py \
106
+ tests/test_aws_deploy.py tests/test_lineage_and_catalogs.py tests/test_secretstore.py \
107
+ tests/test_label_free_monitoring.py
108
+
109
+ docker-stacks:
110
+ # Real Prometheus + Grafana, Vault, OPA (+ `opa test`, `promtool test rules`), Marquez, actionlint.
111
+ name: test (Prometheus, Grafana, Vault, OPA, Marquez in Docker)
112
+ runs-on: ubuntu-latest
113
+ env:
114
+ AC_DOCKER_TESTS: "1"
115
+ steps:
116
+ - uses: actions/checkout@v4
117
+ - uses: actions/setup-python@v5
118
+ with:
119
+ python-version: "3.12"
120
+ cache: pip
121
+ - uses: dtolnay/rust-toolchain@stable
122
+ - uses: Swatinem/rust-cache@v2
123
+ - run: python -m pip install ".[test,server]"
124
+ - run: python -m pytest tests/test_docker_stacks.py -q -p no:cacheprovider -rs
125
+
126
+ typecheck:
127
+ runs-on: ubuntu-latest
128
+ steps:
129
+ - uses: actions/checkout@v4
130
+ - uses: actions/setup-python@v5
131
+ with:
132
+ python-version: "3.12"
133
+ - run: python -m pip install mypy polars numpy pandas
134
+ - run: python -m mypy
135
+
136
+ security:
137
+ name: security audit
138
+ runs-on: ubuntu-latest
139
+ steps:
140
+ - uses: actions/checkout@v4
141
+ - uses: actions/setup-python@v5
142
+ with:
143
+ python-version: "3.12"
144
+ - uses: dtolnay/rust-toolchain@stable
145
+ - name: Python dependency vulnerabilities (pip-audit)
146
+ run: |
147
+ python -m pip install pip-audit
148
+ python -m pip install ".[all]"
149
+ pip-audit --skip-editable --progress-spinner off
150
+ - name: Rust dependency vulnerabilities (cargo audit)
151
+ run: |
152
+ cargo install cargo-audit --locked
153
+ cargo generate-lockfile
154
+ cargo audit
155
+
156
+ linux:
157
+ runs-on: ubuntu-latest
158
+ strategy:
159
+ fail-fast: false
160
+ matrix:
161
+ target: [x86_64, aarch64]
162
+ steps:
163
+ - uses: actions/checkout@v4
164
+ - uses: actions/setup-python@v5
165
+ with:
166
+ python-version: "3.11"
167
+ - name: Build wheels
168
+ uses: PyO3/maturin-action@v1
169
+ with:
170
+ target: ${{ matrix.target }}
171
+ args: --release --out dist
172
+ sccache: "true"
173
+ manylinux: auto
174
+ - uses: actions/upload-artifact@v4
175
+ with:
176
+ name: wheels-linux-${{ matrix.target }}
177
+ path: dist
178
+
179
+ windows:
180
+ runs-on: windows-latest
181
+ steps:
182
+ - uses: actions/checkout@v4
183
+ - uses: actions/setup-python@v5
184
+ with:
185
+ python-version: "3.11"
186
+ architecture: x64
187
+ - name: Build wheels
188
+ uses: PyO3/maturin-action@v1
189
+ with:
190
+ target: x64
191
+ args: --release --out dist
192
+ sccache: "true"
193
+ - uses: actions/upload-artifact@v4
194
+ with:
195
+ name: wheels-windows-x64
196
+ path: dist
197
+
198
+ macos:
199
+ runs-on: macos-latest
200
+ strategy:
201
+ fail-fast: false
202
+ matrix:
203
+ target: [x86_64, aarch64]
204
+ steps:
205
+ - uses: actions/checkout@v4
206
+ - uses: actions/setup-python@v5
207
+ with:
208
+ python-version: "3.11"
209
+ - name: Build wheels
210
+ uses: PyO3/maturin-action@v1
211
+ with:
212
+ target: ${{ matrix.target }}
213
+ args: --release --out dist
214
+ sccache: "true"
215
+ - uses: actions/upload-artifact@v4
216
+ with:
217
+ name: wheels-macos-${{ matrix.target }}
218
+ path: dist
219
+
220
+ sdist:
221
+ runs-on: ubuntu-latest
222
+ steps:
223
+ - uses: actions/checkout@v4
224
+ - name: Build sdist
225
+ uses: PyO3/maturin-action@v1
226
+ with:
227
+ command: sdist
228
+ args: --out dist
229
+ - uses: actions/upload-artifact@v4
230
+ with:
231
+ name: wheels-sdist
232
+ path: dist
233
+
234
+ # Publishes only when a version tag (v*) is pushed, and only if every check passed.
235
+ release:
236
+ name: Release to PyPI
237
+ runs-on: ubuntu-latest
238
+ if: startsWith(github.ref, 'refs/tags/v')
239
+ needs: [test, core-only, typecheck, security, linux, windows, macos, sdist]
240
+ environment:
241
+ name: pypi
242
+ url: https://pypi.org/project/dataspoof-mlpilot/
243
+ permissions:
244
+ id-token: write # PyPI Trusted Publishing (OIDC) + Sigstore signing
245
+ attestations: write # build-provenance attestations
246
+ contents: write # attach the SBOM to the GitHub release
247
+ steps:
248
+ - uses: actions/checkout@v4
249
+ - uses: actions/download-artifact@v4
250
+ with:
251
+ pattern: wheels-*
252
+ merge-multiple: true
253
+ path: dist
254
+ - name: Build provenance attestation (signed, verifiable with `gh attestation verify`)
255
+ uses: actions/attest-build-provenance@v2
256
+ with:
257
+ subject-path: dist/*
258
+ - name: Software bill of materials (CycloneDX)
259
+ run: |
260
+ python -m pip install cyclonedx-bom
261
+ python -m pip install dist/*manylinux*x86_64*.whl
262
+ cyclonedx-py environment --of JSON -o sbom.cdx.json
263
+ - name: Publish to PyPI
264
+ # Trusted Publishing; also uploads PEP 740 attestations for every file.
265
+ uses: pypa/gh-action-pypi-publish@release/v1
266
+ with:
267
+ skip-existing: true
268
+ - name: GitHub release with SBOM
269
+ env:
270
+ GH_TOKEN: ${{ github.token }}
271
+ run: gh release create "${GITHUB_REF_NAME}" dist/* sbom.cdx.json --generate-notes --verify-tag || gh release upload "${GITHUB_REF_NAME}" sbom.cdx.json --clobber
@@ -0,0 +1,47 @@
1
+ # Build the documentation site on every push / PR; deploy it to GitHub Pages from main.
2
+ # One-time setup: repository Settings -> Pages -> Source: "GitHub Actions".
3
+ name: docs
4
+
5
+ on:
6
+ push:
7
+ branches: [main]
8
+ pull_request:
9
+ workflow_dispatch:
10
+
11
+ permissions:
12
+ contents: read
13
+
14
+ concurrency:
15
+ group: docs-${{ github.ref }}
16
+ cancel-in-progress: true
17
+
18
+ jobs:
19
+ build:
20
+ runs-on: ubuntu-latest
21
+ steps:
22
+ - uses: actions/checkout@v4
23
+ - uses: actions/setup-python@v5
24
+ with:
25
+ python-version: "3.12"
26
+ - run: python -m pip install mkdocs-material "mkdocstrings[python]"
27
+ - run: mkdocs build --strict
28
+ env:
29
+ NO_MKDOCS_2_WARNING: "1"
30
+ - uses: actions/upload-pages-artifact@v3
31
+ if: github.event_name != 'pull_request'
32
+ with:
33
+ path: site
34
+
35
+ deploy:
36
+ if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
37
+ needs: build
38
+ runs-on: ubuntu-latest
39
+ permissions:
40
+ pages: write
41
+ id-token: write
42
+ environment:
43
+ name: github-pages
44
+ url: ${{ steps.deployment.outputs.page_url }}
45
+ steps:
46
+ - id: deployment
47
+ uses: actions/deploy-pages@v4
@@ -0,0 +1,26 @@
1
+ pypi/
2
+ venv/
3
+ dist/
4
+ build/
5
+ *.egg-info/
6
+ *.log
7
+ output/
8
+ *.csv
9
+
10
+ # Rust build artifacts
11
+ /target
12
+ Cargo.lock
13
+
14
+ # Compiled Rust extension dropped into the package for in-place testing
15
+ mlpilot/*.pyd
16
+ mlpilot/*.so
17
+
18
+ # Python bytecode
19
+ __pycache__/
20
+ *.pyc
21
+
22
+ # Documentation build output
23
+ site/
24
+
25
+ # Windows debug symbols from local Rust builds
26
+ *.pdb
@@ -0,0 +1,54 @@
1
+ # Build & Architecture
2
+
3
+ This library uses a **Rust compute backend** with a **Python frontend**, packaged
4
+ as a single `pip`-installable wheel via [maturin](https://www.maturin.rs/) + [PyO3](https://pyo3.rs/).
5
+
6
+ ## Layout
7
+
8
+ ```
9
+ Cargo.toml # Rust crate (cdylib -> _rustcore)
10
+ pyproject.toml # maturin build backend + Python deps
11
+ src/
12
+ lib.rs # PyO3 module: exposes functions to Python
13
+ text.rs # text preprocessing (contractions, regex, stopwords)
14
+ stats.rs # skewness, IQR, correlation, multicollinearity
15
+ mlpilot/
16
+ __init__.py # from .cleaning import *
17
+ cleaning.py # Python frontend (I/O, ML, dashboard, PII, AI)
18
+ _rustcore.*.pyd # compiled Rust extension (built artifact)
19
+ ```
20
+
21
+ ## What runs where
22
+
23
+ | Concern | Location | Why |
24
+ |---|---|---|
25
+ | Text cleaning, symbol stripping, IQR outliers, skewness, correlation / multicollinearity, negatives, JSON detection | **Rust** (`_rustcore`) | CPU-bound, parallelised with `rayon`, GIL released |
26
+ | WordNet lemmatization | Python | preserves exact NLTK output |
27
+ | CSV/Parquet I/O, KNN imputation, class balancing | Python (Polars / scikit-learn) | mature, no benefit to reimplementing |
28
+ | Plotly dashboard, Presidio PII masking, Claude spell-correction | Python | ecosystem-only |
29
+
30
+ The Python functions delegate the heavy loops to `_rustcore`; results are verified
31
+ to match Polars to machine precision (skewness, IQR bounds, correlation drop-list).
32
+
33
+ ## Build
34
+
35
+ Prereqs: Rust toolchain (`rustup`) and `maturin` (`pip install maturin`).
36
+
37
+ ```bash
38
+ # Build a release wheel (dist-ready, abi3 -> works on CPython 3.11+)
39
+ maturin build --release
40
+ # -> target/wheels/mlpilot-1.5.1-cp311-abi3-win_amd64.whl
41
+
42
+ # Install it
43
+ pip install target/wheels/mlpilot-1.5.1-cp311-abi3-win_amd64.whl
44
+
45
+ # --- OR, for development (requires an active virtualenv) ---
46
+ python -m venv .venv && .venv/Scripts/activate # Windows
47
+ maturin develop --release
48
+ ```
49
+
50
+ ## Publish to PyPI
51
+
52
+ `maturin build --release` produces the wheel; upload with `twine upload target/wheels/*`
53
+ (or `maturin publish`). For cross-platform wheels, build in CI with
54
+ `PyO3/maturin-action`.
@@ -0,0 +1,204 @@
1
+ # Changelog
2
+
3
+ All notable changes are listed here. The project follows [Semantic Versioning](https://semver.org/)
4
+ (see the [versioning policy](https://dataspoof.github.io/dataspoof-data-cleaning/policies/)).
5
+
6
+ ## [2.1.0] — 2026-11-09
7
+
8
+ ### Renamed: AutomatedCleaning is now MLPilot
9
+ - **Names:**
10
+ - the distribution is `dataspoof-mlpilot` (PyPI refused `mlpilot` as too similar to an existing project);
11
+ - the import is `import mlpilot`, and the CLI is `mlpilot`;
12
+ - `MLPilotWarning` replaces `AutomatedCleaningWarning`, which remains as an alias.
13
+ - **Compatibility:**
14
+ - `import automatedcleaning` and pickles that refer to `automatedcleaning.*` classes resolve to
15
+ the same `mlpilot` modules, so models saved by AutomatedCleaning keep loading with the same
16
+ classes;
17
+ - the `AutomatedCleaning` project on PyPI gets a final 2.0.1 that installs `dataspoof-mlpilot`.
18
+ - **Unchanged:** the `AC_*` environment variables, the `ac_*` Prometheus metrics and the saved-file
19
+ formats (`dataspoof.*`).
20
+ - **Renamed with the library:**
21
+ - the Helm chart (`mlpilot-hub`), the OPA policy package (`data.mlpilot`) and the Prometheus job;
22
+ - the Grafana dashboard file;
23
+ - the MLflow tag prefix and the default OpenLineage namespace.
24
+
25
+ ## [2.0.0] — 2026-11-09
26
+
27
+ ### Breaking
28
+ - **Lean core install.** `pip install AutomatedCleaning` now installs only the core
29
+ dependencies. The heavy ones moved to extras:
30
+ - `[nlp]`: nltk
31
+ - `[dashboard]`: plotly, matplotlib, missingno, pyfiglet
32
+ - `[pii]`: presidio
33
+ - `[llm]`: anthropic, langchain-anthropic
34
+
35
+ To get what 1.x installed, use `pip install "AutomatedCleaning[all]"`. A feature whose extra
36
+ is missing raises an `ImportError` naming the extra to install.
37
+ - `setup.py` was removed. The build is defined only in `pyproject.toml` (maturin).
38
+
39
+ ### Changed
40
+ - **No side effects at import.**
41
+ - Importing the library no longer downloads NLTK data, changes the application's warning
42
+ filters, or reconfigures logging.
43
+ - NLTK corpora are downloaded on first use, and never when `AC_OFFLINE` is set. Text cleaning
44
+ falls back to a built-in stopword list when NLTK isn't installed.
45
+ - **Typed package.** It ships a `py.typed` marker and stubs for the Rust extension, and passes
46
+ `mypy`.
47
+ - **CI** now runs:
48
+ - tests across Linux, macOS and Windows × Python 3.11–3.13 × Polars 1/2;
49
+ - a core-only install test;
50
+ - mypy;
51
+ - pip-audit and cargo-audit.
52
+
53
+ Releases publish with PyPI Trusted Publishing, signed build-provenance attestations and a
54
+ CycloneDX SBOM.
55
+ - Interactive AI corrections use `ANTHROPIC_API_KEY` when it is set, and the current Claude model.
56
+
57
+ ### Added
58
+ - **Hub, the multi-user platform server** (`automatedcleaning.hub`, `[server]` extra):
59
+ - workspaces, with SSO through OIDC and identity-provider group mapping;
60
+ - API tokens and service accounts;
61
+ - four roles (viewer, data scientist, approver, admin);
62
+ - runs and a model registry with immutable, checksummed artifacts on disk, S3, GCS or Azure;
63
+ - two-person production approvals with policy gates;
64
+ - monitoring results;
65
+ - a hash-chained, verifiable and exportable audit log;
66
+ - a web UI, the Python client `HubClient`, and `automatedcleaning hub serve | create-admin`.
67
+ - **Hub deployment:** a Docker image with compose (plus PostgreSQL), a Helm chart, and a
68
+ Terraform module for AWS (S3, RDS, Secrets Manager, IAM role, Helm release).
69
+ - **Forecasting** (`Forecaster`):
70
+ - multiple series and multi-step horizons;
71
+ - rolling-origin backtests against seasonal-naive;
72
+ - conformal intervals and hierarchical reconciliation.
73
+ - **Causal inference** (`estimate_effect`): doubly-robust AIPW with overlap and balance
74
+ diagnostics. **Uplift:** `UpliftModel` and `qini`.
75
+ - **Anomaly detection** (`AutoAnomaly`): an ensemble of three detectors with plain-language
76
+ explanations.
77
+ - **AutoML:**
78
+ - conformal `predict_interval` and `predict_set`;
79
+ - `search="optuna"`, `imbalance="auto"`, `n_jobs`;
80
+ - `predict_with_proba` and `set_inference_threads`.
81
+ - **Serving:**
82
+ - Prometheus `/metrics` and OpenTelemetry tracing;
83
+ - canary and shadow rollouts, and `app_from_registry`;
84
+ - SageMaker, Vertex AI and Azure ML routes and Docker targets;
85
+ - an ASGI app and the `container` entry point;
86
+ - single-row latency down from about 500 ms to about 50 ms.
87
+ - **Security:**
88
+ - Ed25519 model signing, verified before unpickling whenever `AC_MODEL_PUBLIC_KEY` is set;
89
+ - `skops` export;
90
+ - signing on registration with `register(..., sign_with=...)`.
91
+ - **Monitoring:**
92
+ - PagerDuty and Opsgenie alert handlers;
93
+ - `join_labels` and `Monitor.check_labels` for labels that arrive later;
94
+ - `sql_sink` and `Monitor.on_check`;
95
+ - `SQLSource.insert` and `execute`.
96
+ - **Scale:**
97
+ - `transform_file` for files of any size;
98
+ - Spark, Ray and Dask scoring;
99
+ - `to_sql` SQL pushdown of fitted cleaning pipelines.
100
+ - **Integrations:** Airflow operators, Prefect tasks and flow, Dagster assets, Kafka stream
101
+ scoring, and plain `integrations.tasks`.
102
+ - **MLflow** (`integrations.mlflow`, `[mlflow]` extra):
103
+ - batched, idempotent export of runs and whole trackers;
104
+ - AutoML models logged as pyfunc models with a signature, so they serve with `mlflow models serve`;
105
+ - `export_registry` (stages become aliases) and `load_automl`.
106
+ - **DVC** (`integrations.dvc`, `[dvc]` extra): `DVCRepo` (init, remotes, track, push and pull,
107
+ read any revision, checkout, `log_to_run`) and `export_repository` for `DataRepository`
108
+ history.
109
+ - **Feast** (`integrations.feast`, `[feast]` extra): `export_to_feast` writes, applies and
110
+ materializes a Feast repository.
111
+ - **Data splitting** (`split_data`, `cv_folds`, `check_split`): stratified, grouped and time-ordered
112
+ splits with gaps; every split is leak-checked (shared groups, time overlap, duplicate rows, unseen classes).
113
+ - **Model security:** `robustness_report` (noise stability, a minimal adversarial attack,
114
+ feature sensitivity, extreme inputs) and `poisoning_report` (confident-learning label errors,
115
+ conflicting duplicates, backdoor triggers, outliers).
116
+ - **Environments** (`Environments`, `environments.yml`): isolated dev / QA / prod registries;
117
+ `release` enforces order, source stage, metric gates, four-eyes approvals and signatures, and
118
+ records provenance on both sides. The registry and Hub gain a `qa` stage.
119
+ - **Model CI/CD:** the CLI commands `train`, `gate`, `register`, `promote`, `release`, `latest`,
120
+ `deploy-sagemaker` and `cicd init` (GitHub Actions / GitLab CI pipelines, checked with actionlint).
121
+ - **Secrets** (`secretstore`): `vault://` (token, AppRole, Kubernetes; KV v1 and v2), `aws-sm://`,
122
+ `env://` and `file://` references, resolved wherever credentials are read.
123
+ - **Open Policy Agent for the Hub** (`hub.policy`): deny-overrides access rules, promotion rules
124
+ and fail-closed behaviour, plus Rego policies with unit tests in `deploy/opa`.
125
+ - **Serving:**
126
+ - `POST /feedback` with prediction IDs, live-accuracy metrics and `join_feedback`;
127
+ - hot reload (`watch=`, `POST /admin/reload`, registry-following);
128
+ - live drift, estimated-performance and alert gauges (`monitor=`).
129
+ - **Monitoring before labels:** `PerformanceEstimator` (CBPE, direct loss estimation),
130
+ `domain_classifier_drift`, and `estimated_performance` alerts in `Monitor.check`.
131
+ - **Grafana + Prometheus** (`deploy/monitoring`): a dashboard, 14 alert rules with `promtool`
132
+ unit tests, and a compose stack.
133
+ - **Lineage** (`lineage`): OpenLineage run events with schema, quality and version facets; HTTP
134
+ and file transports; `emit_tracked_run`, `emit_dataset_version`, `LineageGraph`.
135
+ - **Catalogs and orchestration:** AWS Glue (catalog tables, partitions, reads, job runs) and Azure
136
+ Data Factory pipeline runs.
137
+ - **AWS deployment** (`deploy.aws`): ECR push and blue/green SageMaker endpoints (serverless or
138
+ instances, autoscaling, data capture, rollback alarms); Terraform for SageMaker and ECS Fargate.
139
+ - **Evidently** (`integrations.evidently`): reports plus agreement with the built-in drift report.
140
+ - **Tests against real infrastructure:** Prometheus, Grafana, Vault, OPA and Marquez in Docker
141
+ (`AC_DOCKER_TESTS=1`); moto for AWS; dedicated CI jobs.
142
+ - These three are tested against the real tools (MLflow server and model server, git + DVC
143
+ with a remote, Feast), including in a dedicated CI job.
144
+ - **Image and audio data** (`media`):
145
+ - quality audits (corrupt files, exact and near duplicates, blur, exposure, clipping, silence,
146
+ mixed formats);
147
+ - preprocessors (EXIF-aware image resize and normalise; audio resample, trim and level);
148
+ - augmentation;
149
+ - features for AutoML (colour and edge-orientation histograms; MFCC and spectral features).
150
+ - **Statistical testing** (`stats`):
151
+ - automatic test selection with assumption checks, effect sizes and confidence intervals;
152
+ - chi-square and Fisher's exact test, correlations;
153
+ - A/B tests, power analysis, multiple-testing correction, `test_all`.
154
+ - **Data version control** (`DataRepository`): content-addressed commits, tags, row-level
155
+ diffs, checksum-verified checkout, links to experiment runs, garbage collection.
156
+ - **Feature store** (`FeatureStore`, `Entity`, `FeatureView`): point-in-time-correct
157
+ historical joins with TTL, an SQLite online store, `materialize`, schema guards.
158
+ - **Documentation site** (MkDocs Material with an API reference), plus `SECURITY.md`,
159
+ `CONTRIBUTING.md` and a versioning policy.
160
+
161
+ ### Removed
162
+ - `automatedcleaning/cleaning_update.py`: dead code that nothing imported.
163
+
164
+ ## [1.14.0] — 2026-11-08
165
+ - **Ingestion & connectors:**
166
+ - files, globs, HTTP, and S3 / GCS / Azure;
167
+ - any SQL database (bound parameters);
168
+ - Snowflake and BigQuery;
169
+ - REST APIs with five pagination styles;
170
+ - reusable schema inference and incremental loads.
171
+ - **Experiment tracking & model registry:** runs, dataset fingerprints, dev → staging →
172
+ production promotion with metric gates, audit history and rollback.
173
+ - **Deployment:**
174
+ - batch scoring and a REST API (API keys, prediction log);
175
+ - bundle, pickle and ONNX export;
176
+ - Docker build contexts and a CLI.
177
+ - **Monitoring:** data and prediction drift (PSI), accuracy decay, alerts and webhooks, and
178
+ champion-vs-challenger retraining.
179
+ - **Insights & reporting:** significance-tested findings, executive reports, and optional
180
+ narratives written by Claude.
181
+
182
+ ## [1.13.0] — 2026-11-06
183
+ - **Feature engineering:**
184
+ - dates, out-of-fold target encoding, interactions, aggregations, lags, text embeddings;
185
+ - feature selection and leakage detection.
186
+ - **AutoML:** task detection, time-budgeted model search, an ensemble, and AutoCluster.
187
+ - **Evaluation & explainability:** metrics with confidence intervals, built-in SHAP, error
188
+ analysis, fairness checks and model cards.
189
+
190
+ ## [1.12.0] — 2026-11-06
191
+ - Profiling & data quality: automated EDA, the 0–100 quality score, YAML data contracts, schema
192
+ diffs and quality gates.
193
+
194
+ ## [1.11.0] — 2026-11-05
195
+ - Shared `Step` / `Pipeline` core and the stateful cleaning pipeline (JSON save/load, audit
196
+ reports, schema checks).
197
+
198
+ ## [1.10.0] and earlier — 2026-07
199
+ - 1.10.0: 8 more data-quality checks in the dashboard.
200
+ - 1.9.0: column pickers in the dashboard.
201
+ - 1.8.x: the dashboard data-quality panel.
202
+ - 1.7.0: the enterprise dashboard redesign.
203
+ - 1.6.x: 43 additional cleaning steps and a dependency cleanup.
204
+ - 1.5.x: the Rust compute backend.
@@ -0,0 +1,40 @@
1
+ # Contributing
2
+
3
+ Thanks for helping! The basics:
4
+
5
+ ## Set up
6
+
7
+ ```bash
8
+ # needs the Rust toolchain: https://rustup.rs
9
+ git clone https://github.com/DataSpoof/dataspoof-data-cleaning.git
10
+ cd dataspoof-data-cleaning
11
+ python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
12
+ pip install maturin
13
+ maturin develop --release --extras dev # builds the Rust extension in place
14
+ ```
15
+
16
+ ## Before you open a pull request
17
+
18
+ ```bash
19
+ pytest # the whole suite; must pass on Polars 1.x and 2.x
20
+ mypy # type-checks the package (config in pyproject.toml)
21
+ mkdocs build --strict # if you changed the docs
22
+ ```
23
+
24
+ - **Tests:** add tests for every change. Tests must run offline; set `AC_OFFLINE=1` to check.
25
+ Use `pytest.importorskip` for optional dependencies.
26
+ - **New pipeline steps:** subclass `mlpilot.Step` and register it with
27
+ `@register_step`. Learn in `_fit` (return JSON-serialisable state) and replay in
28
+ `_transform`. Steps that drop rows must implement `row_mask()`.
29
+ - **`core/` stays independent:** it must not import from the rest of the package. A test
30
+ enforces this.
31
+ - **Imports stay light:** don't import optional dependencies at module import time; import
32
+ them where they're used, and raise an `ImportError` that names the extra.
33
+ - **Persistence:** nothing new may be pickled except trained estimators, which are checksummed.
34
+ - **Changelog:** add an entry to `CHANGELOG.md` under the unreleased version.
35
+
36
+ ## Releases (maintainers)
37
+
38
+ 1. Bump the version in `pyproject.toml` and `Cargo.toml`, and date the changelog entry.
39
+ 2. Tag `vX.Y.Z` and push the tag. CI tests, builds and attests the wheels for every platform,
40
+ then publishes them to PyPI with Trusted Publishing.