dataspoof-mlpilot 2.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dataspoof_mlpilot-2.1.0/.dockerignore +11 -0
- dataspoof_mlpilot-2.1.0/.github/workflows/CI.yml +271 -0
- dataspoof_mlpilot-2.1.0/.github/workflows/docs.yml +47 -0
- dataspoof_mlpilot-2.1.0/.gitignore +26 -0
- dataspoof_mlpilot-2.1.0/BUILD.md +54 -0
- dataspoof_mlpilot-2.1.0/CHANGELOG.md +204 -0
- dataspoof_mlpilot-2.1.0/CONTRIBUTING.md +40 -0
- dataspoof_mlpilot-2.1.0/Cargo.lock +358 -0
- dataspoof_mlpilot-2.1.0/Cargo.toml +26 -0
- dataspoof_mlpilot-2.1.0/PKG-INFO +1822 -0
- dataspoof_mlpilot-2.1.0/README.md +1707 -0
- dataspoof_mlpilot-2.1.0/SECURITY.md +50 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/Chart.yaml +6 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/_helpers.tpl +13 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/deployment.yaml +80 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/ingress.yaml +29 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/pdb.yaml +11 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/pvc.yaml +15 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/secret.yaml +11 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/service.yaml +10 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/templates/serviceaccount.yaml +11 -0
- dataspoof_mlpilot-2.1.0/deploy/helm/mlpilot-hub/values.yaml +58 -0
- dataspoof_mlpilot-2.1.0/deploy/hub/Dockerfile +24 -0
- dataspoof_mlpilot-2.1.0/deploy/hub/docker-compose.yml +33 -0
- dataspoof_mlpilot-2.1.0/deploy/monitoring/docker-compose.yml +49 -0
- dataspoof_mlpilot-2.1.0/deploy/monitoring/grafana/dashboards/mlpilot-model.json +1173 -0
- dataspoof_mlpilot-2.1.0/deploy/monitoring/grafana/provisioning/dashboards/mlpilot.yml +9 -0
- dataspoof_mlpilot-2.1.0/deploy/monitoring/grafana/provisioning/datasources/prometheus.yml +10 -0
- dataspoof_mlpilot-2.1.0/deploy/monitoring/prometheus/alerts.yml +117 -0
- dataspoof_mlpilot-2.1.0/deploy/monitoring/prometheus/alerts_test.yml +110 -0
- dataspoof_mlpilot-2.1.0/deploy/monitoring/prometheus/prometheus.yml +23 -0
- dataspoof_mlpilot-2.1.0/deploy/opa/policies/mlpilot/authz.rego +39 -0
- dataspoof_mlpilot-2.1.0/deploy/opa/policies/mlpilot/config/data.json +8 -0
- dataspoof_mlpilot-2.1.0/deploy/opa/policies/mlpilot/policy_test.rego +128 -0
- dataspoof_mlpilot-2.1.0/deploy/opa/policies/mlpilot/promotion.rego +66 -0
- dataspoof_mlpilot-2.1.0/deploy/terraform/aws/main.tf +194 -0
- dataspoof_mlpilot-2.1.0/deploy/terraform/aws/outputs.tf +20 -0
- dataspoof_mlpilot-2.1.0/deploy/terraform/aws/variables.tf +109 -0
- dataspoof_mlpilot-2.1.0/deploy/terraform/aws-ecs/main.tf +293 -0
- dataspoof_mlpilot-2.1.0/deploy/terraform/aws-ecs/outputs.tf +15 -0
- dataspoof_mlpilot-2.1.0/deploy/terraform/aws-ecs/variables.tf +126 -0
- dataspoof_mlpilot-2.1.0/deploy/terraform/aws-sagemaker/main.tf +232 -0
- dataspoof_mlpilot-2.1.0/deploy/terraform/aws-sagemaker/outputs.tf +16 -0
- dataspoof_mlpilot-2.1.0/deploy/terraform/aws-sagemaker/variables.tf +128 -0
- dataspoof_mlpilot-2.1.0/docs/architecture.md +32 -0
- dataspoof_mlpilot-2.1.0/docs/assets/workflow.svg +19 -0
- dataspoof_mlpilot-2.1.0/docs/changelog.md +1 -0
- dataspoof_mlpilot-2.1.0/docs/contributing.md +1 -0
- dataspoof_mlpilot-2.1.0/docs/guide/anomaly.md +16 -0
- dataspoof_mlpilot-2.1.0/docs/guide/automl.md +78 -0
- dataspoof_mlpilot-2.1.0/docs/guide/aws.md +31 -0
- dataspoof_mlpilot-2.1.0/docs/guide/causal.md +20 -0
- dataspoof_mlpilot-2.1.0/docs/guide/cleaning.md +186 -0
- dataspoof_mlpilot-2.1.0/docs/guide/deployment.md +64 -0
- dataspoof_mlpilot-2.1.0/docs/guide/environments-cicd.md +28 -0
- dataspoof_mlpilot-2.1.0/docs/guide/evaluation.md +78 -0
- dataspoof_mlpilot-2.1.0/docs/guide/feature-store.md +26 -0
- dataspoof_mlpilot-2.1.0/docs/guide/features.md +50 -0
- dataspoof_mlpilot-2.1.0/docs/guide/forecasting.md +17 -0
- dataspoof_mlpilot-2.1.0/docs/guide/hub.md +58 -0
- dataspoof_mlpilot-2.1.0/docs/guide/ingestion.md +62 -0
- dataspoof_mlpilot-2.1.0/docs/guide/insights.md +36 -0
- dataspoof_mlpilot-2.1.0/docs/guide/integrations.md +73 -0
- dataspoof_mlpilot-2.1.0/docs/guide/lineage.md +29 -0
- dataspoof_mlpilot-2.1.0/docs/guide/media.md +32 -0
- dataspoof_mlpilot-2.1.0/docs/guide/model-security.md +31 -0
- dataspoof_mlpilot-2.1.0/docs/guide/monitoring.md +38 -0
- dataspoof_mlpilot-2.1.0/docs/guide/observability.md +33 -0
- dataspoof_mlpilot-2.1.0/docs/guide/quality.md +100 -0
- dataspoof_mlpilot-2.1.0/docs/guide/registry.md +33 -0
- dataspoof_mlpilot-2.1.0/docs/guide/secrets-policy.md +34 -0
- dataspoof_mlpilot-2.1.0/docs/guide/splitting.md +24 -0
- dataspoof_mlpilot-2.1.0/docs/guide/stats.md +28 -0
- dataspoof_mlpilot-2.1.0/docs/guide/versioning.md +23 -0
- dataspoof_mlpilot-2.1.0/docs/index.md +105 -0
- dataspoof_mlpilot-2.1.0/docs/installation.md +89 -0
- dataspoof_mlpilot-2.1.0/docs/policies.md +40 -0
- dataspoof_mlpilot-2.1.0/docs/reference/anomaly.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/automl.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/causal.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/cicd.md +3 -0
- dataspoof_mlpilot-2.1.0/docs/reference/cleaners.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/core.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/deploy.md +9 -0
- dataspoof_mlpilot-2.1.0/docs/reference/environments.md +3 -0
- dataspoof_mlpilot-2.1.0/docs/reference/evaluation.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/feature_store.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/features.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/forecast.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/hub.md +9 -0
- dataspoof_mlpilot-2.1.0/docs/reference/ingestion.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/insights.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/integrations.md +29 -0
- dataspoof_mlpilot-2.1.0/docs/reference/lineage.md +3 -0
- dataspoof_mlpilot-2.1.0/docs/reference/media.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/monitoring.md +9 -0
- dataspoof_mlpilot-2.1.0/docs/reference/poisoning.md +3 -0
- dataspoof_mlpilot-2.1.0/docs/reference/quality.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/registry.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/robustness.md +3 -0
- dataspoof_mlpilot-2.1.0/docs/reference/scale.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/secretstore.md +3 -0
- dataspoof_mlpilot-2.1.0/docs/reference/security.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/splitting.md +3 -0
- dataspoof_mlpilot-2.1.0/docs/reference/stats.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/reference/versioning.md +5 -0
- dataspoof_mlpilot-2.1.0/docs/security.md +1 -0
- dataspoof_mlpilot-2.1.0/docs/workflow.md +24 -0
- dataspoof_mlpilot-2.1.0/mkdocs.yml +138 -0
- dataspoof_mlpilot-2.1.0/mlpilot/__init__.py +60 -0
- dataspoof_mlpilot-2.1.0/mlpilot/_compat.py +64 -0
- dataspoof_mlpilot-2.1.0/mlpilot/_detect.py +136 -0
- dataspoof_mlpilot-2.1.0/mlpilot/_rustcore.pyi +22 -0
- dataspoof_mlpilot-2.1.0/mlpilot/anomaly.py +137 -0
- dataspoof_mlpilot-2.1.0/mlpilot/automl/__init__.py +47 -0
- dataspoof_mlpilot-2.1.0/mlpilot/automl/_persist.py +92 -0
- dataspoof_mlpilot-2.1.0/mlpilot/automl/automl.py +630 -0
- dataspoof_mlpilot-2.1.0/mlpilot/automl/cluster.py +169 -0
- dataspoof_mlpilot-2.1.0/mlpilot/automl/frame.py +67 -0
- dataspoof_mlpilot-2.1.0/mlpilot/automl/models.py +147 -0
- dataspoof_mlpilot-2.1.0/mlpilot/automl/tasks.py +76 -0
- dataspoof_mlpilot-2.1.0/mlpilot/causal.py +227 -0
- dataspoof_mlpilot-2.1.0/mlpilot/cicd.py +396 -0
- dataspoof_mlpilot-2.1.0/mlpilot/cleaners.py +584 -0
- dataspoof_mlpilot-2.1.0/mlpilot/cleaning.py +1368 -0
- dataspoof_mlpilot-2.1.0/mlpilot/core/__init__.py +28 -0
- dataspoof_mlpilot-2.1.0/mlpilot/core/pipeline.py +516 -0
- dataspoof_mlpilot-2.1.0/mlpilot/core/registry.py +91 -0
- dataspoof_mlpilot-2.1.0/mlpilot/core/step.py +150 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/__init__.py +18 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/__main__.py +5 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/_names.py +7 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/asgi.py +11 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/aws.py +187 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/cli.py +284 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/export.py +285 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/scoring.py +179 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/server.py +753 -0
- dataspoof_mlpilot-2.1.0/mlpilot/deploy/wsgi.py +53 -0
- dataspoof_mlpilot-2.1.0/mlpilot/environments.py +240 -0
- dataspoof_mlpilot-2.1.0/mlpilot/evaluation/__init__.py +23 -0
- dataspoof_mlpilot-2.1.0/mlpilot/evaluation/_slicing.py +90 -0
- dataspoof_mlpilot-2.1.0/mlpilot/evaluation/adapter.py +127 -0
- dataspoof_mlpilot-2.1.0/mlpilot/evaluation/card.py +604 -0
- dataspoof_mlpilot-2.1.0/mlpilot/evaluation/errors.py +177 -0
- dataspoof_mlpilot-2.1.0/mlpilot/evaluation/explain.py +312 -0
- dataspoof_mlpilot-2.1.0/mlpilot/evaluation/fairness.py +377 -0
- dataspoof_mlpilot-2.1.0/mlpilot/evaluation/metrics.py +309 -0
- dataspoof_mlpilot-2.1.0/mlpilot/feature_store.py +313 -0
- dataspoof_mlpilot-2.1.0/mlpilot/features/__init__.py +33 -0
- dataspoof_mlpilot-2.1.0/mlpilot/features/dates.py +91 -0
- dataspoof_mlpilot-2.1.0/mlpilot/features/defaults.py +55 -0
- dataspoof_mlpilot-2.1.0/mlpilot/features/encoding.py +130 -0
- dataspoof_mlpilot-2.1.0/mlpilot/features/generators.py +184 -0
- dataspoof_mlpilot-2.1.0/mlpilot/features/leakage.py +221 -0
- dataspoof_mlpilot-2.1.0/mlpilot/features/selection.py +119 -0
- dataspoof_mlpilot-2.1.0/mlpilot/features/text.py +133 -0
- dataspoof_mlpilot-2.1.0/mlpilot/forecast.py +331 -0
- dataspoof_mlpilot-2.1.0/mlpilot/hub/__init__.py +22 -0
- dataspoof_mlpilot-2.1.0/mlpilot/hub/app.py +700 -0
- dataspoof_mlpilot-2.1.0/mlpilot/hub/client.py +142 -0
- dataspoof_mlpilot-2.1.0/mlpilot/hub/db.py +171 -0
- dataspoof_mlpilot-2.1.0/mlpilot/hub/policy.py +82 -0
- dataspoof_mlpilot-2.1.0/mlpilot/hub/security.py +180 -0
- dataspoof_mlpilot-2.1.0/mlpilot/hub/storage.py +103 -0
- dataspoof_mlpilot-2.1.0/mlpilot/hub/ui.html +71 -0
- dataspoof_mlpilot-2.1.0/mlpilot/hub/ui.js +150 -0
- dataspoof_mlpilot-2.1.0/mlpilot/ingestion/__init__.py +23 -0
- dataspoof_mlpilot-2.1.0/mlpilot/ingestion/_util.py +107 -0
- dataspoof_mlpilot-2.1.0/mlpilot/ingestion/api.py +178 -0
- dataspoof_mlpilot-2.1.0/mlpilot/ingestion/core.py +136 -0
- dataspoof_mlpilot-2.1.0/mlpilot/ingestion/files.py +144 -0
- dataspoof_mlpilot-2.1.0/mlpilot/ingestion/incremental.py +174 -0
- dataspoof_mlpilot-2.1.0/mlpilot/ingestion/schema.py +68 -0
- dataspoof_mlpilot-2.1.0/mlpilot/ingestion/sql.py +217 -0
- dataspoof_mlpilot-2.1.0/mlpilot/ingestion/warehouse.py +134 -0
- dataspoof_mlpilot-2.1.0/mlpilot/insights/__init__.py +15 -0
- dataspoof_mlpilot-2.1.0/mlpilot/insights/finder.py +340 -0
- dataspoof_mlpilot-2.1.0/mlpilot/insights/narrate.py +75 -0
- dataspoof_mlpilot-2.1.0/mlpilot/insights/report.py +147 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/__init__.py +20 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/airflow.py +63 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/aws_glue.py +211 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/azure_data_factory.py +117 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/dagster.py +35 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/dvc.py +272 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/evidently.py +140 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/feast.py +166 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/kafka.py +96 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/mlflow.py +296 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/prefect.py +29 -0
- dataspoof_mlpilot-2.1.0/mlpilot/integrations/tasks.py +95 -0
- dataspoof_mlpilot-2.1.0/mlpilot/lineage.py +312 -0
- dataspoof_mlpilot-2.1.0/mlpilot/media/__init__.py +19 -0
- dataspoof_mlpilot-2.1.0/mlpilot/media/audio.py +279 -0
- dataspoof_mlpilot-2.1.0/mlpilot/media/image.py +353 -0
- dataspoof_mlpilot-2.1.0/mlpilot/monitoring/__init__.py +22 -0
- dataspoof_mlpilot-2.1.0/mlpilot/monitoring/drift.py +275 -0
- dataspoof_mlpilot-2.1.0/mlpilot/monitoring/estimate.py +178 -0
- dataspoof_mlpilot-2.1.0/mlpilot/monitoring/monitor.py +490 -0
- dataspoof_mlpilot-2.1.0/mlpilot/monitoring/ops.py +130 -0
- dataspoof_mlpilot-2.1.0/mlpilot/poisoning.py +225 -0
- dataspoof_mlpilot-2.1.0/mlpilot/py.typed +0 -0
- dataspoof_mlpilot-2.1.0/mlpilot/quality/__init__.py +25 -0
- dataspoof_mlpilot-2.1.0/mlpilot/quality/checks.py +547 -0
- dataspoof_mlpilot-2.1.0/mlpilot/quality/contracts.py +458 -0
- dataspoof_mlpilot-2.1.0/mlpilot/quality/pipeline_steps.py +115 -0
- dataspoof_mlpilot-2.1.0/mlpilot/quality/profile.py +328 -0
- dataspoof_mlpilot-2.1.0/mlpilot/quality/schema.py +65 -0
- dataspoof_mlpilot-2.1.0/mlpilot/registry/__init__.py +15 -0
- dataspoof_mlpilot-2.1.0/mlpilot/registry/_store.py +141 -0
- dataspoof_mlpilot-2.1.0/mlpilot/registry/models.py +280 -0
- dataspoof_mlpilot-2.1.0/mlpilot/registry/tracking.py +308 -0
- dataspoof_mlpilot-2.1.0/mlpilot/robustness.py +259 -0
- dataspoof_mlpilot-2.1.0/mlpilot/scale.py +236 -0
- dataspoof_mlpilot-2.1.0/mlpilot/secretstore.py +232 -0
- dataspoof_mlpilot-2.1.0/mlpilot/security.py +142 -0
- dataspoof_mlpilot-2.1.0/mlpilot/splitting.py +370 -0
- dataspoof_mlpilot-2.1.0/mlpilot/stats.py +430 -0
- dataspoof_mlpilot-2.1.0/mlpilot/steps.py +926 -0
- dataspoof_mlpilot-2.1.0/mlpilot/versioning.py +270 -0
- dataspoof_mlpilot-2.1.0/pyproject.toml +222 -0
- dataspoof_mlpilot-2.1.0/src/clean.rs +63 -0
- dataspoof_mlpilot-2.1.0/src/lib.rs +195 -0
- dataspoof_mlpilot-2.1.0/src/stats.rs +125 -0
- dataspoof_mlpilot-2.1.0/src/text.rs +99 -0
- dataspoof_mlpilot-2.1.0/tests/test_advanced.py +302 -0
- dataspoof_mlpilot-2.1.0/tests/test_automl.py +173 -0
- dataspoof_mlpilot-2.1.0/tests/test_aws_deploy.py +97 -0
- dataspoof_mlpilot-2.1.0/tests/test_cicd.py +89 -0
- dataspoof_mlpilot-2.1.0/tests/test_cleaners.py +184 -0
- dataspoof_mlpilot-2.1.0/tests/test_core.py +220 -0
- dataspoof_mlpilot-2.1.0/tests/test_data_versions_and_feature_store.py +143 -0
- dataspoof_mlpilot-2.1.0/tests/test_deploy.py +180 -0
- dataspoof_mlpilot-2.1.0/tests/test_docker_stacks.py +409 -0
- dataspoof_mlpilot-2.1.0/tests/test_environments.py +93 -0
- dataspoof_mlpilot-2.1.0/tests/test_evaluation.py +279 -0
- dataspoof_mlpilot-2.1.0/tests/test_features.py +246 -0
- dataspoof_mlpilot-2.1.0/tests/test_hub.py +264 -0
- dataspoof_mlpilot-2.1.0/tests/test_ingestion.py +387 -0
- dataspoof_mlpilot-2.1.0/tests/test_insights.py +121 -0
- dataspoof_mlpilot-2.1.0/tests/test_label_free_monitoring.py +101 -0
- dataspoof_mlpilot-2.1.0/tests/test_lineage_and_catalogs.py +202 -0
- dataspoof_mlpilot-2.1.0/tests/test_media.py +177 -0
- dataspoof_mlpilot-2.1.0/tests/test_model_security.py +105 -0
- dataspoof_mlpilot-2.1.0/tests/test_monitoring.py +132 -0
- dataspoof_mlpilot-2.1.0/tests/test_ops.py +313 -0
- dataspoof_mlpilot-2.1.0/tests/test_quality.py +333 -0
- dataspoof_mlpilot-2.1.0/tests/test_real_integrations.py +397 -0
- dataspoof_mlpilot-2.1.0/tests/test_registry.py +122 -0
- dataspoof_mlpilot-2.1.0/tests/test_rename_compat.py +31 -0
- dataspoof_mlpilot-2.1.0/tests/test_secretstore.py +72 -0
- dataspoof_mlpilot-2.1.0/tests/test_serving_feedback_reload.py +165 -0
- dataspoof_mlpilot-2.1.0/tests/test_splitting.py +90 -0
- dataspoof_mlpilot-2.1.0/tests/test_stats.py +102 -0
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
# Test, type-check, audit, build wheels for Linux / macOS / Windows, and publish to PyPI.
|
|
2
|
+
#
|
|
3
|
+
# - Every push / PR: tests (OS x Python x Polars matrix), mypy, dependency + Rust audits,
|
|
4
|
+
# wheels and sdist, so breakage is caught before a release.
|
|
5
|
+
# - A pushed version tag (e.g. `v2.0.0`): also publish to PyPI with Trusted Publishing (no
|
|
6
|
+
# API token) and signed build-provenance attestations, plus a CycloneDX SBOM on the release.
|
|
7
|
+
#
|
|
8
|
+
# The crate uses pyo3's `abi3-py311` feature, so one wheel per platform covers CPython 3.11+.
|
|
9
|
+
|
|
10
|
+
name: CI
|
|
11
|
+
|
|
12
|
+
on:
|
|
13
|
+
push:
|
|
14
|
+
branches: [main]
|
|
15
|
+
tags: ["v*"]
|
|
16
|
+
pull_request:
|
|
17
|
+
workflow_dispatch:
|
|
18
|
+
|
|
19
|
+
permissions:
|
|
20
|
+
contents: read
|
|
21
|
+
|
|
22
|
+
concurrency:
|
|
23
|
+
group: ci-${{ github.ref }}
|
|
24
|
+
cancel-in-progress: ${{ !startsWith(github.ref, 'refs/tags/') }}
|
|
25
|
+
|
|
26
|
+
jobs:
|
|
27
|
+
test:
|
|
28
|
+
name: test (${{ matrix.os }}, py${{ matrix.python }}, polars ${{ matrix.polars }})
|
|
29
|
+
runs-on: ${{ matrix.os }}
|
|
30
|
+
strategy:
|
|
31
|
+
fail-fast: false
|
|
32
|
+
matrix:
|
|
33
|
+
os: [ubuntu-latest]
|
|
34
|
+
python: ["3.11", "3.12", "3.13"]
|
|
35
|
+
polars: ["1", "2"]
|
|
36
|
+
include:
|
|
37
|
+
- { os: windows-latest, python: "3.12", polars: "2" }
|
|
38
|
+
- { os: macos-latest, python: "3.12", polars: "2" }
|
|
39
|
+
env:
|
|
40
|
+
AC_OFFLINE: "1" # the library must work without network access
|
|
41
|
+
defaults:
|
|
42
|
+
run:
|
|
43
|
+
shell: bash
|
|
44
|
+
steps:
|
|
45
|
+
- uses: actions/checkout@v4
|
|
46
|
+
- uses: actions/setup-python@v5
|
|
47
|
+
with:
|
|
48
|
+
python-version: ${{ matrix.python }}
|
|
49
|
+
cache: pip
|
|
50
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
51
|
+
- uses: Swatinem/rust-cache@v2
|
|
52
|
+
- name: Install (builds the Rust extension)
|
|
53
|
+
run: |
|
|
54
|
+
python -m pip install --upgrade pip
|
|
55
|
+
python -m pip install ".[test,sql,excel,server,optuna,skops,signing]" "polars==${{ matrix.polars }}.*"
|
|
56
|
+
- name: Run tests
|
|
57
|
+
# Run against the installed package (not the source tree) so packaging bugs show up.
|
|
58
|
+
run: |
|
|
59
|
+
mkdir _run
|
|
60
|
+
cp -r tests _run/
|
|
61
|
+
if [ -f titanic.csv ]; then cp titanic.csv _run/; fi
|
|
62
|
+
cd _run
|
|
63
|
+
python -m pytest tests -q -p no:cacheprovider -o pythonpath=
|
|
64
|
+
|
|
65
|
+
core-only:
|
|
66
|
+
name: test (core install, no extras)
|
|
67
|
+
runs-on: ubuntu-latest
|
|
68
|
+
env:
|
|
69
|
+
AC_OFFLINE: "1"
|
|
70
|
+
steps:
|
|
71
|
+
- uses: actions/checkout@v4
|
|
72
|
+
- uses: actions/setup-python@v5
|
|
73
|
+
with:
|
|
74
|
+
python-version: "3.12"
|
|
75
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
76
|
+
- run: python -m pip install . pytest
|
|
77
|
+
- name: Import must not load heavy optional packages or touch the network
|
|
78
|
+
run: |
|
|
79
|
+
cd /tmp && python -c "
|
|
80
|
+
import sys, mlpilot
|
|
81
|
+
heavy = [m for m in ('nltk','plotly','matplotlib','presidio_analyzer','langchain_anthropic') if m in sys.modules]
|
|
82
|
+
assert not heavy, heavy
|
|
83
|
+
print('ok')"
|
|
84
|
+
- run: mkdir _run && cp -r tests _run/ && cd _run && python -m pytest tests -q -p no:cacheprovider -o pythonpath=
|
|
85
|
+
|
|
86
|
+
integrations:
|
|
87
|
+
# Real MLflow server + `mlflow models serve`, real git/DVC with a remote, real Feast.
|
|
88
|
+
name: test (real MLflow, DVC, Feast)
|
|
89
|
+
runs-on: ubuntu-latest
|
|
90
|
+
steps:
|
|
91
|
+
- uses: actions/checkout@v4
|
|
92
|
+
- uses: actions/setup-python@v5
|
|
93
|
+
with:
|
|
94
|
+
python-version: "3.12"
|
|
95
|
+
cache: pip
|
|
96
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
97
|
+
- uses: Swatinem/rust-cache@v2
|
|
98
|
+
- run: python -m pip install ".[test,mlflow,dvc,feast,evidently,aws,server,signing]" "moto[server]"
|
|
99
|
+
- run: |
|
|
100
|
+
git config --global user.email ci@example.com
|
|
101
|
+
git config --global user.name ci
|
|
102
|
+
mkdir _run
|
|
103
|
+
cp -r tests _run/
|
|
104
|
+
cd _run
|
|
105
|
+
python -m pytest -q -p no:cacheprovider -o pythonpath= -rs tests/test_real_integrations.py \
|
|
106
|
+
tests/test_aws_deploy.py tests/test_lineage_and_catalogs.py tests/test_secretstore.py \
|
|
107
|
+
tests/test_label_free_monitoring.py
|
|
108
|
+
|
|
109
|
+
docker-stacks:
|
|
110
|
+
# Real Prometheus + Grafana, Vault, OPA (+ `opa test`, `promtool test rules`), Marquez, actionlint.
|
|
111
|
+
name: test (Prometheus, Grafana, Vault, OPA, Marquez in Docker)
|
|
112
|
+
runs-on: ubuntu-latest
|
|
113
|
+
env:
|
|
114
|
+
AC_DOCKER_TESTS: "1"
|
|
115
|
+
steps:
|
|
116
|
+
- uses: actions/checkout@v4
|
|
117
|
+
- uses: actions/setup-python@v5
|
|
118
|
+
with:
|
|
119
|
+
python-version: "3.12"
|
|
120
|
+
cache: pip
|
|
121
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
122
|
+
- uses: Swatinem/rust-cache@v2
|
|
123
|
+
- run: python -m pip install ".[test,server]"
|
|
124
|
+
- run: python -m pytest tests/test_docker_stacks.py -q -p no:cacheprovider -rs
|
|
125
|
+
|
|
126
|
+
typecheck:
|
|
127
|
+
runs-on: ubuntu-latest
|
|
128
|
+
steps:
|
|
129
|
+
- uses: actions/checkout@v4
|
|
130
|
+
- uses: actions/setup-python@v5
|
|
131
|
+
with:
|
|
132
|
+
python-version: "3.12"
|
|
133
|
+
- run: python -m pip install mypy polars numpy pandas
|
|
134
|
+
- run: python -m mypy
|
|
135
|
+
|
|
136
|
+
security:
|
|
137
|
+
name: security audit
|
|
138
|
+
runs-on: ubuntu-latest
|
|
139
|
+
steps:
|
|
140
|
+
- uses: actions/checkout@v4
|
|
141
|
+
- uses: actions/setup-python@v5
|
|
142
|
+
with:
|
|
143
|
+
python-version: "3.12"
|
|
144
|
+
- uses: dtolnay/rust-toolchain@stable
|
|
145
|
+
- name: Python dependency vulnerabilities (pip-audit)
|
|
146
|
+
run: |
|
|
147
|
+
python -m pip install pip-audit
|
|
148
|
+
python -m pip install ".[all]"
|
|
149
|
+
pip-audit --skip-editable --progress-spinner off
|
|
150
|
+
- name: Rust dependency vulnerabilities (cargo audit)
|
|
151
|
+
run: |
|
|
152
|
+
cargo install cargo-audit --locked
|
|
153
|
+
cargo generate-lockfile
|
|
154
|
+
cargo audit
|
|
155
|
+
|
|
156
|
+
linux:
|
|
157
|
+
runs-on: ubuntu-latest
|
|
158
|
+
strategy:
|
|
159
|
+
fail-fast: false
|
|
160
|
+
matrix:
|
|
161
|
+
target: [x86_64, aarch64]
|
|
162
|
+
steps:
|
|
163
|
+
- uses: actions/checkout@v4
|
|
164
|
+
- uses: actions/setup-python@v5
|
|
165
|
+
with:
|
|
166
|
+
python-version: "3.11"
|
|
167
|
+
- name: Build wheels
|
|
168
|
+
uses: PyO3/maturin-action@v1
|
|
169
|
+
with:
|
|
170
|
+
target: ${{ matrix.target }}
|
|
171
|
+
args: --release --out dist
|
|
172
|
+
sccache: "true"
|
|
173
|
+
manylinux: auto
|
|
174
|
+
- uses: actions/upload-artifact@v4
|
|
175
|
+
with:
|
|
176
|
+
name: wheels-linux-${{ matrix.target }}
|
|
177
|
+
path: dist
|
|
178
|
+
|
|
179
|
+
windows:
|
|
180
|
+
runs-on: windows-latest
|
|
181
|
+
steps:
|
|
182
|
+
- uses: actions/checkout@v4
|
|
183
|
+
- uses: actions/setup-python@v5
|
|
184
|
+
with:
|
|
185
|
+
python-version: "3.11"
|
|
186
|
+
architecture: x64
|
|
187
|
+
- name: Build wheels
|
|
188
|
+
uses: PyO3/maturin-action@v1
|
|
189
|
+
with:
|
|
190
|
+
target: x64
|
|
191
|
+
args: --release --out dist
|
|
192
|
+
sccache: "true"
|
|
193
|
+
- uses: actions/upload-artifact@v4
|
|
194
|
+
with:
|
|
195
|
+
name: wheels-windows-x64
|
|
196
|
+
path: dist
|
|
197
|
+
|
|
198
|
+
macos:
|
|
199
|
+
runs-on: macos-latest
|
|
200
|
+
strategy:
|
|
201
|
+
fail-fast: false
|
|
202
|
+
matrix:
|
|
203
|
+
target: [x86_64, aarch64]
|
|
204
|
+
steps:
|
|
205
|
+
- uses: actions/checkout@v4
|
|
206
|
+
- uses: actions/setup-python@v5
|
|
207
|
+
with:
|
|
208
|
+
python-version: "3.11"
|
|
209
|
+
- name: Build wheels
|
|
210
|
+
uses: PyO3/maturin-action@v1
|
|
211
|
+
with:
|
|
212
|
+
target: ${{ matrix.target }}
|
|
213
|
+
args: --release --out dist
|
|
214
|
+
sccache: "true"
|
|
215
|
+
- uses: actions/upload-artifact@v4
|
|
216
|
+
with:
|
|
217
|
+
name: wheels-macos-${{ matrix.target }}
|
|
218
|
+
path: dist
|
|
219
|
+
|
|
220
|
+
sdist:
|
|
221
|
+
runs-on: ubuntu-latest
|
|
222
|
+
steps:
|
|
223
|
+
- uses: actions/checkout@v4
|
|
224
|
+
- name: Build sdist
|
|
225
|
+
uses: PyO3/maturin-action@v1
|
|
226
|
+
with:
|
|
227
|
+
command: sdist
|
|
228
|
+
args: --out dist
|
|
229
|
+
- uses: actions/upload-artifact@v4
|
|
230
|
+
with:
|
|
231
|
+
name: wheels-sdist
|
|
232
|
+
path: dist
|
|
233
|
+
|
|
234
|
+
# Publishes only when a version tag (v*) is pushed, and only if every check passed.
|
|
235
|
+
release:
|
|
236
|
+
name: Release to PyPI
|
|
237
|
+
runs-on: ubuntu-latest
|
|
238
|
+
if: startsWith(github.ref, 'refs/tags/v')
|
|
239
|
+
needs: [test, core-only, typecheck, security, linux, windows, macos, sdist]
|
|
240
|
+
environment:
|
|
241
|
+
name: pypi
|
|
242
|
+
url: https://pypi.org/project/dataspoof-mlpilot/
|
|
243
|
+
permissions:
|
|
244
|
+
id-token: write # PyPI Trusted Publishing (OIDC) + Sigstore signing
|
|
245
|
+
attestations: write # build-provenance attestations
|
|
246
|
+
contents: write # attach the SBOM to the GitHub release
|
|
247
|
+
steps:
|
|
248
|
+
- uses: actions/checkout@v4
|
|
249
|
+
- uses: actions/download-artifact@v4
|
|
250
|
+
with:
|
|
251
|
+
pattern: wheels-*
|
|
252
|
+
merge-multiple: true
|
|
253
|
+
path: dist
|
|
254
|
+
- name: Build provenance attestation (signed, verifiable with `gh attestation verify`)
|
|
255
|
+
uses: actions/attest-build-provenance@v2
|
|
256
|
+
with:
|
|
257
|
+
subject-path: dist/*
|
|
258
|
+
- name: Software bill of materials (CycloneDX)
|
|
259
|
+
run: |
|
|
260
|
+
python -m pip install cyclonedx-bom
|
|
261
|
+
python -m pip install dist/*manylinux*x86_64*.whl
|
|
262
|
+
cyclonedx-py environment --of JSON -o sbom.cdx.json
|
|
263
|
+
- name: Publish to PyPI
|
|
264
|
+
# Trusted Publishing; also uploads PEP 740 attestations for every file.
|
|
265
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
266
|
+
with:
|
|
267
|
+
skip-existing: true
|
|
268
|
+
- name: GitHub release with SBOM
|
|
269
|
+
env:
|
|
270
|
+
GH_TOKEN: ${{ github.token }}
|
|
271
|
+
run: gh release create "${GITHUB_REF_NAME}" dist/* sbom.cdx.json --generate-notes --verify-tag || gh release upload "${GITHUB_REF_NAME}" sbom.cdx.json --clobber
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Build the documentation site on every push / PR; deploy it to GitHub Pages from main.
|
|
2
|
+
# One-time setup: repository Settings -> Pages -> Source: "GitHub Actions".
|
|
3
|
+
name: docs
|
|
4
|
+
|
|
5
|
+
on:
|
|
6
|
+
push:
|
|
7
|
+
branches: [main]
|
|
8
|
+
pull_request:
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
|
|
11
|
+
permissions:
|
|
12
|
+
contents: read
|
|
13
|
+
|
|
14
|
+
concurrency:
|
|
15
|
+
group: docs-${{ github.ref }}
|
|
16
|
+
cancel-in-progress: true
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
build:
|
|
20
|
+
runs-on: ubuntu-latest
|
|
21
|
+
steps:
|
|
22
|
+
- uses: actions/checkout@v4
|
|
23
|
+
- uses: actions/setup-python@v5
|
|
24
|
+
with:
|
|
25
|
+
python-version: "3.12"
|
|
26
|
+
- run: python -m pip install mkdocs-material "mkdocstrings[python]"
|
|
27
|
+
- run: mkdocs build --strict
|
|
28
|
+
env:
|
|
29
|
+
NO_MKDOCS_2_WARNING: "1"
|
|
30
|
+
- uses: actions/upload-pages-artifact@v3
|
|
31
|
+
if: github.event_name != 'pull_request'
|
|
32
|
+
with:
|
|
33
|
+
path: site
|
|
34
|
+
|
|
35
|
+
deploy:
|
|
36
|
+
if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
|
|
37
|
+
needs: build
|
|
38
|
+
runs-on: ubuntu-latest
|
|
39
|
+
permissions:
|
|
40
|
+
pages: write
|
|
41
|
+
id-token: write
|
|
42
|
+
environment:
|
|
43
|
+
name: github-pages
|
|
44
|
+
url: ${{ steps.deployment.outputs.page_url }}
|
|
45
|
+
steps:
|
|
46
|
+
- id: deployment
|
|
47
|
+
uses: actions/deploy-pages@v4
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
pypi/
|
|
2
|
+
venv/
|
|
3
|
+
dist/
|
|
4
|
+
build/
|
|
5
|
+
*.egg-info/
|
|
6
|
+
*.log
|
|
7
|
+
output/
|
|
8
|
+
*.csv
|
|
9
|
+
|
|
10
|
+
# Rust build artifacts
|
|
11
|
+
/target
|
|
12
|
+
Cargo.lock
|
|
13
|
+
|
|
14
|
+
# Compiled Rust extension dropped into the package for in-place testing
|
|
15
|
+
mlpilot/*.pyd
|
|
16
|
+
mlpilot/*.so
|
|
17
|
+
|
|
18
|
+
# Python bytecode
|
|
19
|
+
__pycache__/
|
|
20
|
+
*.pyc
|
|
21
|
+
|
|
22
|
+
# Documentation build output
|
|
23
|
+
site/
|
|
24
|
+
|
|
25
|
+
# Windows debug symbols from local Rust builds
|
|
26
|
+
*.pdb
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# Build & Architecture
|
|
2
|
+
|
|
3
|
+
This library uses a **Rust compute backend** with a **Python frontend**, packaged
|
|
4
|
+
as a single `pip`-installable wheel via [maturin](https://www.maturin.rs/) + [PyO3](https://pyo3.rs/).
|
|
5
|
+
|
|
6
|
+
## Layout
|
|
7
|
+
|
|
8
|
+
```
|
|
9
|
+
Cargo.toml # Rust crate (cdylib -> _rustcore)
|
|
10
|
+
pyproject.toml # maturin build backend + Python deps
|
|
11
|
+
src/
|
|
12
|
+
lib.rs # PyO3 module: exposes functions to Python
|
|
13
|
+
text.rs # text preprocessing (contractions, regex, stopwords)
|
|
14
|
+
stats.rs # skewness, IQR, correlation, multicollinearity
|
|
15
|
+
mlpilot/
|
|
16
|
+
__init__.py # from .cleaning import *
|
|
17
|
+
cleaning.py # Python frontend (I/O, ML, dashboard, PII, AI)
|
|
18
|
+
_rustcore.*.pyd # compiled Rust extension (built artifact)
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## What runs where
|
|
22
|
+
|
|
23
|
+
| Concern | Location | Why |
|
|
24
|
+
|---|---|---|
|
|
25
|
+
| Text cleaning, symbol stripping, IQR outliers, skewness, correlation / multicollinearity, negatives, JSON detection | **Rust** (`_rustcore`) | CPU-bound, parallelised with `rayon`, GIL released |
|
|
26
|
+
| WordNet lemmatization | Python | preserves exact NLTK output |
|
|
27
|
+
| CSV/Parquet I/O, KNN imputation, class balancing | Python (Polars / scikit-learn) | mature, no benefit to reimplementing |
|
|
28
|
+
| Plotly dashboard, Presidio PII masking, Claude spell-correction | Python | ecosystem-only |
|
|
29
|
+
|
|
30
|
+
The Python functions delegate the heavy loops to `_rustcore`; results are verified
|
|
31
|
+
to match Polars to machine precision (skewness, IQR bounds, correlation drop-list).
|
|
32
|
+
|
|
33
|
+
## Build
|
|
34
|
+
|
|
35
|
+
Prereqs: Rust toolchain (`rustup`) and `maturin` (`pip install maturin`).
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
# Build a release wheel (dist-ready, abi3 -> works on CPython 3.11+)
|
|
39
|
+
maturin build --release
|
|
40
|
+
# -> target/wheels/mlpilot-1.5.1-cp311-abi3-win_amd64.whl
|
|
41
|
+
|
|
42
|
+
# Install it
|
|
43
|
+
pip install target/wheels/mlpilot-1.5.1-cp311-abi3-win_amd64.whl
|
|
44
|
+
|
|
45
|
+
# --- OR, for development (requires an active virtualenv) ---
|
|
46
|
+
python -m venv .venv && .venv/Scripts/activate # Windows
|
|
47
|
+
maturin develop --release
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
## Publish to PyPI
|
|
51
|
+
|
|
52
|
+
`maturin build --release` produces the wheel; upload with `twine upload target/wheels/*`
|
|
53
|
+
(or `maturin publish`). For cross-platform wheels, build in CI with
|
|
54
|
+
`PyO3/maturin-action`.
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes are listed here. The project follows [Semantic Versioning](https://semver.org/)
|
|
4
|
+
(see the [versioning policy](https://dataspoof.github.io/dataspoof-data-cleaning/policies/)).
|
|
5
|
+
|
|
6
|
+
## [2.1.0] — 2026-11-09
|
|
7
|
+
|
|
8
|
+
### Renamed: AutomatedCleaning is now MLPilot
|
|
9
|
+
- **Names:**
|
|
10
|
+
- the distribution is `dataspoof-mlpilot` (PyPI refused `mlpilot` as too similar to an existing project);
|
|
11
|
+
- the import is `import mlpilot`, and the CLI is `mlpilot`;
|
|
12
|
+
- `MLPilotWarning` replaces `AutomatedCleaningWarning`, which remains as an alias.
|
|
13
|
+
- **Compatibility:**
|
|
14
|
+
- `import automatedcleaning` and pickles that refer to `automatedcleaning.*` classes resolve to
|
|
15
|
+
the same `mlpilot` modules, so models saved by AutomatedCleaning keep loading with the same
|
|
16
|
+
classes;
|
|
17
|
+
- the `AutomatedCleaning` project on PyPI gets a final 2.0.1 that installs `dataspoof-mlpilot`.
|
|
18
|
+
- **Unchanged:** the `AC_*` environment variables, the `ac_*` Prometheus metrics and the saved-file
|
|
19
|
+
formats (`dataspoof.*`).
|
|
20
|
+
- **Renamed with the library:**
|
|
21
|
+
- the Helm chart (`mlpilot-hub`), the OPA policy package (`data.mlpilot`) and the Prometheus job;
|
|
22
|
+
- the Grafana dashboard file;
|
|
23
|
+
- the MLflow tag prefix and the default OpenLineage namespace.
|
|
24
|
+
|
|
25
|
+
## [2.0.0] — 2026-11-09
|
|
26
|
+
|
|
27
|
+
### Breaking
|
|
28
|
+
- **Lean core install.** `pip install AutomatedCleaning` now installs only the core
|
|
29
|
+
dependencies. The heavy ones moved to extras:
|
|
30
|
+
- `[nlp]`: nltk
|
|
31
|
+
- `[dashboard]`: plotly, matplotlib, missingno, pyfiglet
|
|
32
|
+
- `[pii]`: presidio
|
|
33
|
+
- `[llm]`: anthropic, langchain-anthropic
|
|
34
|
+
|
|
35
|
+
To get what 1.x installed, use `pip install "AutomatedCleaning[all]"`. A feature whose extra
|
|
36
|
+
is missing raises an `ImportError` naming the extra to install.
|
|
37
|
+
- `setup.py` was removed. The build is defined only in `pyproject.toml` (maturin).
|
|
38
|
+
|
|
39
|
+
### Changed
|
|
40
|
+
- **No side effects at import.**
|
|
41
|
+
- Importing the library no longer downloads NLTK data, changes the application's warning
|
|
42
|
+
filters, or reconfigures logging.
|
|
43
|
+
- NLTK corpora are downloaded on first use, and never when `AC_OFFLINE` is set. Text cleaning
|
|
44
|
+
falls back to a built-in stopword list when NLTK isn't installed.
|
|
45
|
+
- **Typed package.** It ships a `py.typed` marker and stubs for the Rust extension, and passes
|
|
46
|
+
`mypy`.
|
|
47
|
+
- **CI** now runs:
|
|
48
|
+
- tests across Linux, macOS and Windows × Python 3.11–3.13 × Polars 1/2;
|
|
49
|
+
- a core-only install test;
|
|
50
|
+
- mypy;
|
|
51
|
+
- pip-audit and cargo-audit.
|
|
52
|
+
|
|
53
|
+
Releases publish with PyPI Trusted Publishing, signed build-provenance attestations and a
|
|
54
|
+
CycloneDX SBOM.
|
|
55
|
+
- Interactive AI corrections use `ANTHROPIC_API_KEY` when it is set, and the current Claude model.
|
|
56
|
+
|
|
57
|
+
### Added
|
|
58
|
+
- **Hub, the multi-user platform server** (`automatedcleaning.hub`, `[server]` extra):
|
|
59
|
+
- workspaces, with SSO through OIDC and identity-provider group mapping;
|
|
60
|
+
- API tokens and service accounts;
|
|
61
|
+
- four roles (viewer, data scientist, approver, admin);
|
|
62
|
+
- runs and a model registry with immutable, checksummed artifacts on disk, S3, GCS or Azure;
|
|
63
|
+
- two-person production approvals with policy gates;
|
|
64
|
+
- monitoring results;
|
|
65
|
+
- a hash-chained, verifiable and exportable audit log;
|
|
66
|
+
- a web UI, the Python client `HubClient`, and `automatedcleaning hub serve | create-admin`.
|
|
67
|
+
- **Hub deployment:** a Docker image with compose (plus PostgreSQL), a Helm chart, and a
|
|
68
|
+
Terraform module for AWS (S3, RDS, Secrets Manager, IAM role, Helm release).
|
|
69
|
+
- **Forecasting** (`Forecaster`):
|
|
70
|
+
- multiple series and multi-step horizons;
|
|
71
|
+
- rolling-origin backtests against seasonal-naive;
|
|
72
|
+
- conformal intervals and hierarchical reconciliation.
|
|
73
|
+
- **Causal inference** (`estimate_effect`): doubly-robust AIPW with overlap and balance
|
|
74
|
+
diagnostics. **Uplift:** `UpliftModel` and `qini`.
|
|
75
|
+
- **Anomaly detection** (`AutoAnomaly`): an ensemble of three detectors with plain-language
|
|
76
|
+
explanations.
|
|
77
|
+
- **AutoML:**
|
|
78
|
+
- conformal `predict_interval` and `predict_set`;
|
|
79
|
+
- `search="optuna"`, `imbalance="auto"`, `n_jobs`;
|
|
80
|
+
- `predict_with_proba` and `set_inference_threads`.
|
|
81
|
+
- **Serving:**
|
|
82
|
+
- Prometheus `/metrics` and OpenTelemetry tracing;
|
|
83
|
+
- canary and shadow rollouts, and `app_from_registry`;
|
|
84
|
+
- SageMaker, Vertex AI and Azure ML routes and Docker targets;
|
|
85
|
+
- an ASGI app and the `container` entry point;
|
|
86
|
+
- single-row latency down from about 500 ms to about 50 ms.
|
|
87
|
+
- **Security:**
|
|
88
|
+
- Ed25519 model signing, verified before unpickling whenever `AC_MODEL_PUBLIC_KEY` is set;
|
|
89
|
+
- `skops` export;
|
|
90
|
+
- signing on registration with `register(..., sign_with=...)`.
|
|
91
|
+
- **Monitoring:**
|
|
92
|
+
- PagerDuty and Opsgenie alert handlers;
|
|
93
|
+
- `join_labels` and `Monitor.check_labels` for labels that arrive later;
|
|
94
|
+
- `sql_sink` and `Monitor.on_check`;
|
|
95
|
+
- `SQLSource.insert` and `execute`.
|
|
96
|
+
- **Scale:**
|
|
97
|
+
- `transform_file` for files of any size;
|
|
98
|
+
- Spark, Ray and Dask scoring;
|
|
99
|
+
- `to_sql` SQL pushdown of fitted cleaning pipelines.
|
|
100
|
+
- **Integrations:** Airflow operators, Prefect tasks and flow, Dagster assets, Kafka stream
|
|
101
|
+
scoring, and plain `integrations.tasks`.
|
|
102
|
+
- **MLflow** (`integrations.mlflow`, `[mlflow]` extra):
|
|
103
|
+
- batched, idempotent export of runs and whole trackers;
|
|
104
|
+
- AutoML models logged as pyfunc models with a signature, so they serve with `mlflow models serve`;
|
|
105
|
+
- `export_registry` (stages become aliases) and `load_automl`.
|
|
106
|
+
- **DVC** (`integrations.dvc`, `[dvc]` extra): `DVCRepo` (init, remotes, track, push and pull,
|
|
107
|
+
read any revision, checkout, `log_to_run`) and `export_repository` for `DataRepository`
|
|
108
|
+
history.
|
|
109
|
+
- **Feast** (`integrations.feast`, `[feast]` extra): `export_to_feast` writes, applies and
|
|
110
|
+
materializes a Feast repository.
|
|
111
|
+
- **Data splitting** (`split_data`, `cv_folds`, `check_split`): stratified, grouped and time-ordered
|
|
112
|
+
splits with gaps; every split is leak-checked (shared groups, time overlap, duplicate rows, unseen classes).
|
|
113
|
+
- **Model security:** `robustness_report` (noise stability, a minimal adversarial attack,
|
|
114
|
+
feature sensitivity, extreme inputs) and `poisoning_report` (confident-learning label errors,
|
|
115
|
+
conflicting duplicates, backdoor triggers, outliers).
|
|
116
|
+
- **Environments** (`Environments`, `environments.yml`): isolated dev / QA / prod registries;
|
|
117
|
+
`release` enforces order, source stage, metric gates, four-eyes approvals and signatures, and
|
|
118
|
+
records provenance on both sides. The registry and Hub gain a `qa` stage.
|
|
119
|
+
- **Model CI/CD:** the CLI commands `train`, `gate`, `register`, `promote`, `release`, `latest`,
|
|
120
|
+
`deploy-sagemaker` and `cicd init` (GitHub Actions / GitLab CI pipelines, checked with actionlint).
|
|
121
|
+
- **Secrets** (`secretstore`): `vault://` (token, AppRole, Kubernetes; KV v1 and v2), `aws-sm://`,
|
|
122
|
+
`env://` and `file://` references, resolved wherever credentials are read.
|
|
123
|
+
- **Open Policy Agent for the Hub** (`hub.policy`): deny-overrides access rules, promotion rules
|
|
124
|
+
and fail-closed behaviour, plus Rego policies with unit tests in `deploy/opa`.
|
|
125
|
+
- **Serving:**
|
|
126
|
+
- `POST /feedback` with prediction IDs, live-accuracy metrics and `join_feedback`;
|
|
127
|
+
- hot reload (`watch=`, `POST /admin/reload`, registry-following);
|
|
128
|
+
- live drift, estimated-performance and alert gauges (`monitor=`).
|
|
129
|
+
- **Monitoring before labels:** `PerformanceEstimator` (CBPE, direct loss estimation),
|
|
130
|
+
`domain_classifier_drift`, and `estimated_performance` alerts in `Monitor.check`.
|
|
131
|
+
- **Grafana + Prometheus** (`deploy/monitoring`): a dashboard, 14 alert rules with `promtool`
|
|
132
|
+
unit tests, and a compose stack.
|
|
133
|
+
- **Lineage** (`lineage`): OpenLineage run events with schema, quality and version facets; HTTP
|
|
134
|
+
and file transports; `emit_tracked_run`, `emit_dataset_version`, `LineageGraph`.
|
|
135
|
+
- **Catalogs and orchestration:** AWS Glue (catalog tables, partitions, reads, job runs) and Azure
|
|
136
|
+
Data Factory pipeline runs.
|
|
137
|
+
- **AWS deployment** (`deploy.aws`): ECR push and blue/green SageMaker endpoints (serverless or
|
|
138
|
+
instances, autoscaling, data capture, rollback alarms); Terraform for SageMaker and ECS Fargate.
|
|
139
|
+
- **Evidently** (`integrations.evidently`): reports plus agreement with the built-in drift report.
|
|
140
|
+
- **Tests against real infrastructure:** Prometheus, Grafana, Vault, OPA and Marquez in Docker
|
|
141
|
+
(`AC_DOCKER_TESTS=1`); moto for AWS; dedicated CI jobs.
|
|
142
|
+
- These three are tested against the real tools (MLflow server and model server, git + DVC
|
|
143
|
+
with a remote, Feast), including in a dedicated CI job.
|
|
144
|
+
- **Image and audio data** (`media`):
|
|
145
|
+
- quality audits (corrupt files, exact and near duplicates, blur, exposure, clipping, silence,
|
|
146
|
+
mixed formats);
|
|
147
|
+
- preprocessors (EXIF-aware image resize and normalise; audio resample, trim and level);
|
|
148
|
+
- augmentation;
|
|
149
|
+
- features for AutoML (colour and edge-orientation histograms; MFCC and spectral features).
|
|
150
|
+
- **Statistical testing** (`stats`):
|
|
151
|
+
- automatic test selection with assumption checks, effect sizes and confidence intervals;
|
|
152
|
+
- chi-square and Fisher's exact test, correlations;
|
|
153
|
+
- A/B tests, power analysis, multiple-testing correction, `test_all`.
|
|
154
|
+
- **Data version control** (`DataRepository`): content-addressed commits, tags, row-level
|
|
155
|
+
diffs, checksum-verified checkout, links to experiment runs, garbage collection.
|
|
156
|
+
- **Feature store** (`FeatureStore`, `Entity`, `FeatureView`): point-in-time-correct
|
|
157
|
+
historical joins with TTL, an SQLite online store, `materialize`, schema guards.
|
|
158
|
+
- **Documentation site** (MkDocs Material with an API reference), plus `SECURITY.md`,
|
|
159
|
+
`CONTRIBUTING.md` and a versioning policy.
|
|
160
|
+
|
|
161
|
+
### Removed
|
|
162
|
+
- `automatedcleaning/cleaning_update.py`: dead code that nothing imported.
|
|
163
|
+
|
|
164
|
+
## [1.14.0] — 2026-11-08
|
|
165
|
+
- **Ingestion & connectors:**
|
|
166
|
+
- files, globs, HTTP, and S3 / GCS / Azure;
|
|
167
|
+
- any SQL database (bound parameters);
|
|
168
|
+
- Snowflake and BigQuery;
|
|
169
|
+
- REST APIs with five pagination styles;
|
|
170
|
+
- reusable schema inference and incremental loads.
|
|
171
|
+
- **Experiment tracking & model registry:** runs, dataset fingerprints, dev → staging →
|
|
172
|
+
production promotion with metric gates, audit history and rollback.
|
|
173
|
+
- **Deployment:**
|
|
174
|
+
- batch scoring and a REST API (API keys, prediction log);
|
|
175
|
+
- bundle, pickle and ONNX export;
|
|
176
|
+
- Docker build contexts and a CLI.
|
|
177
|
+
- **Monitoring:** data and prediction drift (PSI), accuracy decay, alerts and webhooks, and
|
|
178
|
+
champion-vs-challenger retraining.
|
|
179
|
+
- **Insights & reporting:** significance-tested findings, executive reports, and optional
|
|
180
|
+
narratives written by Claude.
|
|
181
|
+
|
|
182
|
+
## [1.13.0] — 2026-11-06
|
|
183
|
+
- **Feature engineering:**
|
|
184
|
+
- dates, out-of-fold target encoding, interactions, aggregations, lags, text embeddings;
|
|
185
|
+
- feature selection and leakage detection.
|
|
186
|
+
- **AutoML:** task detection, time-budgeted model search, an ensemble, and AutoCluster.
|
|
187
|
+
- **Evaluation & explainability:** metrics with confidence intervals, built-in SHAP, error
|
|
188
|
+
analysis, fairness checks and model cards.
|
|
189
|
+
|
|
190
|
+
## [1.12.0] — 2026-11-06
|
|
191
|
+
- Profiling & data quality: automated EDA, the 0–100 quality score, YAML data contracts, schema
|
|
192
|
+
diffs and quality gates.
|
|
193
|
+
|
|
194
|
+
## [1.11.0] — 2026-11-05
|
|
195
|
+
- Shared `Step` / `Pipeline` core and the stateful cleaning pipeline (JSON save/load, audit
|
|
196
|
+
reports, schema checks).
|
|
197
|
+
|
|
198
|
+
## [1.10.0] and earlier — 2026-07
|
|
199
|
+
- 1.10.0: 8 more data-quality checks in the dashboard.
|
|
200
|
+
- 1.9.0: column pickers in the dashboard.
|
|
201
|
+
- 1.8.x: the dashboard data-quality panel.
|
|
202
|
+
- 1.7.0: the enterprise dashboard redesign.
|
|
203
|
+
- 1.6.x: 43 additional cleaning steps and a dependency cleanup.
|
|
204
|
+
- 1.5.x: the Rust compute backend.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
Thanks for helping! The basics:
|
|
4
|
+
|
|
5
|
+
## Set up
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
# needs the Rust toolchain: https://rustup.rs
|
|
9
|
+
git clone https://github.com/DataSpoof/dataspoof-data-cleaning.git
|
|
10
|
+
cd dataspoof-data-cleaning
|
|
11
|
+
python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate
|
|
12
|
+
pip install maturin
|
|
13
|
+
maturin develop --release --extras dev # builds the Rust extension in place
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Before you open a pull request
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
pytest # the whole suite; must pass on Polars 1.x and 2.x
|
|
20
|
+
mypy # type-checks the package (config in pyproject.toml)
|
|
21
|
+
mkdocs build --strict # if you changed the docs
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
- **Tests:** add tests for every change. Tests must run offline; set `AC_OFFLINE=1` to check.
|
|
25
|
+
Use `pytest.importorskip` for optional dependencies.
|
|
26
|
+
- **New pipeline steps:** subclass `mlpilot.Step` and register it with
|
|
27
|
+
`@register_step`. Learn in `_fit` (return JSON-serialisable state) and replay in
|
|
28
|
+
`_transform`. Steps that drop rows must implement `row_mask()`.
|
|
29
|
+
- **`core/` stays independent:** it must not import from the rest of the package. A test
|
|
30
|
+
enforces this.
|
|
31
|
+
- **Imports stay light:** don't import optional dependencies at module import time; import
|
|
32
|
+
them where they're used, and raise an `ImportError` that names the extra.
|
|
33
|
+
- **Persistence:** nothing new may be pickled except trained estimators, which are checksummed.
|
|
34
|
+
- **Changelog:** add an entry to `CHANGELOG.md` under the unreleased version.
|
|
35
|
+
|
|
36
|
+
## Releases (maintainers)
|
|
37
|
+
|
|
38
|
+
1. Bump the version in `pyproject.toml` and `Cargo.toml`, and date the changelog entry.
|
|
39
|
+
2. Tag `vX.Y.Z` and push the tag. CI tests, builds and attests the wheels for every platform,
|
|
40
|
+
then publishes them to PyPI with Trusted Publishing.
|