de-agent 0.2.4__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {de_agent-0.2.4 → de_agent-0.3.3}/PKG-INFO +7 -7
- {de_agent-0.2.4 → de_agent-0.3.3}/README.md +1 -1
- {de_agent-0.2.4 → de_agent-0.3.3}/pyproject.toml +6 -6
- de_agent-0.3.3/src/de_agent/__init__.py +1 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/cli.py +161 -5
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/aws/__init__.py +3 -2
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/aws/_aws_core.py +93 -6
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/base.py +1 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/pack_base.py +34 -5
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/postgres.py +85 -4
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/redshift.py +235 -8
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/config.py +17 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/data/sql_cases.yaml +41 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/diff.py +59 -7
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/find.py +18 -4
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/lint.py +87 -21
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/bigquery.yaml +14 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/check.py +25 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/databricks-system.yaml +14 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/model.py +1 -1
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/mssql.yaml +15 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/mysql.yaml +21 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/postgres-lite.yaml +35 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/snowflake.yaml +18 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/view.py +5 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/render.py +44 -3
- de_agent-0.3.3/src/de_agent/routines.py +269 -0
- de_agent-0.3.3/src/de_agent/routines_render.py +178 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/runner.py +5 -3
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/airflow.py +121 -5
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/catalog.py +33 -2
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/common.py +1 -1
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/edges.py +190 -27
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/headers.py +3 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/pyexpr.py +1 -1
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/python.py +4 -1
- de_agent-0.3.3/src/de_agent/scanners/routines.py +237 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/run.py +36 -4
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/sql.py +9 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/terraform.py +104 -7
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/secretscan.py +7 -2
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/snapshot.py +36 -3
- de_agent-0.3.3/tests/fixtures/range_loops_repo/jobs/loops.py +205 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_bigquery.md +9 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_databricks-system.md +9 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_mssql.md +9 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_mysql.md +10 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_postgres-lite.md +9 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_snowflake.md +9 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/redshift_degraded.md +10 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/redshift_healthy.md +10 -0
- de_agent-0.3.3/tests/golden/schedule_cron.txt +2 -0
- de_agent-0.3.3/tests/golden/schedule_launchd.plist +30 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/packs_fixtures.py +25 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/redshift_fixtures.py +32 -0
- de_agent-0.3.3/tests/test_aws_dms_mappings.py +142 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_cli_guard.py +3 -1
- de_agent-0.3.3/tests/test_config_routine_edges.py +155 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_demo_e2e.py +45 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_diff.py +4 -2
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_evals.py +22 -0
- de_agent-0.3.3/tests/test_export_schedule.py +115 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_find.py +7 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_hooks.py +151 -1
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_lint.py +77 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_packs_live.py +44 -2
- de_agent-0.3.3/tests/test_packs_routines.py +109 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_postgres_collector.py +119 -0
- de_agent-0.3.3/tests/test_pyexpr_recursion.py +75 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_redshift_collector.py +140 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_redteam.py +2 -1
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_release_meta.py +4 -0
- de_agent-0.3.3/tests/test_routines_core.py +206 -0
- de_agent-0.3.3/tests/test_routines_edges.py +291 -0
- de_agent-0.3.3/tests/test_routines_redteam.py +134 -0
- de_agent-0.3.3/tests/test_routines_render.py +163 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_scan_v02.py +20 -0
- de_agent-0.3.3/tests/test_self_guard_script_writes.py +102 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_session_hook.py +31 -0
- de_agent-0.3.3/tests/test_status_dates.py +84 -0
- de_agent-0.3.3/tests/test_terraform_dms.py +138 -0
- de_agent-0.2.4/src/de_agent/__init__.py +0 -1
- {de_agent-0.2.4 → de_agent-0.3.3}/.gitignore +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/LICENSE +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/NOTICE +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/applyplan.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/approval.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/cliclass.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/clients.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/__init__.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/_airflow_core.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/_pack_http_core.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/airflow.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/dbt.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/pack_cli.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/pack_http.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/collectors/pack_sql.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/configdoc.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/extract.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/hubpath.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/initcmd.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/__init__.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/argv.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/cli.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/dagster.yaml +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/databricks.yaml +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/errors.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/expr.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/fivetran.yaml +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/gcp.yaml +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/kubernetes.yaml +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/loader.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/scaffold.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/packs/template.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/plans.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/pyscan.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/__init__.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/dbt.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/scanners/pycode.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/sqlclass.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/sqlexec.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/stepexec.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/example/integrations/orders_etl--warehouse.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/example/map_row.txt +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/example/projects/orders_etl.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/hub/README.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/hub/conventions.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/hub/de-agent.yaml +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/hub/domains/_template.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/hub/gitignore.tmpl +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/hub/integrations/_template.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/hub/map.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/hub/projects/_template.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/src/de_agent/templates/hub/status.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/aws_fixtures.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/conftest.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/corpus/cli_guard.yaml +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/corpus/redteam.yaml +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_2_10/failing.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_2_10/healthy.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_3_0/basic.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/common/__init__.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/common/pg_utils.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/call_proc.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/datasets_2x.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/factory.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/nested_with.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/ops/nested_folder.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/sensors.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/sql/load_x.sql +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/sql_file.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/sql_hook.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/sql_kwarg.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/taskflow.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/unrolled.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/dags/util_imports.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/airflow_repo/utils/dates.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/aws_snapshot.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/dbt/failing/manifest.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/dbt/failing/run_results.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/dbt/failing/sources.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/dbt/healthy/manifest.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/dbt/healthy/run_results.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/dbt/healthy/sources.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/dbt_proj/target/manifest.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/pg_full.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/pg_healthy.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/pg_partial.json +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/python_repo/jobs/broken.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/python_repo/jobs/load_orders.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/python_repo/jobs/pandas_job.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/python_repo/jobs/spark_job.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/python_repo/lib/__init__.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/python_repo/lib/etl.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/python_repo/notebooks/dbx_export.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/python_repo/notebooks/orders_rollup.ipynb +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/tf_repo/main.tf +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/tf_repo/modules/warehouse.tf +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/util_repo/dags/util_hops.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/util_repo/dags/util_loop.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/util_repo/dags/util_proc.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/util_repo/sql/procs.sql +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/util_repo/utils/__init__.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/util_repo/utils/db.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/util_repo/utils/etl.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/fixtures/warehouse_sql/procs.sql +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/af3_basic.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/af_failing.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/af_healthy.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/aws_athena_section.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/aws_cloudwatch_section.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/aws_dms_section.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/aws_firehose_section.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/aws_full.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/aws_glue_section.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/aws_s3_section.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/aws_split_index.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/dbt_failing.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/dbt_healthy.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_dagster.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_databricks.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_fivetran.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_gcp.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pack_kubernetes.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pg_full.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pg_healthy.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/pg_partial.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/scan_python.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/scan_terraform.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/scan_util_airflow.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/golden/scan_util_edges.md +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/packs_helpers.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_airflow_collector.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_audit_pack.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_aws_collector.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_aws_guard.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_aws_more.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_aws_render_edges.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_cli.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_closeout_plans.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_cold_imports.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_config.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_dbt_collector.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_denylist.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_docs.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_docs_examples.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_extract.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_packs_builtin.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_packs_cli.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_packs_enforcement.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_packs_expr.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_packs_http.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_plan_steps.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_plans.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_recovered.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_redshift_single_pass.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_release_script.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_render.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_render_m1b.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_scan_airflow.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_scan_perf.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_scan_run.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_snapshot_m3.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_sqlclass.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_sqlexec.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_stop_check.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_test_guard.py +0 -0
- {de_agent-0.2.4 → de_agent-0.3.3}/tests/test_walkthrough.py +0 -0
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: de-agent
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Ground-truth context, guardrails and workflows for data engineering with Claude Code
|
|
5
|
-
Project-URL: Homepage, https://github.com/
|
|
6
|
-
Project-URL: Documentation, https://github.com/
|
|
7
|
-
Project-URL: Repository, https://github.com/
|
|
8
|
-
Project-URL: Issues, https://github.com/
|
|
9
|
-
Project-URL: Changelog, https://github.com/
|
|
5
|
+
Project-URL: Homepage, https://github.com/OWNER/de-agent
|
|
6
|
+
Project-URL: Documentation, https://github.com/OWNER/de-agent#readme
|
|
7
|
+
Project-URL: Repository, https://github.com/OWNER/de-agent
|
|
8
|
+
Project-URL: Issues, https://github.com/OWNER/de-agent/issues
|
|
9
|
+
Project-URL: Changelog, https://github.com/OWNER/de-agent/blob/main/CHANGELOG.md
|
|
10
10
|
Author: de-agent maintainers
|
|
11
11
|
License-Expression: Apache-2.0
|
|
12
12
|
License-File: LICENSE
|
|
@@ -91,4 +91,4 @@ uv tool install 'de-agent[pg,aws]' # or: pipx install 'de-agent[pg,aws]'
|
|
|
91
91
|
dea init --name "My platform" && dea doctor
|
|
92
92
|
```
|
|
93
93
|
|
|
94
|
-
Documentation, the plugin, the safety model and the eval results: <https://github.com/
|
|
94
|
+
Documentation, the plugin, the safety model and the eval results: <https://github.com/OWNER/de-agent>. Apache-2.0.
|
|
@@ -13,4 +13,4 @@ uv tool install 'de-agent[pg,aws]' # or: pipx install 'de-agent[pg,aws]'
|
|
|
13
13
|
dea init --name "My platform" && dea doctor
|
|
14
14
|
```
|
|
15
15
|
|
|
16
|
-
Documentation, the plugin, the safety model and the eval results: <https://github.com/
|
|
16
|
+
Documentation, the plugin, the safety model and the eval results: <https://github.com/OWNER/de-agent>. Apache-2.0.
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "de-agent"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.3"
|
|
8
8
|
description = "Ground-truth context, guardrails and workflows for data engineering with Claude Code"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -41,11 +41,11 @@ all = ["de-agent[aws,pg,mysql,snowflake,bigquery,databricks,mssql,terraform]"]
|
|
|
41
41
|
dev = ["pytest", "pytest-cov", "ruff", "boto3", "moto[all]", "psycopg[binary]>=3.1", "sqlalchemy>=2", "pymysql", "python-hcl2>=4"]
|
|
42
42
|
|
|
43
43
|
[project.urls]
|
|
44
|
-
Homepage = "https://github.com/
|
|
45
|
-
Documentation = "https://github.com/
|
|
46
|
-
Repository = "https://github.com/
|
|
47
|
-
Issues = "https://github.com/
|
|
48
|
-
Changelog = "https://github.com/
|
|
44
|
+
Homepage = "https://github.com/OWNER/de-agent"
|
|
45
|
+
Documentation = "https://github.com/OWNER/de-agent#readme"
|
|
46
|
+
Repository = "https://github.com/OWNER/de-agent"
|
|
47
|
+
Issues = "https://github.com/OWNER/de-agent/issues"
|
|
48
|
+
Changelog = "https://github.com/OWNER/de-agent/blob/main/CHANGELOG.md"
|
|
49
49
|
|
|
50
50
|
[project.scripts]
|
|
51
51
|
dea = "de_agent.cli:app"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.3"
|
|
@@ -243,15 +243,19 @@ def edges(table: str = typer.Argument(..., help="schema.table, e.g. analytics.fc
|
|
|
243
243
|
|
|
244
244
|
@app.command("impact")
|
|
245
245
|
def impact(
|
|
246
|
-
target: str = typer.Argument(..., help="util symbol (module.func) or schema.
|
|
246
|
+
target: str = typer.Argument(..., help="util symbol (module.func), schema.table, or proc:/function:/view:/trigger:<schema.name>"),
|
|
247
247
|
depth: int = typer.Option(5, "--depth", min=1, max=10, help="Max downstream hops for a table (1-10)"),
|
|
248
248
|
) -> None:
|
|
249
|
-
"""Symbol: the DAGs that use it. Table: downstream readers, transitively, with the path.
|
|
250
|
-
|
|
249
|
+
"""Symbol: the DAGs that use it. Table: downstream readers, transitively, with the path (routines list their callers).
|
|
250
|
+
proc:/function:/view:/trigger: <name>: what a live routine reads and writes, who calls it, and everything downstream."""
|
|
251
|
+
from de_agent.scanners.edges import callers, downstream
|
|
251
252
|
|
|
252
253
|
hub = _require_hub()
|
|
253
|
-
key = target.lower()
|
|
254
254
|
data = _load_edges(hub)
|
|
255
|
+
if target.split(":", 1)[0] in ("proc", "function", "view", "trigger") and ":" in target:
|
|
256
|
+
_impact_routine(data, target, depth)
|
|
257
|
+
return
|
|
258
|
+
key = target.lower()
|
|
255
259
|
syms = _load_symbols(hub)
|
|
256
260
|
if key not in data["tables"] and not any(q == target or q.endswith("." + target) for q in syms):
|
|
257
261
|
key = _resolve_table(data, target) or key
|
|
@@ -260,6 +264,9 @@ def impact(
|
|
|
260
264
|
typer.echo(f"impact of table {key} (downstream readers, depth <= {depth}): {len(rows)}")
|
|
261
265
|
for r in sorted(rows, key=lambda r: (r["depth"], r["reader"])):
|
|
262
266
|
typer.echo(f" depth {r['depth']} {r['reader']} [{r['project']}] {' -> '.join(r['path'])}")
|
|
267
|
+
if r["reader"] in data.get("routines", {}):
|
|
268
|
+
for c in callers(data, r["reader"]):
|
|
269
|
+
typer.echo(f" called by {c}")
|
|
263
270
|
return
|
|
264
271
|
match = [q for q in syms if q == target or q.endswith("." + target)]
|
|
265
272
|
if match:
|
|
@@ -269,7 +276,37 @@ def impact(
|
|
|
269
276
|
for d in s["dags"]:
|
|
270
277
|
typer.echo(f" {d['dag_id']} {d['file']}")
|
|
271
278
|
return
|
|
272
|
-
_unknown("table or symbol", target, [*data["tables"], *syms])
|
|
279
|
+
_unknown("table or symbol", target, [*data["tables"], *syms, *sorted(data.get("routines", {}))])
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _impact_routine(data: dict, target: str, depth: int) -> None:
|
|
283
|
+
from de_agent.scanners.edges import callers, downstream, find_routines
|
|
284
|
+
|
|
285
|
+
ids = find_routines(data, target)
|
|
286
|
+
if not ids:
|
|
287
|
+
_unknown("routine", target, sorted(data.get("routines", {})))
|
|
288
|
+
for rid in ids:
|
|
289
|
+
r = data["routines"][rid]
|
|
290
|
+
typer.echo(f"impact of {rid} ({r['kind']}, {r['source']}{r['args'] and ' ' + r['args']}, parse {r['parse']})")
|
|
291
|
+
typer.echo("reads: " + (", ".join(r["reads"]) or "none"))
|
|
292
|
+
typer.echo("writes: " + (", ".join(r["writes"]) or "none"))
|
|
293
|
+
typer.echo("calls: " + (", ".join(r["calls"]) or "none"))
|
|
294
|
+
cs = callers(data, rid)
|
|
295
|
+
typer.echo("called by:" + ("" if cs else " none"))
|
|
296
|
+
for c in cs:
|
|
297
|
+
typer.echo(f" {c}")
|
|
298
|
+
seen: set[tuple[str, str]] = set()
|
|
299
|
+
outs = r["writes"] if r["kind"] != "view" else [r["name"]]
|
|
300
|
+
typer.echo(f"downstream of {'/'.join(outs) or 'nothing it writes'} (depth <= {depth}):")
|
|
301
|
+
for t in outs:
|
|
302
|
+
for row in sorted(downstream(data, t, depth), key=lambda x: (x["depth"], x["reader"])):
|
|
303
|
+
if (row["reader"], t) in seen or row["reader"] == rid:
|
|
304
|
+
continue
|
|
305
|
+
seen.add((row["reader"], t))
|
|
306
|
+
typer.echo(f" depth {row['depth']} {row['reader']} [{row['project']}] {' -> '.join(row['path'])}")
|
|
307
|
+
if row["reader"] in data.get("routines", {}):
|
|
308
|
+
for c in callers(data, row["reader"]):
|
|
309
|
+
typer.echo(f" called by {c}")
|
|
273
310
|
|
|
274
311
|
|
|
275
312
|
def _fmt_cell(v: object) -> str:
|
|
@@ -593,6 +630,125 @@ def doctor() -> None:
|
|
|
593
630
|
raise typer.Exit(1 if any(m == "✗" for m, _, _ in rows) else 0)
|
|
594
631
|
|
|
595
632
|
|
|
633
|
+
export_app = typer.Typer(help="Export snapshot data (read-only, no live calls).", no_args_is_help=True)
|
|
634
|
+
app.add_typer(export_app, name="export")
|
|
635
|
+
|
|
636
|
+
DMS_MAPPING_COLUMNS = ["task", "schema", "table", "target_schema", "target_table", "rule", "source_endpoint", "target_endpoint"]
|
|
637
|
+
|
|
638
|
+
|
|
639
|
+
@export_app.command("dms-mappings")
|
|
640
|
+
def export_dms_mappings(
|
|
641
|
+
csv_path: Path | None = typer.Option(None, "--csv", help="Write the CSV to this file (default: stdout)"),
|
|
642
|
+
source: str | None = typer.Option(None, "--source", help="Only this aws source (default: every aws source with the section)"),
|
|
643
|
+
) -> None:
|
|
644
|
+
"""Export the dms_table_mappings section of the latest snapshot as CSV. No AWS call."""
|
|
645
|
+
import csv
|
|
646
|
+
import io
|
|
647
|
+
|
|
648
|
+
from de_agent.snapshot import list_snapshots, load_result
|
|
649
|
+
|
|
650
|
+
hub = _require_hub()
|
|
651
|
+
cfg = _load(hub)
|
|
652
|
+
names = [s.name for s in cfg.sources if s.type == "aws" and (source is None or s.name == source)]
|
|
653
|
+
snaps = list_snapshots(hub)
|
|
654
|
+
if not snaps:
|
|
655
|
+
typer.echo("no snapshot yet: run `dea snapshot` first", err=True)
|
|
656
|
+
raise typer.Exit(2)
|
|
657
|
+
rows: list[dict] = []
|
|
658
|
+
found = False
|
|
659
|
+
for name in names:
|
|
660
|
+
# newest snapshot that ran this source (a later `dea snapshot --source other` does not hide it)
|
|
661
|
+
res = next((r for sid in reversed(snaps) if (r := load_result(hub, sid, name)) is not None), None)
|
|
662
|
+
data = (res or {}).get("data") or {}
|
|
663
|
+
section = data.get("dms_table_mappings")
|
|
664
|
+
if not isinstance(section, list):
|
|
665
|
+
continue
|
|
666
|
+
found = True
|
|
667
|
+
rows.extend(section)
|
|
668
|
+
if "dms_table_mappings" in ((data.get("meta") or {}).get("truncated") or []):
|
|
669
|
+
typer.echo(f"warning: {name}: list is capped; raise options.dms_mappings_max", err=True)
|
|
670
|
+
if not found:
|
|
671
|
+
typer.echo("no dms_table_mappings section in the latest snapshot (needs an aws source with services.dms)", err=True)
|
|
672
|
+
raise typer.Exit(2)
|
|
673
|
+
buf = io.StringIO()
|
|
674
|
+
w = csv.DictWriter(buf, fieldnames=DMS_MAPPING_COLUMNS, extrasaction="ignore", lineterminator="\n")
|
|
675
|
+
w.writeheader()
|
|
676
|
+
for r in rows:
|
|
677
|
+
w.writerow({k: ("" if r.get(k) is None else r.get(k)) for k in DMS_MAPPING_COLUMNS})
|
|
678
|
+
if csv_path:
|
|
679
|
+
csv_path.write_text(buf.getvalue(), encoding="utf-8")
|
|
680
|
+
typer.echo(f"wrote {csv_path} ({len(rows)} rows)", err=True)
|
|
681
|
+
else:
|
|
682
|
+
sys.stdout.write(buf.getvalue())
|
|
683
|
+
|
|
684
|
+
|
|
685
|
+
schedule_app = typer.Typer(help="Print a job that keeps snapshots fresh (prints only, never installs).", no_args_is_help=True)
|
|
686
|
+
app.add_typer(schedule_app, name="schedule")
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
def _snapshot_wrapper(hub: Path, dea: str, sources: list[str]) -> str:
|
|
690
|
+
import shlex
|
|
691
|
+
|
|
692
|
+
log = shlex.quote(str(hub / ".dea" / "logs" / "snapshot.log"))
|
|
693
|
+
logs = shlex.quote(str(hub / ".dea" / "logs"))
|
|
694
|
+
cmd = shlex.quote(dea) + " snapshot" + "".join(f" --source {shlex.quote(x)}" for x in sources)
|
|
695
|
+
rotate = (f'if [ -f {log} ] && [ "$(wc -c < {log})" -gt 1048576 ]; then i=6; '
|
|
696
|
+
f'while [ $i -ge 1 ]; do [ -f {log}.$i ] && mv {log}.$i {log}.$((i+1)); i=$((i-1)); done; mv {log} {log}.1; fi')
|
|
697
|
+
return f"mkdir -p {logs}; {rotate}; {cmd} >> {log} 2>&1"
|
|
698
|
+
|
|
699
|
+
|
|
700
|
+
@schedule_app.command("print")
|
|
701
|
+
def schedule_print(
|
|
702
|
+
at: str = typer.Option("06:00", "--at", help="Daily run time, HH:MM (local)"),
|
|
703
|
+
sources: str = typer.Option("", "--sources", help="Comma-separated source names (default: all)"),
|
|
704
|
+
os_name: str | None = typer.Option(None, "--os", hidden=True, help="macos or linux (default: this machine)"),
|
|
705
|
+
) -> None:
|
|
706
|
+
"""Print a launchd plist (macOS) or a crontab line (elsewhere) that runs `dea snapshot` daily. Installs nothing."""
|
|
707
|
+
import re
|
|
708
|
+
import shlex
|
|
709
|
+
from xml.sax.saxutils import escape
|
|
710
|
+
|
|
711
|
+
m = re.fullmatch(r"(\d{1,2}):(\d{2})", at)
|
|
712
|
+
if not m or int(m.group(1)) > 23 or int(m.group(2)) > 59:
|
|
713
|
+
typer.echo(f"bad --at {at!r}: expected HH:MM (00:00-23:59)", err=True)
|
|
714
|
+
raise typer.Exit(2)
|
|
715
|
+
hour, minute = int(m.group(1)), int(m.group(2))
|
|
716
|
+
if os_name not in (None, "macos", "linux"):
|
|
717
|
+
typer.echo("bad --os: macos or linux", err=True)
|
|
718
|
+
raise typer.Exit(2)
|
|
719
|
+
hub = _require_hub()
|
|
720
|
+
dea = shutil.which("dea") or str(Path(sys.argv[0]).resolve())
|
|
721
|
+
names = [x.strip() for x in sources.split(",") if x.strip()]
|
|
722
|
+
wrapper = _snapshot_wrapper(hub, dea, names)
|
|
723
|
+
macos = (os_name == "macos") if os_name else sys.platform == "darwin"
|
|
724
|
+
if macos:
|
|
725
|
+
def x(v: object) -> str:
|
|
726
|
+
return escape(str(v))
|
|
727
|
+
|
|
728
|
+
typer.echo("\n".join([
|
|
729
|
+
'<?xml version="1.0" encoding="UTF-8"?>',
|
|
730
|
+
'<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">',
|
|
731
|
+
'<plist version="1.0">', '<dict>',
|
|
732
|
+
' <key>Label</key><string>io.de-agent.snapshot</string>',
|
|
733
|
+
' <key>ProgramArguments</key>', ' <array>',
|
|
734
|
+
' <string>/bin/sh</string>', ' <string>-c</string>', f' <string>{x(wrapper)}</string>', ' </array>',
|
|
735
|
+
' <key>EnvironmentVariables</key>', ' <dict>',
|
|
736
|
+
f' <key>DEA_HUB</key><string>{x(hub)}</string>',
|
|
737
|
+
f' <key>PATH</key><string>{x(os.environ.get("PATH", ""))}</string>', ' </dict>',
|
|
738
|
+
' <key>StartCalendarInterval</key>', ' <dict>',
|
|
739
|
+
f' <key>Hour</key><integer>{hour}</integer>', f' <key>Minute</key><integer>{minute}</integer>', ' </dict>',
|
|
740
|
+
' <key>RunAtLoad</key><false/>', '</dict>', '</plist>',
|
|
741
|
+
"<!--",
|
|
742
|
+
"de-agent prints this and installs nothing. To install it, save it to",
|
|
743
|
+
" ~/Library/LaunchAgents/io.de-agent.snapshot.plist",
|
|
744
|
+
"and run",
|
|
745
|
+
" launchctl load ~/Library/LaunchAgents/io.de-agent.snapshot.plist",
|
|
746
|
+
"-->"]))
|
|
747
|
+
else:
|
|
748
|
+
typer.echo(f"{minute} {hour} * * * DEA_HUB={shlex.quote(str(hub))} /bin/sh -c {shlex.quote(wrapper)}")
|
|
749
|
+
typer.echo("# de-agent prints this and installs nothing. Add the line above with `crontab -e`.")
|
|
750
|
+
|
|
751
|
+
|
|
596
752
|
config_app = typer.Typer(help="de-agent.yaml tooling.", no_args_is_help=True)
|
|
597
753
|
app.add_typer(config_app, name="config")
|
|
598
754
|
|
|
@@ -14,7 +14,7 @@ from typing import Any
|
|
|
14
14
|
from de_agent.collectors.aws import _aws_core as core
|
|
15
15
|
from de_agent.collectors.base import CollectorError, RunContext
|
|
16
16
|
|
|
17
|
-
OPTION_KEYS = {"max_metrics", "max_age_h", "max_scan_gb"}
|
|
17
|
+
OPTION_KEYS = {"max_metrics", "max_age_h", "max_scan_gb", "dms_mappings_max"}
|
|
18
18
|
AWS_KEYS = {"profile", "region", "instance_role"}
|
|
19
19
|
SERVICE_KEYS = {
|
|
20
20
|
"dms": set(), "glue": {"jobs", "crawlers", "catalog_databases"}, "s3": {"prefixes"}, "athena": {"workgroups"},
|
|
@@ -122,7 +122,8 @@ class AwsCollector:
|
|
|
122
122
|
schema_version = 1
|
|
123
123
|
list_keys = {
|
|
124
124
|
"dms_instances": ["id"], "dms_tasks": ["id"], "dms_table_issues": ["task", "schema", "table"],
|
|
125
|
-
"dms_top_tables": ["task", "schema", "table"], "
|
|
125
|
+
"dms_top_tables": ["task", "schema", "table"], "dms_table_mappings": ["task", "schema", "table"],
|
|
126
|
+
"glue_job_runs": ["job", "run_id"], "glue_crawlers": ["name"],
|
|
126
127
|
"glue_tables": ["database", "name"], "s3_prefixes": ["prefix"], "athena_workgroups": ["workgroup"],
|
|
127
128
|
"athena_top_queries": ["workgroup", "query_id"], "athena_failures": ["workgroup", "query_id"],
|
|
128
129
|
"firehose_streams": ["stream"], "cloudwatch_alarms": ["name"],
|
|
@@ -22,6 +22,8 @@ MAX_PARTITIONS = 5000
|
|
|
22
22
|
DEFAULT_MAX_METRICS = 300
|
|
23
23
|
GMD_BATCH = 500 # GetMetricData accepts up to 500 queries per call
|
|
24
24
|
TOP_N = 10
|
|
25
|
+
DMS_STATS_CAP = 200000 # table statistics per task (the mapping list needs every table, not a top-N)
|
|
26
|
+
DEFAULT_DMS_MAPPINGS_MAX = 5000
|
|
25
27
|
S3_METRIC_WINDOW_H = 72 # BucketSizeBytes / NumberOfObjects are daily metrics
|
|
26
28
|
|
|
27
29
|
# ---- read-only guard ---------------------------------------------------------------------------------------------
|
|
@@ -329,7 +331,75 @@ def _rules(table_mappings):
|
|
|
329
331
|
return out
|
|
330
332
|
|
|
331
333
|
|
|
332
|
-
def
|
|
334
|
+
def _like(pattern, value):
|
|
335
|
+
"""DMS object-locator match: `%` is a wildcard, everything else is literal and case-sensitive."""
|
|
336
|
+
rx = ".*".join(re.escape(part) for part in str(pattern if pattern is not None else "%").split("%"))
|
|
337
|
+
return re.fullmatch(rx, str(value or ""), re.S) is not None
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def _transform(rule, name):
|
|
341
|
+
"""Apply one DMS schema/table transformation rule to a name; unknown actions leave it unchanged."""
|
|
342
|
+
action, val = rule.get("action"), rule.get("value")
|
|
343
|
+
if name is None:
|
|
344
|
+
return name
|
|
345
|
+
if action == "rename" and val is not None:
|
|
346
|
+
return val
|
|
347
|
+
if action == "convert-lowercase":
|
|
348
|
+
return name.lower()
|
|
349
|
+
if action == "convert-uppercase":
|
|
350
|
+
return name.upper()
|
|
351
|
+
if action == "add-prefix" and val is not None:
|
|
352
|
+
return val + name
|
|
353
|
+
if action == "add-suffix" and val is not None:
|
|
354
|
+
return name + val
|
|
355
|
+
if action == "remove-prefix" and val and name.startswith(val):
|
|
356
|
+
return name[len(val):]
|
|
357
|
+
if action == "remove-suffix" and val and name.endswith(val):
|
|
358
|
+
return name[:-len(val)]
|
|
359
|
+
return name
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def dms_mapping_rows(task, rules, stat_rows, source_ep, target_ep):
|
|
363
|
+
"""Tables a task's selection rules include, after excludes and schema/table transformation rules, as plain rows.
|
|
364
|
+
|
|
365
|
+
Explicit include rules give their table directly; wildcard includes (`%`) are resolved from the task's table
|
|
366
|
+
statistics, and one row with `table` = the pattern and `(unresolved)` is kept when nothing matches."""
|
|
367
|
+
found = {} # (schema, table) -> rule label; explicit wins over wildcard
|
|
368
|
+
for r in rules:
|
|
369
|
+
if r.get("type") != "selection" or r.get("action") not in ("include", "explicit"):
|
|
370
|
+
continue
|
|
371
|
+
sp, tp = r.get("schema"), r.get("table")
|
|
372
|
+
if sp is not None and tp is not None and "%" not in sp and "%" not in tp:
|
|
373
|
+
found[(sp, tp)] = "explicit"
|
|
374
|
+
continue
|
|
375
|
+
label = "wildcard %s.%s" % (sp if sp is not None else "%", tp if tp is not None else "%")
|
|
376
|
+
hits = [(s.get("SchemaName"), s.get("TableName")) for s in stat_rows
|
|
377
|
+
if _like(sp, s.get("SchemaName")) and _like(tp, s.get("TableName"))]
|
|
378
|
+
for key in hits:
|
|
379
|
+
found.setdefault(key, label)
|
|
380
|
+
if not hits:
|
|
381
|
+
found.setdefault((sp if sp is not None else "%", tp if tp is not None else "%"), label + " (unresolved)")
|
|
382
|
+
excludes = [r for r in rules if r.get("type") == "selection" and r.get("action") == "exclude"]
|
|
383
|
+
rows = []
|
|
384
|
+
for (sch, tbl), label in found.items():
|
|
385
|
+
if any(_like(x.get("schema"), sch) and _like(x.get("table"), tbl) for x in excludes):
|
|
386
|
+
continue
|
|
387
|
+
tgt_s, tgt_t = sch, tbl
|
|
388
|
+
for r in rules:
|
|
389
|
+
if r.get("type") != "transformation" or r.get("target") not in ("schema", "table"):
|
|
390
|
+
continue
|
|
391
|
+
if not _like(r.get("schema"), sch) or (r.get("target") == "table" and not _like(r.get("table"), tbl)):
|
|
392
|
+
continue
|
|
393
|
+
if r.get("target") == "schema":
|
|
394
|
+
tgt_s = _transform(r, tgt_s)
|
|
395
|
+
else:
|
|
396
|
+
tgt_t = _transform(r, tgt_t)
|
|
397
|
+
rows.append({"task": task, "schema": sch, "table": tbl, "target_schema": tgt_s, "target_table": tgt_t,
|
|
398
|
+
"rule": label, "source_endpoint": source_ep, "target_endpoint": target_ep})
|
|
399
|
+
return sorted(rows, key=lambda r: (str(r["schema"]), str(r["table"])))
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def gather_dms(session, cfg, now, metrics, errors, opts=None, truncated=None):
|
|
333
403
|
dms = session.client("dms")
|
|
334
404
|
out = {}
|
|
335
405
|
try:
|
|
@@ -407,7 +477,12 @@ def gather_dms(session, cfg, now, metrics, errors):
|
|
|
407
477
|
"folder": s3.get("BucketFolder")}
|
|
408
478
|
return {k: v for k, v in row.items() if v}
|
|
409
479
|
|
|
410
|
-
|
|
480
|
+
def endpoint_id(arn):
|
|
481
|
+
e = endpoints.get(arn)
|
|
482
|
+
return (e.get("EndpointIdentifier") if e else None) or _arn_tail(arn) or None
|
|
483
|
+
|
|
484
|
+
mappings_max = int((opts or {}).get("dms_mappings_max", DEFAULT_DMS_MAPPINGS_MAX))
|
|
485
|
+
trows, issues, top, mapping = [], [], [], []
|
|
411
486
|
for t in tasks:
|
|
412
487
|
tid = t.get("ReplicationTaskIdentifier")
|
|
413
488
|
stats = t.get("ReplicationTaskStats") or {}
|
|
@@ -427,12 +502,16 @@ def gather_dms(session, cfg, now, metrics, errors):
|
|
|
427
502
|
"rules": _rules(t.get("TableMappings")),
|
|
428
503
|
}
|
|
429
504
|
try:
|
|
430
|
-
stat_rows,
|
|
431
|
-
|
|
432
|
-
|
|
505
|
+
stat_rows, stat_trunc = _paginate(dms, "describe_table_statistics", "TableStatistics", DMS_STATS_CAP,
|
|
506
|
+
ReplicationTaskArn=t.get("ReplicationTaskArn"),
|
|
507
|
+
PaginationConfig={"PageSize": 500})
|
|
508
|
+
if stat_trunc and truncated is not None and "dms_table_mappings" not in truncated:
|
|
509
|
+
truncated.append("dms_table_mappings")
|
|
433
510
|
except Exception as ex:
|
|
434
511
|
errors.append("dms.table_statistics %s: %s" % (tid, describe_error(ex)))
|
|
435
512
|
stat_rows = []
|
|
513
|
+
if truncated is not None and "dms_table_mappings" not in truncated:
|
|
514
|
+
truncated.append("dms_table_mappings") # wildcards unresolved: the list is partial, not shrunk
|
|
436
515
|
n_susp = 0
|
|
437
516
|
cand = []
|
|
438
517
|
for s in stat_rows:
|
|
@@ -446,6 +525,8 @@ def gather_dms(session, cfg, now, metrics, errors):
|
|
|
446
525
|
cand.append(dict(base, state=state, cdc_changes=changes, inserts=int(s.get("Inserts") or 0),
|
|
447
526
|
updates=int(s.get("Updates") or 0), deletes=int(s.get("Deletes") or 0),
|
|
448
527
|
ddls=int(s.get("Ddls") or 0), full_load_rows=int(s.get("FullLoadRows") or 0)))
|
|
528
|
+
mapping += dms_mapping_rows(tid, row["rules"], stat_rows, endpoint_id(t.get("SourceEndpointArn")),
|
|
529
|
+
endpoint_id(t.get("TargetEndpointArn")))
|
|
449
530
|
row["tables_suspended"] = n_susp
|
|
450
531
|
row["tables_total"] = len(stat_rows)
|
|
451
532
|
top += sorted(cand, key=lambda r: (-r["cdc_changes"], str(r["schema"]), str(r["table"])))[:TOP_N]
|
|
@@ -453,6 +534,12 @@ def gather_dms(session, cfg, now, metrics, errors):
|
|
|
453
534
|
out["dms_tasks"] = sorted(trows, key=lambda r: str(r["id"]))
|
|
454
535
|
out["dms_table_issues"] = sorted(issues, key=lambda r: (str(r["task"]), str(r["schema"]), str(r["table"])))
|
|
455
536
|
out["dms_top_tables"] = sorted(top, key=lambda r: (str(r["task"]), -r["cdc_changes"], str(r["table"])))
|
|
537
|
+
mapping.sort(key=lambda r: (str(r["task"]), str(r["schema"]), str(r["table"])))
|
|
538
|
+
if len(mapping) > mappings_max:
|
|
539
|
+
mapping = mapping[:mappings_max]
|
|
540
|
+
if truncated is not None and "dms_table_mappings" not in truncated:
|
|
541
|
+
truncated.append("dms_table_mappings")
|
|
542
|
+
out["dms_table_mappings"] = mapping
|
|
456
543
|
return out, trunc or ttrunc
|
|
457
544
|
|
|
458
545
|
|
|
@@ -744,7 +831,7 @@ def gather(cfg, session=None, now=None):
|
|
|
744
831
|
metrics = Metrics(session, now, WINDOW_H, cap)
|
|
745
832
|
if "dms" in services:
|
|
746
833
|
try:
|
|
747
|
-
part, trunc = gather_dms(session, services["dms"] or {}, now, metrics, errors)
|
|
834
|
+
part, trunc = gather_dms(session, services["dms"] or {}, now, metrics, errors, opts, truncated)
|
|
748
835
|
data.update(part)
|
|
749
836
|
if trunc:
|
|
750
837
|
truncated.append("dms")
|
|
@@ -29,6 +29,7 @@ class RunContext:
|
|
|
29
29
|
prior: dict[str, Any] | None = None # latest usable earlier result of this source (heavy-section reuse)
|
|
30
30
|
deep: bool = False # `dea snapshot --deep`: recompute everything, ignore `prior`
|
|
31
31
|
now: datetime | None = None # injectable clock (tests)
|
|
32
|
+
routine_store: Any = None # de_agent.routines.RoutineStore: where routine bodies go (None: bodies are hashed, not stored)
|
|
32
33
|
pack_fn: Callable[..., Any] | None = None # injectable executor for pack collectors (tests): (section, item, payload) -> result
|
|
33
34
|
|
|
34
35
|
def remaining(self) -> float:
|
|
@@ -13,11 +13,18 @@ from typing import Any
|
|
|
13
13
|
|
|
14
14
|
from de_agent.collectors.base import CollectorError, RunContext
|
|
15
15
|
from de_agent.packs import template
|
|
16
|
-
from de_agent.packs.check import MSG_FIELD, check_pack
|
|
16
|
+
from de_agent.packs.check import MSG_FIELD, ROUTINES_SECTION, check_pack
|
|
17
17
|
from de_agent.packs.errors import PackError
|
|
18
18
|
from de_agent.packs.expr import compile_expr
|
|
19
19
|
from de_agent.packs.loader import load_pack
|
|
20
20
|
from de_agent.packs.model import Pack, Section
|
|
21
|
+
from de_agent.routines import (
|
|
22
|
+
finalize_rows,
|
|
23
|
+
routine_settings,
|
|
24
|
+
split_routine_options,
|
|
25
|
+
validate_routine_options,
|
|
26
|
+
)
|
|
27
|
+
from de_agent.scanners.routines import make_parser
|
|
21
28
|
|
|
22
29
|
MAX_SIGNALS_PER_RULE = 10
|
|
23
30
|
MAX_JSON_BYTES = 5 * 1024 * 1024
|
|
@@ -33,7 +40,7 @@ def load_for_source(source: Any, kind: str) -> Pack:
|
|
|
33
40
|
if pack.kind != kind:
|
|
34
41
|
raise ValueError(f"pack {pack.pack!r} is kind {pack.kind}; a type: {kind} source needs a {kind} pack")
|
|
35
42
|
try:
|
|
36
|
-
template.resolve_params(pack.params, source.options)
|
|
43
|
+
template.resolve_params(pack.params, split_routine_options(source.options)[0])
|
|
37
44
|
except PackError as ex:
|
|
38
45
|
raise ValueError(str(ex)) from None
|
|
39
46
|
return pack
|
|
@@ -57,7 +64,9 @@ def validate_for_config(source: Any, kind: str) -> Pack | None:
|
|
|
57
64
|
if pack.kind != kind:
|
|
58
65
|
raise ValueError(f"pack {pack.pack!r} is kind {pack.kind}; a type: {kind} source needs a {kind} pack")
|
|
59
66
|
try:
|
|
60
|
-
|
|
67
|
+
params, rt = split_routine_options(source.options)
|
|
68
|
+
validate_routine_options(rt)
|
|
69
|
+
template.resolve_params(pack.params, params)
|
|
61
70
|
except PackError as ex:
|
|
62
71
|
raise ValueError(str(ex)) from None
|
|
63
72
|
return pack
|
|
@@ -175,7 +184,9 @@ class PackCollector:
|
|
|
175
184
|
def collect(self, source: Any, ctx: RunContext) -> dict[str, Any]:
|
|
176
185
|
try:
|
|
177
186
|
pack = load_for_source(source, self.type)
|
|
178
|
-
|
|
187
|
+
pack_opts, routine_opts = split_routine_options(source.options)
|
|
188
|
+
validate_routine_options(routine_opts)
|
|
189
|
+
params = template.resolve_params(pack.params, pack_opts)
|
|
179
190
|
except (ValueError, PackError) as ex:
|
|
180
191
|
return {"data": {}, "errors": [str(ex)]}
|
|
181
192
|
problems = check_pack(pack)
|
|
@@ -187,7 +198,8 @@ class PackCollector:
|
|
|
187
198
|
meta: dict[str, Any] = {
|
|
188
199
|
"pack": pack.pack, "pack_version": pack.version, "kind": pack.kind, "validated": pack.validated,
|
|
189
200
|
"list_keys": {s.name: s.keys for s in pack.sections if s.keys},
|
|
190
|
-
"volatile": sorted({v for s in pack.sections for v in s.volatile}
|
|
201
|
+
"volatile": sorted({v for s in pack.sections for v in s.volatile}
|
|
202
|
+
| ({"owner", "reads", "writes", "calls", "parse", "body_bytes"} if any(s.name == ROUTINES_SECTION for s in pack.sections) else set())),
|
|
191
203
|
"sections": {s.name: {"render": s.render, "title": s.title, "columns": s.columns or None} for s in pack.sections},
|
|
192
204
|
"signals": [], "truncated": {},
|
|
193
205
|
}
|
|
@@ -198,6 +210,12 @@ class PackCollector:
|
|
|
198
210
|
return {"data": {}, "errors": [str(ex)]}
|
|
199
211
|
try:
|
|
200
212
|
for sec in pack.sections:
|
|
213
|
+
is_routines = sec.name == ROUTINES_SECTION
|
|
214
|
+
if is_routines:
|
|
215
|
+
rcfg = routine_settings(routine_opts)
|
|
216
|
+
if not rcfg["enabled"]:
|
|
217
|
+
continue
|
|
218
|
+
sec = sec.model_copy(update={"limit": rcfg["max_rows"]}) # the row cap is routines_max, not the pack's limit
|
|
201
219
|
items: list[str | None] = [None]
|
|
202
220
|
if sec.for_each:
|
|
203
221
|
items = list(params.get(sec.for_each) or [])
|
|
@@ -216,6 +234,17 @@ class PackCollector:
|
|
|
216
234
|
if items and failed == len(items):
|
|
217
235
|
continue # nothing usable: the section is absent from data, the error is recorded
|
|
218
236
|
meta["signals"] += eval_signals(sec, rows, params, now, errors)
|
|
237
|
+
if is_routines:
|
|
238
|
+
nobody = sum(1 for r in rows if r.get("definition") in (None, ""))
|
|
239
|
+
if nobody:
|
|
240
|
+
meta["signals"].append({"section": sec.name, "severity": "info", "text": (
|
|
241
|
+
f"{nobody} routine(s) listed without a body: the login lacks the privilege that exposes "
|
|
242
|
+
"definitions (see docs/routines.md)")})
|
|
243
|
+
try:
|
|
244
|
+
data[sec.name] = finalize_rows(rows, routine_opts, ctx.routine_store, parser=make_parser(pack.dialect or "postgres"))
|
|
245
|
+
except ValueError as ex:
|
|
246
|
+
errors.append(f"{sec.name}: {ex}")
|
|
247
|
+
continue
|
|
219
248
|
if len(rows) > sec.limit:
|
|
220
249
|
meta["truncated"][sec.name] = len(rows)
|
|
221
250
|
data[sec.name] = rows[: sec.limit]
|