de-agent 0.2.3__tar.gz → 0.3.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (247) hide show
  1. {de_agent-0.2.3 → de_agent-0.3.2}/PKG-INFO +1 -1
  2. {de_agent-0.2.3 → de_agent-0.3.2}/pyproject.toml +2 -2
  3. de_agent-0.3.2/src/de_agent/__init__.py +1 -0
  4. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/cli.py +161 -5
  5. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/aws/__init__.py +3 -2
  6. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/aws/_aws_core.py +93 -6
  7. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/base.py +1 -0
  8. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/pack_base.py +34 -5
  9. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/postgres.py +85 -4
  10. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/redshift.py +235 -8
  11. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/config.py +17 -0
  12. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/data/sql_cases.yaml +41 -0
  13. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/diff.py +59 -7
  14. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/find.py +18 -4
  15. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/lint.py +87 -21
  16. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/bigquery.yaml +14 -0
  17. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/check.py +25 -0
  18. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/databricks-system.yaml +14 -0
  19. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/model.py +1 -1
  20. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/mssql.yaml +15 -0
  21. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/mysql.yaml +21 -0
  22. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/postgres-lite.yaml +35 -0
  23. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/snowflake.yaml +18 -0
  24. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/view.py +5 -0
  25. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/render.py +44 -3
  26. de_agent-0.3.2/src/de_agent/routines.py +269 -0
  27. de_agent-0.3.2/src/de_agent/routines_render.py +178 -0
  28. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/runner.py +5 -3
  29. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/airflow.py +121 -5
  30. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/catalog.py +25 -1
  31. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/common.py +1 -1
  32. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/edges.py +190 -27
  33. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/headers.py +3 -0
  34. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/pyexpr.py +1 -1
  35. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/python.py +4 -1
  36. de_agent-0.3.2/src/de_agent/scanners/routines.py +237 -0
  37. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/run.py +35 -3
  38. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/sql.py +9 -0
  39. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/secretscan.py +7 -2
  40. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/snapshot.py +36 -3
  41. de_agent-0.3.2/tests/fixtures/range_loops_repo/jobs/loops.py +205 -0
  42. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_bigquery.md +9 -0
  43. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_databricks-system.md +9 -0
  44. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_mssql.md +9 -0
  45. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_mysql.md +10 -0
  46. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_postgres-lite.md +9 -0
  47. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_snowflake.md +9 -0
  48. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/redshift_degraded.md +10 -0
  49. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/redshift_healthy.md +10 -0
  50. de_agent-0.3.2/tests/golden/schedule_cron.txt +2 -0
  51. de_agent-0.3.2/tests/golden/schedule_launchd.plist +30 -0
  52. {de_agent-0.2.3 → de_agent-0.3.2}/tests/packs_fixtures.py +25 -0
  53. {de_agent-0.2.3 → de_agent-0.3.2}/tests/redshift_fixtures.py +32 -0
  54. de_agent-0.3.2/tests/test_aws_dms_mappings.py +142 -0
  55. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_cli_guard.py +3 -1
  56. de_agent-0.3.2/tests/test_config_routine_edges.py +155 -0
  57. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_demo_e2e.py +45 -0
  58. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_diff.py +4 -2
  59. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_evals.py +22 -0
  60. de_agent-0.3.2/tests/test_export_schedule.py +115 -0
  61. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_find.py +8 -0
  62. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_hooks.py +151 -1
  63. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_lint.py +77 -0
  64. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_packs_live.py +44 -2
  65. de_agent-0.3.2/tests/test_packs_routines.py +109 -0
  66. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_postgres_collector.py +119 -0
  67. de_agent-0.3.2/tests/test_pyexpr_recursion.py +75 -0
  68. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_redshift_collector.py +140 -0
  69. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_redteam.py +2 -1
  70. de_agent-0.3.2/tests/test_routines_core.py +206 -0
  71. de_agent-0.3.2/tests/test_routines_edges.py +291 -0
  72. de_agent-0.3.2/tests/test_routines_redteam.py +134 -0
  73. de_agent-0.3.2/tests/test_routines_render.py +163 -0
  74. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_scan_v02.py +20 -0
  75. de_agent-0.3.2/tests/test_self_guard_script_writes.py +102 -0
  76. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_session_hook.py +31 -0
  77. de_agent-0.3.2/tests/test_status_dates.py +77 -0
  78. de_agent-0.2.3/src/de_agent/__init__.py +0 -1
  79. {de_agent-0.2.3 → de_agent-0.3.2}/.gitignore +0 -0
  80. {de_agent-0.2.3 → de_agent-0.3.2}/LICENSE +0 -0
  81. {de_agent-0.2.3 → de_agent-0.3.2}/NOTICE +0 -0
  82. {de_agent-0.2.3 → de_agent-0.3.2}/README.md +0 -0
  83. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/applyplan.py +0 -0
  84. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/approval.py +0 -0
  85. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/cliclass.py +0 -0
  86. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/clients.py +0 -0
  87. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/__init__.py +0 -0
  88. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/_airflow_core.py +0 -0
  89. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/_pack_http_core.py +0 -0
  90. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/airflow.py +0 -0
  91. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/dbt.py +0 -0
  92. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/pack_cli.py +0 -0
  93. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/pack_http.py +0 -0
  94. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/collectors/pack_sql.py +0 -0
  95. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/configdoc.py +0 -0
  96. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/extract.py +0 -0
  97. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/hubpath.py +0 -0
  98. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/initcmd.py +0 -0
  99. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/__init__.py +0 -0
  100. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/argv.py +0 -0
  101. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/cli.py +0 -0
  102. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/dagster.yaml +0 -0
  103. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/databricks.yaml +0 -0
  104. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/errors.py +0 -0
  105. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/expr.py +0 -0
  106. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/fivetran.yaml +0 -0
  107. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/gcp.yaml +0 -0
  108. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/kubernetes.yaml +0 -0
  109. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/loader.py +0 -0
  110. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/scaffold.py +0 -0
  111. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/packs/template.py +0 -0
  112. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/plans.py +0 -0
  113. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/pyscan.py +0 -0
  114. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/__init__.py +0 -0
  115. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/dbt.py +0 -0
  116. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/pycode.py +0 -0
  117. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/scanners/terraform.py +0 -0
  118. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/sqlclass.py +0 -0
  119. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/sqlexec.py +0 -0
  120. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/stepexec.py +0 -0
  121. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/example/integrations/orders_etl--warehouse.md +0 -0
  122. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/example/map_row.txt +0 -0
  123. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/example/projects/orders_etl.md +0 -0
  124. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/hub/README.md +0 -0
  125. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/hub/conventions.md +0 -0
  126. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/hub/de-agent.yaml +0 -0
  127. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/hub/domains/_template.md +0 -0
  128. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/hub/gitignore.tmpl +0 -0
  129. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/hub/integrations/_template.md +0 -0
  130. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/hub/map.md +0 -0
  131. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/hub/projects/_template.md +0 -0
  132. {de_agent-0.2.3 → de_agent-0.3.2}/src/de_agent/templates/hub/status.md +0 -0
  133. {de_agent-0.2.3 → de_agent-0.3.2}/tests/aws_fixtures.py +0 -0
  134. {de_agent-0.2.3 → de_agent-0.3.2}/tests/conftest.py +0 -0
  135. {de_agent-0.2.3 → de_agent-0.3.2}/tests/corpus/cli_guard.yaml +0 -0
  136. {de_agent-0.2.3 → de_agent-0.3.2}/tests/corpus/redteam.yaml +0 -0
  137. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_2_10/failing.json +0 -0
  138. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_2_10/healthy.json +0 -0
  139. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_3_0/basic.json +0 -0
  140. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/common/__init__.py +0 -0
  141. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/common/pg_utils.py +0 -0
  142. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/call_proc.py +0 -0
  143. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/datasets_2x.py +0 -0
  144. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/factory.py +0 -0
  145. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/nested_with.py +0 -0
  146. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/ops/nested_folder.py +0 -0
  147. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/sensors.py +0 -0
  148. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/sql/load_x.sql +0 -0
  149. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/sql_file.py +0 -0
  150. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/sql_hook.py +0 -0
  151. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/sql_kwarg.py +0 -0
  152. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/taskflow.py +0 -0
  153. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/unrolled.py +0 -0
  154. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/dags/util_imports.py +0 -0
  155. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/airflow_repo/utils/dates.py +0 -0
  156. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/aws_snapshot.json +0 -0
  157. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/dbt/failing/manifest.json +0 -0
  158. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/dbt/failing/run_results.json +0 -0
  159. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/dbt/failing/sources.json +0 -0
  160. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/dbt/healthy/manifest.json +0 -0
  161. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/dbt/healthy/run_results.json +0 -0
  162. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/dbt/healthy/sources.json +0 -0
  163. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/dbt_proj/target/manifest.json +0 -0
  164. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/pg_full.json +0 -0
  165. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/pg_healthy.json +0 -0
  166. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/pg_partial.json +0 -0
  167. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/python_repo/jobs/broken.py +0 -0
  168. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/python_repo/jobs/load_orders.py +0 -0
  169. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/python_repo/jobs/pandas_job.py +0 -0
  170. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/python_repo/jobs/spark_job.py +0 -0
  171. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/python_repo/lib/__init__.py +0 -0
  172. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/python_repo/lib/etl.py +0 -0
  173. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/python_repo/notebooks/dbx_export.py +0 -0
  174. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/python_repo/notebooks/orders_rollup.ipynb +0 -0
  175. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/tf_repo/main.tf +0 -0
  176. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/tf_repo/modules/warehouse.tf +0 -0
  177. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/util_repo/dags/util_hops.py +0 -0
  178. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/util_repo/dags/util_loop.py +0 -0
  179. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/util_repo/dags/util_proc.py +0 -0
  180. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/util_repo/sql/procs.sql +0 -0
  181. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/util_repo/utils/__init__.py +0 -0
  182. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/util_repo/utils/db.py +0 -0
  183. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/util_repo/utils/etl.py +0 -0
  184. {de_agent-0.2.3 → de_agent-0.3.2}/tests/fixtures/warehouse_sql/procs.sql +0 -0
  185. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/af3_basic.md +0 -0
  186. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/af_failing.md +0 -0
  187. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/af_healthy.md +0 -0
  188. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/aws_athena_section.md +0 -0
  189. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/aws_cloudwatch_section.md +0 -0
  190. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/aws_dms_section.md +0 -0
  191. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/aws_firehose_section.md +0 -0
  192. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/aws_full.md +0 -0
  193. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/aws_glue_section.md +0 -0
  194. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/aws_s3_section.md +0 -0
  195. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/aws_split_index.md +0 -0
  196. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/dbt_failing.md +0 -0
  197. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/dbt_healthy.md +0 -0
  198. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_dagster.md +0 -0
  199. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_databricks.md +0 -0
  200. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_fivetran.md +0 -0
  201. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_gcp.md +0 -0
  202. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pack_kubernetes.md +0 -0
  203. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pg_full.md +0 -0
  204. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pg_healthy.md +0 -0
  205. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/pg_partial.md +0 -0
  206. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/scan_python.md +0 -0
  207. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/scan_terraform.md +0 -0
  208. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/scan_util_airflow.md +0 -0
  209. {de_agent-0.2.3 → de_agent-0.3.2}/tests/golden/scan_util_edges.md +0 -0
  210. {de_agent-0.2.3 → de_agent-0.3.2}/tests/packs_helpers.py +0 -0
  211. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_airflow_collector.py +0 -0
  212. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_audit_pack.py +0 -0
  213. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_aws_collector.py +0 -0
  214. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_aws_guard.py +0 -0
  215. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_aws_more.py +0 -0
  216. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_aws_render_edges.py +0 -0
  217. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_cli.py +0 -0
  218. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_closeout_plans.py +0 -0
  219. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_cold_imports.py +0 -0
  220. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_config.py +0 -0
  221. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_dbt_collector.py +0 -0
  222. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_denylist.py +0 -0
  223. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_docs.py +0 -0
  224. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_docs_examples.py +0 -0
  225. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_extract.py +0 -0
  226. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_packs_builtin.py +0 -0
  227. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_packs_cli.py +0 -0
  228. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_packs_enforcement.py +0 -0
  229. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_packs_expr.py +0 -0
  230. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_packs_http.py +0 -0
  231. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_plan_steps.py +0 -0
  232. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_plans.py +0 -0
  233. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_recovered.py +0 -0
  234. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_redshift_single_pass.py +0 -0
  235. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_release_meta.py +0 -0
  236. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_release_script.py +0 -0
  237. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_render.py +0 -0
  238. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_render_m1b.py +0 -0
  239. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_scan_airflow.py +0 -0
  240. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_scan_perf.py +0 -0
  241. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_scan_run.py +0 -0
  242. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_snapshot_m3.py +0 -0
  243. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_sqlclass.py +0 -0
  244. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_sqlexec.py +0 -0
  245. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_stop_check.py +0 -0
  246. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_test_guard.py +0 -0
  247. {de_agent-0.2.3 → de_agent-0.3.2}/tests/test_walkthrough.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: de-agent
3
- Version: 0.2.3
3
+ Version: 0.3.2
4
4
  Summary: Ground-truth context, guardrails and workflows for data engineering with Claude Code
5
5
  Project-URL: Homepage, https://github.com/Rafid-Bin-Sadeque/de-agent
6
6
  Project-URL: Documentation, https://github.com/Rafid-Bin-Sadeque/de-agent#readme
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "de-agent"
7
- version = "0.2.3"
7
+ version = "0.3.2"
8
8
  description = "Ground-truth context, guardrails and workflows for data engineering with Claude Code"
9
9
  readme = "README.md"
10
10
  license = "Apache-2.0"
@@ -63,5 +63,5 @@ markers = [
63
63
  "integration: needs docker (postgres:16); skipped when unavailable",
64
64
  "demo: end-to-end against the demo stack (set DEA_DEMO=1)",
65
65
  "walkthrough: scripted claude -p flow on demo fault 01 (needs DEA_DEMO=1 and the claude CLI; run locally, not in CI)",
66
- "perf: 1,000-DAG synthetic scan timings; excluded by default, run with -m perf",
66
+ "perf: timing budgets (1,000-DAG scan, 5,000-page find index); excluded by default, run with -m perf",
67
67
  ]
@@ -0,0 +1 @@
1
+ __version__ = "0.3.2"
@@ -243,15 +243,19 @@ def edges(table: str = typer.Argument(..., help="schema.table, e.g. analytics.fc
243
243
 
244
244
  @app.command("impact")
245
245
  def impact(
246
- target: str = typer.Argument(..., help="util symbol (module.func) or schema.table"),
246
+ target: str = typer.Argument(..., help="util symbol (module.func), schema.table, or proc:/function:/view:/trigger:<schema.name>"),
247
247
  depth: int = typer.Option(5, "--depth", min=1, max=10, help="Max downstream hops for a table (1-10)"),
248
248
  ) -> None:
249
- """Symbol: the DAGs that use it. Table: downstream readers, transitively, with the path."""
250
- from de_agent.scanners.edges import downstream
249
+ """Symbol: the DAGs that use it. Table: downstream readers, transitively, with the path (routines list their callers).
250
+ proc:/function:/view:/trigger: <name>: what a live routine reads and writes, who calls it, and everything downstream."""
251
+ from de_agent.scanners.edges import callers, downstream
251
252
 
252
253
  hub = _require_hub()
253
- key = target.lower()
254
254
  data = _load_edges(hub)
255
+ if target.split(":", 1)[0] in ("proc", "function", "view", "trigger") and ":" in target:
256
+ _impact_routine(data, target, depth)
257
+ return
258
+ key = target.lower()
255
259
  syms = _load_symbols(hub)
256
260
  if key not in data["tables"] and not any(q == target or q.endswith("." + target) for q in syms):
257
261
  key = _resolve_table(data, target) or key
@@ -260,6 +264,9 @@ def impact(
260
264
  typer.echo(f"impact of table {key} (downstream readers, depth <= {depth}): {len(rows)}")
261
265
  for r in sorted(rows, key=lambda r: (r["depth"], r["reader"])):
262
266
  typer.echo(f" depth {r['depth']} {r['reader']} [{r['project']}] {' -> '.join(r['path'])}")
267
+ if r["reader"] in data.get("routines", {}):
268
+ for c in callers(data, r["reader"]):
269
+ typer.echo(f" called by {c}")
263
270
  return
264
271
  match = [q for q in syms if q == target or q.endswith("." + target)]
265
272
  if match:
@@ -269,7 +276,37 @@ def impact(
269
276
  for d in s["dags"]:
270
277
  typer.echo(f" {d['dag_id']} {d['file']}")
271
278
  return
272
- _unknown("table or symbol", target, [*data["tables"], *syms])
279
+ _unknown("table or symbol", target, [*data["tables"], *syms, *sorted(data.get("routines", {}))])
280
+
281
+
282
+ def _impact_routine(data: dict, target: str, depth: int) -> None:
283
+ from de_agent.scanners.edges import callers, downstream, find_routines
284
+
285
+ ids = find_routines(data, target)
286
+ if not ids:
287
+ _unknown("routine", target, sorted(data.get("routines", {})))
288
+ for rid in ids:
289
+ r = data["routines"][rid]
290
+ typer.echo(f"impact of {rid} ({r['kind']}, {r['source']}{r['args'] and ' ' + r['args']}, parse {r['parse']})")
291
+ typer.echo("reads: " + (", ".join(r["reads"]) or "none"))
292
+ typer.echo("writes: " + (", ".join(r["writes"]) or "none"))
293
+ typer.echo("calls: " + (", ".join(r["calls"]) or "none"))
294
+ cs = callers(data, rid)
295
+ typer.echo("called by:" + ("" if cs else " none"))
296
+ for c in cs:
297
+ typer.echo(f" {c}")
298
+ seen: set[tuple[str, str]] = set()
299
+ outs = r["writes"] if r["kind"] != "view" else [r["name"]]
300
+ typer.echo(f"downstream of {'/'.join(outs) or 'nothing it writes'} (depth <= {depth}):")
301
+ for t in outs:
302
+ for row in sorted(downstream(data, t, depth), key=lambda x: (x["depth"], x["reader"])):
303
+ if (row["reader"], t) in seen or row["reader"] == rid:
304
+ continue
305
+ seen.add((row["reader"], t))
306
+ typer.echo(f" depth {row['depth']} {row['reader']} [{row['project']}] {' -> '.join(row['path'])}")
307
+ if row["reader"] in data.get("routines", {}):
308
+ for c in callers(data, row["reader"]):
309
+ typer.echo(f" called by {c}")
273
310
 
274
311
 
275
312
  def _fmt_cell(v: object) -> str:
@@ -593,6 +630,125 @@ def doctor() -> None:
593
630
  raise typer.Exit(1 if any(m == "✗" for m, _, _ in rows) else 0)
594
631
 
595
632
 
633
+ export_app = typer.Typer(help="Export snapshot data (read-only, no live calls).", no_args_is_help=True)
634
+ app.add_typer(export_app, name="export")
635
+
636
+ DMS_MAPPING_COLUMNS = ["task", "schema", "table", "target_schema", "target_table", "rule", "source_endpoint", "target_endpoint"]
637
+
638
+
639
+ @export_app.command("dms-mappings")
640
+ def export_dms_mappings(
641
+ csv_path: Path | None = typer.Option(None, "--csv", help="Write the CSV to this file (default: stdout)"),
642
+ source: str | None = typer.Option(None, "--source", help="Only this aws source (default: every aws source with the section)"),
643
+ ) -> None:
644
+ """Export the dms_table_mappings section of the latest snapshot as CSV. No AWS call."""
645
+ import csv
646
+ import io
647
+
648
+ from de_agent.snapshot import list_snapshots, load_result
649
+
650
+ hub = _require_hub()
651
+ cfg = _load(hub)
652
+ names = [s.name for s in cfg.sources if s.type == "aws" and (source is None or s.name == source)]
653
+ snaps = list_snapshots(hub)
654
+ if not snaps:
655
+ typer.echo("no snapshot yet: run `dea snapshot` first", err=True)
656
+ raise typer.Exit(2)
657
+ rows: list[dict] = []
658
+ found = False
659
+ for name in names:
660
+ # newest snapshot that ran this source (a later `dea snapshot --source other` does not hide it)
661
+ res = next((r for sid in reversed(snaps) if (r := load_result(hub, sid, name)) is not None), None)
662
+ data = (res or {}).get("data") or {}
663
+ section = data.get("dms_table_mappings")
664
+ if not isinstance(section, list):
665
+ continue
666
+ found = True
667
+ rows.extend(section)
668
+ if "dms_table_mappings" in ((data.get("meta") or {}).get("truncated") or []):
669
+ typer.echo(f"warning: {name}: list is capped; raise options.dms_mappings_max", err=True)
670
+ if not found:
671
+ typer.echo("no dms_table_mappings section in the latest snapshot (needs an aws source with services.dms)", err=True)
672
+ raise typer.Exit(2)
673
+ buf = io.StringIO()
674
+ w = csv.DictWriter(buf, fieldnames=DMS_MAPPING_COLUMNS, extrasaction="ignore", lineterminator="\n")
675
+ w.writeheader()
676
+ for r in rows:
677
+ w.writerow({k: ("" if r.get(k) is None else r.get(k)) for k in DMS_MAPPING_COLUMNS})
678
+ if csv_path:
679
+ csv_path.write_text(buf.getvalue(), encoding="utf-8")
680
+ typer.echo(f"wrote {csv_path} ({len(rows)} rows)", err=True)
681
+ else:
682
+ sys.stdout.write(buf.getvalue())
683
+
684
+
685
+ schedule_app = typer.Typer(help="Print a job that keeps snapshots fresh (prints only, never installs).", no_args_is_help=True)
686
+ app.add_typer(schedule_app, name="schedule")
687
+
688
+
689
+ def _snapshot_wrapper(hub: Path, dea: str, sources: list[str]) -> str:
690
+ import shlex
691
+
692
+ log = shlex.quote(str(hub / ".dea" / "logs" / "snapshot.log"))
693
+ logs = shlex.quote(str(hub / ".dea" / "logs"))
694
+ cmd = shlex.quote(dea) + " snapshot" + "".join(f" --source {shlex.quote(x)}" for x in sources)
695
+ rotate = (f'if [ -f {log} ] && [ "$(wc -c < {log})" -gt 1048576 ]; then i=6; '
696
+ f'while [ $i -ge 1 ]; do [ -f {log}.$i ] && mv {log}.$i {log}.$((i+1)); i=$((i-1)); done; mv {log} {log}.1; fi')
697
+ return f"mkdir -p {logs}; {rotate}; {cmd} >> {log} 2>&1"
698
+
699
+
700
+ @schedule_app.command("print")
701
+ def schedule_print(
702
+ at: str = typer.Option("06:00", "--at", help="Daily run time, HH:MM (local)"),
703
+ sources: str = typer.Option("", "--sources", help="Comma-separated source names (default: all)"),
704
+ os_name: str | None = typer.Option(None, "--os", hidden=True, help="macos or linux (default: this machine)"),
705
+ ) -> None:
706
+ """Print a launchd plist (macOS) or a crontab line (elsewhere) that runs `dea snapshot` daily. Installs nothing."""
707
+ import re
708
+ import shlex
709
+ from xml.sax.saxutils import escape
710
+
711
+ m = re.fullmatch(r"(\d{1,2}):(\d{2})", at)
712
+ if not m or int(m.group(1)) > 23 or int(m.group(2)) > 59:
713
+ typer.echo(f"bad --at {at!r}: expected HH:MM (00:00-23:59)", err=True)
714
+ raise typer.Exit(2)
715
+ hour, minute = int(m.group(1)), int(m.group(2))
716
+ if os_name not in (None, "macos", "linux"):
717
+ typer.echo("bad --os: macos or linux", err=True)
718
+ raise typer.Exit(2)
719
+ hub = _require_hub()
720
+ dea = shutil.which("dea") or str(Path(sys.argv[0]).resolve())
721
+ names = [x.strip() for x in sources.split(",") if x.strip()]
722
+ wrapper = _snapshot_wrapper(hub, dea, names)
723
+ macos = (os_name == "macos") if os_name else sys.platform == "darwin"
724
+ if macos:
725
+ def x(v: object) -> str:
726
+ return escape(str(v))
727
+
728
+ typer.echo("\n".join([
729
+ '<?xml version="1.0" encoding="UTF-8"?>',
730
+ '<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">',
731
+ '<plist version="1.0">', '<dict>',
732
+ ' <key>Label</key><string>io.de-agent.snapshot</string>',
733
+ ' <key>ProgramArguments</key>', ' <array>',
734
+ ' <string>/bin/sh</string>', ' <string>-c</string>', f' <string>{x(wrapper)}</string>', ' </array>',
735
+ ' <key>EnvironmentVariables</key>', ' <dict>',
736
+ f' <key>DEA_HUB</key><string>{x(hub)}</string>',
737
+ f' <key>PATH</key><string>{x(os.environ.get("PATH", ""))}</string>', ' </dict>',
738
+ ' <key>StartCalendarInterval</key>', ' <dict>',
739
+ f' <key>Hour</key><integer>{hour}</integer>', f' <key>Minute</key><integer>{minute}</integer>', ' </dict>',
740
+ ' <key>RunAtLoad</key><false/>', '</dict>', '</plist>',
741
+ "<!--",
742
+ "de-agent prints this and installs nothing. To install it, save it to",
743
+ " ~/Library/LaunchAgents/io.de-agent.snapshot.plist",
744
+ "and run",
745
+ " launchctl load ~/Library/LaunchAgents/io.de-agent.snapshot.plist",
746
+ "-->"]))
747
+ else:
748
+ typer.echo(f"{minute} {hour} * * * DEA_HUB={shlex.quote(str(hub))} /bin/sh -c {shlex.quote(wrapper)}")
749
+ typer.echo("# de-agent prints this and installs nothing. Add the line above with `crontab -e`.")
750
+
751
+
596
752
  config_app = typer.Typer(help="de-agent.yaml tooling.", no_args_is_help=True)
597
753
  app.add_typer(config_app, name="config")
598
754
 
@@ -14,7 +14,7 @@ from typing import Any
14
14
  from de_agent.collectors.aws import _aws_core as core
15
15
  from de_agent.collectors.base import CollectorError, RunContext
16
16
 
17
- OPTION_KEYS = {"max_metrics", "max_age_h", "max_scan_gb"}
17
+ OPTION_KEYS = {"max_metrics", "max_age_h", "max_scan_gb", "dms_mappings_max"}
18
18
  AWS_KEYS = {"profile", "region", "instance_role"}
19
19
  SERVICE_KEYS = {
20
20
  "dms": set(), "glue": {"jobs", "crawlers", "catalog_databases"}, "s3": {"prefixes"}, "athena": {"workgroups"},
@@ -122,7 +122,8 @@ class AwsCollector:
122
122
  schema_version = 1
123
123
  list_keys = {
124
124
  "dms_instances": ["id"], "dms_tasks": ["id"], "dms_table_issues": ["task", "schema", "table"],
125
- "dms_top_tables": ["task", "schema", "table"], "glue_job_runs": ["job", "run_id"], "glue_crawlers": ["name"],
125
+ "dms_top_tables": ["task", "schema", "table"], "dms_table_mappings": ["task", "schema", "table"],
126
+ "glue_job_runs": ["job", "run_id"], "glue_crawlers": ["name"],
126
127
  "glue_tables": ["database", "name"], "s3_prefixes": ["prefix"], "athena_workgroups": ["workgroup"],
127
128
  "athena_top_queries": ["workgroup", "query_id"], "athena_failures": ["workgroup", "query_id"],
128
129
  "firehose_streams": ["stream"], "cloudwatch_alarms": ["name"],
@@ -22,6 +22,8 @@ MAX_PARTITIONS = 5000
22
22
  DEFAULT_MAX_METRICS = 300
23
23
  GMD_BATCH = 500 # GetMetricData accepts up to 500 queries per call
24
24
  TOP_N = 10
25
+ DMS_STATS_CAP = 200000 # table statistics per task (the mapping list needs every table, not a top-N)
26
+ DEFAULT_DMS_MAPPINGS_MAX = 5000
25
27
  S3_METRIC_WINDOW_H = 72 # BucketSizeBytes / NumberOfObjects are daily metrics
26
28
 
27
29
  # ---- read-only guard ---------------------------------------------------------------------------------------------
@@ -329,7 +331,75 @@ def _rules(table_mappings):
329
331
  return out
330
332
 
331
333
 
332
- def gather_dms(session, cfg, now, metrics, errors):
334
+ def _like(pattern, value):
335
+ """DMS object-locator match: `%` is a wildcard, everything else is literal and case-sensitive."""
336
+ rx = ".*".join(re.escape(part) for part in str(pattern if pattern is not None else "%").split("%"))
337
+ return re.fullmatch(rx, str(value or ""), re.S) is not None
338
+
339
+
340
+ def _transform(rule, name):
341
+ """Apply one DMS schema/table transformation rule to a name; unknown actions leave it unchanged."""
342
+ action, val = rule.get("action"), rule.get("value")
343
+ if name is None:
344
+ return name
345
+ if action == "rename" and val is not None:
346
+ return val
347
+ if action == "convert-lowercase":
348
+ return name.lower()
349
+ if action == "convert-uppercase":
350
+ return name.upper()
351
+ if action == "add-prefix" and val is not None:
352
+ return val + name
353
+ if action == "add-suffix" and val is not None:
354
+ return name + val
355
+ if action == "remove-prefix" and val and name.startswith(val):
356
+ return name[len(val):]
357
+ if action == "remove-suffix" and val and name.endswith(val):
358
+ return name[:-len(val)]
359
+ return name
360
+
361
+
362
+ def dms_mapping_rows(task, rules, stat_rows, source_ep, target_ep):
363
+ """Tables a task's selection rules include, after excludes and schema/table transformation rules, as plain rows.
364
+
365
+ Explicit include rules give their table directly; wildcard includes (`%`) are resolved from the task's table
366
+ statistics, and one row with `table` = the pattern and `(unresolved)` is kept when nothing matches."""
367
+ found = {} # (schema, table) -> rule label; explicit wins over wildcard
368
+ for r in rules:
369
+ if r.get("type") != "selection" or r.get("action") not in ("include", "explicit"):
370
+ continue
371
+ sp, tp = r.get("schema"), r.get("table")
372
+ if sp is not None and tp is not None and "%" not in sp and "%" not in tp:
373
+ found[(sp, tp)] = "explicit"
374
+ continue
375
+ label = "wildcard %s.%s" % (sp if sp is not None else "%", tp if tp is not None else "%")
376
+ hits = [(s.get("SchemaName"), s.get("TableName")) for s in stat_rows
377
+ if _like(sp, s.get("SchemaName")) and _like(tp, s.get("TableName"))]
378
+ for key in hits:
379
+ found.setdefault(key, label)
380
+ if not hits:
381
+ found.setdefault((sp if sp is not None else "%", tp if tp is not None else "%"), label + " (unresolved)")
382
+ excludes = [r for r in rules if r.get("type") == "selection" and r.get("action") == "exclude"]
383
+ rows = []
384
+ for (sch, tbl), label in found.items():
385
+ if any(_like(x.get("schema"), sch) and _like(x.get("table"), tbl) for x in excludes):
386
+ continue
387
+ tgt_s, tgt_t = sch, tbl
388
+ for r in rules:
389
+ if r.get("type") != "transformation" or r.get("target") not in ("schema", "table"):
390
+ continue
391
+ if not _like(r.get("schema"), sch) or (r.get("target") == "table" and not _like(r.get("table"), tbl)):
392
+ continue
393
+ if r.get("target") == "schema":
394
+ tgt_s = _transform(r, tgt_s)
395
+ else:
396
+ tgt_t = _transform(r, tgt_t)
397
+ rows.append({"task": task, "schema": sch, "table": tbl, "target_schema": tgt_s, "target_table": tgt_t,
398
+ "rule": label, "source_endpoint": source_ep, "target_endpoint": target_ep})
399
+ return sorted(rows, key=lambda r: (str(r["schema"]), str(r["table"])))
400
+
401
+
402
+ def gather_dms(session, cfg, now, metrics, errors, opts=None, truncated=None):
333
403
  dms = session.client("dms")
334
404
  out = {}
335
405
  try:
@@ -407,7 +477,12 @@ def gather_dms(session, cfg, now, metrics, errors):
407
477
  "folder": s3.get("BucketFolder")}
408
478
  return {k: v for k, v in row.items() if v}
409
479
 
410
- trows, issues, top = [], [], []
480
+ def endpoint_id(arn):
481
+ e = endpoints.get(arn)
482
+ return (e.get("EndpointIdentifier") if e else None) or _arn_tail(arn) or None
483
+
484
+ mappings_max = int((opts or {}).get("dms_mappings_max", DEFAULT_DMS_MAPPINGS_MAX))
485
+ trows, issues, top, mapping = [], [], [], []
411
486
  for t in tasks:
412
487
  tid = t.get("ReplicationTaskIdentifier")
413
488
  stats = t.get("ReplicationTaskStats") or {}
@@ -427,12 +502,16 @@ def gather_dms(session, cfg, now, metrics, errors):
427
502
  "rules": _rules(t.get("TableMappings")),
428
503
  }
429
504
  try:
430
- stat_rows, _ = _paginate(dms, "describe_table_statistics", "TableStatistics", 10000,
431
- ReplicationTaskArn=t.get("ReplicationTaskArn"),
432
- PaginationConfig={"PageSize": 500})
505
+ stat_rows, stat_trunc = _paginate(dms, "describe_table_statistics", "TableStatistics", DMS_STATS_CAP,
506
+ ReplicationTaskArn=t.get("ReplicationTaskArn"),
507
+ PaginationConfig={"PageSize": 500})
508
+ if stat_trunc and truncated is not None and "dms_table_mappings" not in truncated:
509
+ truncated.append("dms_table_mappings")
433
510
  except Exception as ex:
434
511
  errors.append("dms.table_statistics %s: %s" % (tid, describe_error(ex)))
435
512
  stat_rows = []
513
+ if truncated is not None and "dms_table_mappings" not in truncated:
514
+ truncated.append("dms_table_mappings") # wildcards unresolved: the list is partial, not shrunk
436
515
  n_susp = 0
437
516
  cand = []
438
517
  for s in stat_rows:
@@ -446,6 +525,8 @@ def gather_dms(session, cfg, now, metrics, errors):
446
525
  cand.append(dict(base, state=state, cdc_changes=changes, inserts=int(s.get("Inserts") or 0),
447
526
  updates=int(s.get("Updates") or 0), deletes=int(s.get("Deletes") or 0),
448
527
  ddls=int(s.get("Ddls") or 0), full_load_rows=int(s.get("FullLoadRows") or 0)))
528
+ mapping += dms_mapping_rows(tid, row["rules"], stat_rows, endpoint_id(t.get("SourceEndpointArn")),
529
+ endpoint_id(t.get("TargetEndpointArn")))
449
530
  row["tables_suspended"] = n_susp
450
531
  row["tables_total"] = len(stat_rows)
451
532
  top += sorted(cand, key=lambda r: (-r["cdc_changes"], str(r["schema"]), str(r["table"])))[:TOP_N]
@@ -453,6 +534,12 @@ def gather_dms(session, cfg, now, metrics, errors):
453
534
  out["dms_tasks"] = sorted(trows, key=lambda r: str(r["id"]))
454
535
  out["dms_table_issues"] = sorted(issues, key=lambda r: (str(r["task"]), str(r["schema"]), str(r["table"])))
455
536
  out["dms_top_tables"] = sorted(top, key=lambda r: (str(r["task"]), -r["cdc_changes"], str(r["table"])))
537
+ mapping.sort(key=lambda r: (str(r["task"]), str(r["schema"]), str(r["table"])))
538
+ if len(mapping) > mappings_max:
539
+ mapping = mapping[:mappings_max]
540
+ if truncated is not None and "dms_table_mappings" not in truncated:
541
+ truncated.append("dms_table_mappings")
542
+ out["dms_table_mappings"] = mapping
456
543
  return out, trunc or ttrunc
457
544
 
458
545
 
@@ -744,7 +831,7 @@ def gather(cfg, session=None, now=None):
744
831
  metrics = Metrics(session, now, WINDOW_H, cap)
745
832
  if "dms" in services:
746
833
  try:
747
- part, trunc = gather_dms(session, services["dms"] or {}, now, metrics, errors)
834
+ part, trunc = gather_dms(session, services["dms"] or {}, now, metrics, errors, opts, truncated)
748
835
  data.update(part)
749
836
  if trunc:
750
837
  truncated.append("dms")
@@ -29,6 +29,7 @@ class RunContext:
29
29
  prior: dict[str, Any] | None = None # latest usable earlier result of this source (heavy-section reuse)
30
30
  deep: bool = False # `dea snapshot --deep`: recompute everything, ignore `prior`
31
31
  now: datetime | None = None # injectable clock (tests)
32
+ routine_store: Any = None # de_agent.routines.RoutineStore: where routine bodies go (None: bodies are hashed, not stored)
32
33
  pack_fn: Callable[..., Any] | None = None # injectable executor for pack collectors (tests): (section, item, payload) -> result
33
34
 
34
35
  def remaining(self) -> float:
@@ -13,11 +13,18 @@ from typing import Any
13
13
 
14
14
  from de_agent.collectors.base import CollectorError, RunContext
15
15
  from de_agent.packs import template
16
- from de_agent.packs.check import MSG_FIELD, check_pack
16
+ from de_agent.packs.check import MSG_FIELD, ROUTINES_SECTION, check_pack
17
17
  from de_agent.packs.errors import PackError
18
18
  from de_agent.packs.expr import compile_expr
19
19
  from de_agent.packs.loader import load_pack
20
20
  from de_agent.packs.model import Pack, Section
21
+ from de_agent.routines import (
22
+ finalize_rows,
23
+ routine_settings,
24
+ split_routine_options,
25
+ validate_routine_options,
26
+ )
27
+ from de_agent.scanners.routines import make_parser
21
28
 
22
29
  MAX_SIGNALS_PER_RULE = 10
23
30
  MAX_JSON_BYTES = 5 * 1024 * 1024
@@ -33,7 +40,7 @@ def load_for_source(source: Any, kind: str) -> Pack:
33
40
  if pack.kind != kind:
34
41
  raise ValueError(f"pack {pack.pack!r} is kind {pack.kind}; a type: {kind} source needs a {kind} pack")
35
42
  try:
36
- template.resolve_params(pack.params, source.options)
43
+ template.resolve_params(pack.params, split_routine_options(source.options)[0])
37
44
  except PackError as ex:
38
45
  raise ValueError(str(ex)) from None
39
46
  return pack
@@ -57,7 +64,9 @@ def validate_for_config(source: Any, kind: str) -> Pack | None:
57
64
  if pack.kind != kind:
58
65
  raise ValueError(f"pack {pack.pack!r} is kind {pack.kind}; a type: {kind} source needs a {kind} pack")
59
66
  try:
60
- template.resolve_params(pack.params, source.options)
67
+ params, rt = split_routine_options(source.options)
68
+ validate_routine_options(rt)
69
+ template.resolve_params(pack.params, params)
61
70
  except PackError as ex:
62
71
  raise ValueError(str(ex)) from None
63
72
  return pack
@@ -175,7 +184,9 @@ class PackCollector:
175
184
  def collect(self, source: Any, ctx: RunContext) -> dict[str, Any]:
176
185
  try:
177
186
  pack = load_for_source(source, self.type)
178
- params = template.resolve_params(pack.params, source.options)
187
+ pack_opts, routine_opts = split_routine_options(source.options)
188
+ validate_routine_options(routine_opts)
189
+ params = template.resolve_params(pack.params, pack_opts)
179
190
  except (ValueError, PackError) as ex:
180
191
  return {"data": {}, "errors": [str(ex)]}
181
192
  problems = check_pack(pack)
@@ -187,7 +198,8 @@ class PackCollector:
187
198
  meta: dict[str, Any] = {
188
199
  "pack": pack.pack, "pack_version": pack.version, "kind": pack.kind, "validated": pack.validated,
189
200
  "list_keys": {s.name: s.keys for s in pack.sections if s.keys},
190
- "volatile": sorted({v for s in pack.sections for v in s.volatile}),
201
+ "volatile": sorted({v for s in pack.sections for v in s.volatile}
202
+ | ({"owner", "reads", "writes", "calls", "parse", "body_bytes"} if any(s.name == ROUTINES_SECTION for s in pack.sections) else set())),
191
203
  "sections": {s.name: {"render": s.render, "title": s.title, "columns": s.columns or None} for s in pack.sections},
192
204
  "signals": [], "truncated": {},
193
205
  }
@@ -198,6 +210,12 @@ class PackCollector:
198
210
  return {"data": {}, "errors": [str(ex)]}
199
211
  try:
200
212
  for sec in pack.sections:
213
+ is_routines = sec.name == ROUTINES_SECTION
214
+ if is_routines:
215
+ rcfg = routine_settings(routine_opts)
216
+ if not rcfg["enabled"]:
217
+ continue
218
+ sec = sec.model_copy(update={"limit": rcfg["max_rows"]}) # the row cap is routines_max, not the pack's limit
201
219
  items: list[str | None] = [None]
202
220
  if sec.for_each:
203
221
  items = list(params.get(sec.for_each) or [])
@@ -216,6 +234,17 @@ class PackCollector:
216
234
  if items and failed == len(items):
217
235
  continue # nothing usable: the section is absent from data, the error is recorded
218
236
  meta["signals"] += eval_signals(sec, rows, params, now, errors)
237
+ if is_routines:
238
+ nobody = sum(1 for r in rows if r.get("definition") in (None, ""))
239
+ if nobody:
240
+ meta["signals"].append({"section": sec.name, "severity": "info", "text": (
241
+ f"{nobody} routine(s) listed without a body: the login lacks the privilege that exposes "
242
+ "definitions (see docs/routines.md)")})
243
+ try:
244
+ data[sec.name] = finalize_rows(rows, routine_opts, ctx.routine_store, parser=make_parser(pack.dialect or "postgres"))
245
+ except ValueError as ex:
246
+ errors.append(f"{sec.name}: {ex}")
247
+ continue
219
248
  if len(rows) > sec.limit:
220
249
  meta["truncated"][sec.name] = len(rows)
221
250
  data[sec.name] = rows[: sec.limit]
@@ -8,6 +8,14 @@ import re
8
8
  from typing import Any, Callable
9
9
 
10
10
  from de_agent.collectors.base import CollectorError, RunContext
11
+ from de_agent.routines import (
12
+ KINDS,
13
+ ROUTINE_OPTION_KEYS,
14
+ finalize_rows,
15
+ routine_settings,
16
+ validate_routine_options,
17
+ )
18
+ from de_agent.scanners.routines import make_parser
11
19
 
12
20
  IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_$]*$")
13
21
  XID_MAX = 2_147_483_647
@@ -16,7 +24,7 @@ SETTINGS = (
16
24
  "idle_in_transaction_session_timeout",
17
25
  )
18
26
  DEFAULT_LONG_QUERY_S = 300
19
- OPTION_KEYS = {"schemas", "top_tables", "long_query_seconds"}
27
+ OPTION_KEYS = {"schemas", "top_tables", "long_query_seconds"} | ROUTINE_OPTION_KEYS
20
28
 
21
29
 
22
30
  def validate_options(options: dict[str, Any]) -> None:
@@ -29,6 +37,7 @@ def validate_options(options: dict[str, Any]) -> None:
29
37
  for key in ("top_tables", "long_query_seconds"):
30
38
  if key in options and not (isinstance(options[key], int) and options[key] > 0):
31
39
  raise ValueError(f"options.{key} must be a positive integer")
40
+ validate_routine_options(options)
32
41
 
33
42
 
34
43
  def validate_source(source: Any) -> None:
@@ -112,8 +121,8 @@ order by 3 desc, 1
112
121
  def q_tables_nolock(schemas: list[str], top: int) -> str:
113
122
  return f"""
114
123
  select s.schemaname as "schema", s.relname as name,
115
- (c.relpages * 8192)::bigint as total_bytes,
116
- (c.relpages * 8192)::bigint as table_bytes,
124
+ (c.relpages::bigint * 8192) as total_bytes,
125
+ (c.relpages::bigint * 8192) as table_bytes,
117
126
  null::bigint as index_bytes,
118
127
  s.n_live_tup, s.n_dead_tup,
119
128
  round(100.0 * s.n_dead_tup / nullif(s.n_live_tup + s.n_dead_tup, 0), 1) as dead_pct,
@@ -180,6 +189,58 @@ def q_settings() -> str:
180
189
  return f"select name, reset_val as setting, unit from pg_settings where name in ({names}) order by 1"
181
190
 
182
191
 
192
+ def q_routines(schemas: list[str], kinds: list[str], max_rows: int, view_bodies: bool = True) -> str:
193
+ """Procedures, functions, views, materialized views and triggers in one UNION ALL (v0.3 section 2.1).
194
+
195
+ Aggregates and window functions are skipped (prokind p/f only); extension-owned objects (pg_depend deptype e)
196
+ are excluded; `internal`/`c` functions keep language and an empty body. Reads definitions only, never calls them.
197
+ pg_get_viewdef() locks the relations a view reads, so it queues behind an ACCESS EXCLUSIVE lock on them:
198
+ `view_bodies=False` (the lock-timeout fallback) lists views without their text."""
199
+ ext = "not exists (select 1 from pg_depend d where d.classid = '{cls}'::regclass and d.objid = {oid} and d.deptype = 'e')"
200
+ parts: list[str] = []
201
+ if "procedure" in kinds or "function" in kinds:
202
+ codes = ", ".join(repr(c) for k, c in (("procedure", "p"), ("function", "f")) if k in kinds)
203
+ parts.append(f"""
204
+ select n.nspname::text as "schema", p.proname::text as name,
205
+ case p.prokind when 'p' then 'procedure' else 'function' end as kind,
206
+ '(' || oidvectortypes(p.proargtypes) || ')' as args,
207
+ l.lanname::text as language,
208
+ coalesce(pg_get_function_result(p.oid), '') as returns,
209
+ pg_get_userbyid(p.proowner)::text as owner,
210
+ case when l.lanname in ('internal', 'c') then '' else pg_get_functiondef(p.oid) end as def
211
+ from pg_proc p
212
+ join pg_namespace n on n.oid = p.pronamespace
213
+ join pg_language l on l.oid = p.prolang
214
+ where p.prokind in ({codes}) and {_schema_filter('n.nspname', schemas)}
215
+ and {ext.format(cls='pg_proc', oid='p.oid')}""")
216
+ if "view" in kinds or "materialized_view" in kinds:
217
+ rk = ", ".join(repr(c) for k, c in (("view", "v"), ("materialized_view", "m")) if k in kinds)
218
+ parts.append(f"""
219
+ select n.nspname::text, c.relname::text,
220
+ case c.relkind when 'm' then 'materialized_view' else 'view' end,
221
+ '', 'sql', '', pg_get_userbyid(c.relowner)::text, {'pg_get_viewdef(c.oid)' if view_bodies else "''"}
222
+ from pg_class c
223
+ join pg_namespace n on n.oid = c.relnamespace
224
+ where c.relkind in ({rk}) and {_schema_filter('n.nspname', schemas)}
225
+ and {ext.format(cls='pg_class', oid='c.oid')}""")
226
+ if "trigger" in kinds:
227
+ parts.append(f"""
228
+ select n.nspname::text, t.tgname::text, 'trigger', 'on ' || c.relname::text, '', '',
229
+ pg_get_userbyid(c.relowner)::text, pg_get_triggerdef(t.oid)
230
+ from pg_trigger t
231
+ join pg_class c on c.oid = t.tgrelid
232
+ join pg_namespace n on n.oid = c.relnamespace
233
+ where not t.tgisinternal and {_schema_filter('n.nspname', schemas)}""")
234
+ body = "\nunion all".join(parts)
235
+ return f"""
236
+ select "schema", name, kind, args, language, returns, owner, octet_length(def) as body_bytes, def as definition
237
+ from ({body}
238
+ ) r("schema", name, kind, args, language, returns, owner, def)
239
+ order by 1, 2, 4, 3
240
+ limit {int(max_rows) + 1}
241
+ """
242
+
243
+
183
244
  Q_HAS_PSS = "select 1 as present from pg_extension where extname = 'pg_stat_statements'"
184
245
 
185
246
 
@@ -214,10 +275,12 @@ class PostgresCollector:
214
275
  "blocking": ["blocked_pid", "blocker_pid"],
215
276
  "replication_slots": ["slot_name"],
216
277
  "settings": ["name"],
278
+ "routines": ["schema", "name", "args"],
217
279
  }
218
280
  volatile_fields = {
219
281
  "last_autovacuum", "last_autoanalyze", "seq_scan", "idx_scan", "duration_s", "wait_s",
220
- "state", "wait_event", "query", "blocked_query", "blocker_query",
282
+ "state", "wait_event", "query", "blocked_query", "blocker_query", "owner",
283
+ "reads", "writes", "calls", "parse", "body_bytes",
221
284
  }
222
285
  validate_options = staticmethod(validate_options)
223
286
  validate_source = staticmethod(validate_source)
@@ -268,8 +331,26 @@ class PostgresCollector:
268
331
  run("replication_slots", q_slots())
269
332
  run("settings", q_settings())
270
333
  self._statements(source, ctx, data)
334
+ self._routines(source, ctx, data, errors, schemas)
271
335
  return {"data": data, "errors": errors}
272
336
 
337
+ def _routines(self, source: Any, ctx: RunContext, data: dict[str, Any], errors: list[str], schemas: list[str]) -> None:
338
+ cfg = routine_settings(source.options)
339
+ if not cfg["enabled"]:
340
+ return
341
+ kinds = [k for k in KINDS if k in cfg["kinds"]]
342
+ try:
343
+ try:
344
+ raw = ctx.query(source, q_routines(schemas, kinds, cfg["max_rows"]))
345
+ except CollectorError as ex:
346
+ if "lock timeout" not in str(ex) or not ({"view", "materialized_view"} & set(kinds)):
347
+ raise
348
+ raw = ctx.query(source, q_routines(schemas, kinds, cfg["max_rows"], view_bodies=False))
349
+ data["meta"]["routine_view_bodies_skipped"] = True
350
+ data["routines"] = finalize_rows(raw, source.options, ctx.routine_store, parser=make_parser("postgres"))
351
+ except (CollectorError, ValueError) as ex:
352
+ errors.append(f"routines: {ex}")
353
+
273
354
  def _statements(self, source: Any, ctx: RunContext, data: dict[str, Any]) -> None:
274
355
  """Optional section: silently skipped when pg_stat_statements is absent or not loaded."""
275
356
  try: