theus-cli-linux-arm64 1.2.2 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/RELEASE.json +6 -6
- package/bin/theus +0 -0
- package/package.json +1 -1
- package/vendor/higiene-motor/PARCHES.md +244 -0
- package/vendor/higiene-motor/audit_dir.py +166 -0
- package/vendor/higiene-motor/audit_lib.py +333 -0
- package/vendor/higiene-motor/audit_website.py +635 -0
- package/vendor/higiene-motor/av_meta.py +498 -0
- package/vendor/higiene-motor/check_staged.py +71 -0
- package/vendor/higiene-motor/clean_ctrlregen.py +212 -0
- package/vendor/higiene-motor/clean_file.py +237 -0
- package/vendor/higiene-motor/clean_image.py +218 -0
- package/vendor/higiene-motor/clean_staged.py +143 -0
- package/vendor/higiene-motor/clean_text.py +108 -0
- package/vendor/higiene-motor/common.py +486 -0
- package/vendor/higiene-motor/container_meta.py +2522 -0
- package/vendor/higiene-motor/detect_gumbel.py +308 -0
- package/vendor/higiene-motor/detect_text_watermark.py +613 -0
- package/vendor/higiene-motor/format_dispatch.py +157 -0
- package/vendor/higiene-motor/hook_written_file.py +253 -0
- package/vendor/higiene-motor/image_meta.py +2171 -0
- package/vendor/higiene-motor/inspect_file.py +138 -0
- package/vendor/higiene-motor/inspect_image.py +72 -0
- package/vendor/higiene-motor/inspect_text.py +86 -0
- package/vendor/higiene-motor/markdiffusion_harness.py +533 -0
- package/vendor/higiene-motor/rewrite_text.py +737 -0
- package/vendor/higiene-motor/score_stylometry.py +460 -0
- package/vendor/higiene-motor/score_synthid.py +165 -0
- package/vendor/higiene-motor/server.py +1059 -0
- package/vendor/higiene-motor/synthid_score_server.py +176 -0
- package/vendor/higiene-motor/text_detectors.py +433 -0
- package/vendor/higiene-motor/text_unicode.py +730 -0
- package/vendor/theus-arsenal/.arsenal-nombres-ola2.json +0 -6
- package/vendor/theus-arsenal/.arsenal-nombres-ola3.json +0 -72
- package/vendor/theus-arsenal/.mailmap +0 -2
- package/vendor/theus-arsenal/.theus-plugin/marketplace.json +0 -25
- package/vendor/theus-arsenal/.theus-plugin/plugin.json +0 -20
- package/vendor/theus-arsenal/AUTHORS.md +0 -8
- package/vendor/theus-arsenal/CLEANROOM_REPLACEMENT_PLAN.md +0 -11
- package/vendor/theus-arsenal/LICENSE +0 -26
- package/vendor/theus-arsenal/PROVENANCE.md +0 -9
- package/vendor/theus-arsenal/README.md +0 -41
- package/vendor/theus-arsenal/THIRD_PARTY.md +0 -80
- package/vendor/theus-arsenal/agents/actualizador-docs.md +0 -123
- package/vendor/theus-arsenal/agents/analista-comentarios.md +0 -53
- package/vendor/theus-arsenal/agents/analista-conversaciones.md +0 -60
- package/vendor/theus-arsenal/agents/analista-tests-pr.md +0 -53
- package/vendor/theus-arsenal/agents/analista-tipos.md +0 -49
- package/vendor/theus-arsenal/agents/arquitecto-accesibilidad.md +0 -148
- package/vendor/theus-arsenal/agents/arquitecto-codigo.md +0 -79
- package/vendor/theus-arsenal/agents/arquitecto-homelab.md +0 -90
- package/vendor/theus-arsenal/agents/arquitecto-redes.md +0 -116
- package/vendor/theus-arsenal/agents/arquitecto-sistemas.md +0 -219
- package/vendor/theus-arsenal/agents/bifurcador-opensource.md +0 -206
- package/vendor/theus-arsenal/agents/cazador-fallos-silenciosos.md +0 -58
- package/vendor/theus-arsenal/agents/consultor-documentacion.md +0 -76
- package/vendor/theus-arsenal/agents/diagnosticador-redes.md +0 -135
- package/vendor/theus-arsenal/agents/ejecutor-e2e.md +0 -96
- package/vendor/theus-arsenal/agents/empaquetador-opensource.md +0 -257
- package/vendor/theus-arsenal/agents/escaner-secretos.md +0 -196
- package/vendor/theus-arsenal/agents/especialista-seo.md +0 -70
- package/vendor/theus-arsenal/agents/estratega-marketing.md +0 -158
- package/vendor/theus-arsenal/agents/evaluador-agentes.md +0 -214
- package/vendor/theus-arsenal/agents/evaluador-harness.md +0 -217
- package/vendor/theus-arsenal/agents/explorador-codigo.md +0 -79
- package/vendor/theus-arsenal/agents/extractor-specs.md +0 -214
- package/vendor/theus-arsenal/agents/generador-harness.md +0 -139
- package/vendor/theus-arsenal/agents/guia-tdd.md +0 -99
- package/vendor/theus-arsenal/agents/jefe-de-gabinete.md +0 -162
- package/vendor/theus-arsenal/agents/limpiador-refactors.md +0 -93
- package/vendor/theus-arsenal/agents/optimizador-rendimiento.md +0 -454
- package/vendor/theus-arsenal/agents/planificador-harness.md +0 -107
- package/vendor/theus-arsenal/agents/planificador.md +0 -221
- package/vendor/theus-arsenal/agents/resolutor-apps-harmonyos.md +0 -181
- package/vendor/theus-arsenal/agents/resolutor-builds-cpp.md +0 -98
- package/vendor/theus-arsenal/agents/resolutor-builds-dart.md +0 -209
- package/vendor/theus-arsenal/agents/resolutor-builds-django.md +0 -253
- package/vendor/theus-arsenal/agents/resolutor-builds-go.md +0 -102
- package/vendor/theus-arsenal/agents/resolutor-builds-java.md +0 -274
- package/vendor/theus-arsenal/agents/resolutor-builds-kotlin.md +0 -128
- package/vendor/theus-arsenal/agents/resolutor-builds-pytorch.md +0 -128
- package/vendor/theus-arsenal/agents/resolutor-builds-react.md +0 -214
- package/vendor/theus-arsenal/agents/resolutor-builds-rust.md +0 -156
- package/vendor/theus-arsenal/agents/resolutor-builds-swift.md +0 -169
- package/vendor/theus-arsenal/agents/resolutor-builds.md +0 -122
- package/vendor/theus-arsenal/agents/revisor-bd.md +0 -97
- package/vendor/theus-arsenal/agents/revisor-codigo.md +0 -291
- package/vendor/theus-arsenal/agents/revisor-config-red.md +0 -116
- package/vendor/theus-arsenal/agents/revisor-cpp.md +0 -80
- package/vendor/theus-arsenal/agents/revisor-csharp.md +0 -109
- package/vendor/theus-arsenal/agents/revisor-django.md +0 -168
- package/vendor/theus-arsenal/agents/revisor-fastapi.md +0 -82
- package/vendor/theus-arsenal/agents/revisor-flutter.md +0 -251
- package/vendor/theus-arsenal/agents/revisor-go.md +0 -84
- package/vendor/theus-arsenal/agents/revisor-java.md +0 -192
- package/vendor/theus-arsenal/agents/revisor-kotlin.md +0 -171
- package/vendor/theus-arsenal/agents/revisor-ml.md +0 -159
- package/vendor/theus-arsenal/agents/revisor-php.md +0 -108
- package/vendor/theus-arsenal/agents/revisor-python.md +0 -106
- package/vendor/theus-arsenal/agents/revisor-react.md +0 -166
- package/vendor/theus-arsenal/agents/revisor-rust.md +0 -102
- package/vendor/theus-arsenal/agents/revisor-seguridad.md +0 -116
- package/vendor/theus-arsenal/agents/revisor-software-clinico.md +0 -95
- package/vendor/theus-arsenal/agents/revisor-swift.md +0 -115
- package/vendor/theus-arsenal/agents/revisor-typescript.md +0 -123
- package/vendor/theus-arsenal/agents/revisor-vue.md +0 -205
- package/vendor/theus-arsenal/agents/simplificador-codigo.md +0 -55
- package/vendor/theus-arsenal/commands/actualizar-docs.md +0 -88
- package/vendor/theus-arsenal/commands/actualizar-mapas-codigo.md +0 -78
- package/vendor/theus-arsenal/commands/aprender-evaluar.md +0 -109
- package/vendor/theus-arsenal/commands/aprender.md +0 -74
- package/vendor/theus-arsenal/commands/arreglar-build.md +0 -66
- package/vendor/theus-arsenal/commands/bucle-multiperspectiva.md +0 -153
- package/vendor/theus-arsenal/commands/build-cpp.md +0 -173
- package/vendor/theus-arsenal/commands/build-flutter.md +0 -164
- package/vendor/theus-arsenal/commands/build-go.md +0 -183
- package/vendor/theus-arsenal/commands/build-gradle.md +0 -75
- package/vendor/theus-arsenal/commands/build-kotlin.md +0 -176
- package/vendor/theus-arsenal/commands/build-react.md +0 -187
- package/vendor/theus-arsenal/commands/build-rust.md +0 -187
- package/vendor/theus-arsenal/commands/campana-marketing.md +0 -125
- package/vendor/theus-arsenal/commands/checkpoint.md +0 -78
- package/vendor/theus-arsenal/commands/cobertura-tests.md +0 -73
- package/vendor/theus-arsenal/commands/configurar-gestor-paquetes.md +0 -86
- package/vendor/theus-arsenal/commands/consulta.md +0 -164
- package/vendor/theus-arsenal/commands/control-calidad.md +0 -50
- package/vendor/theus-arsenal/commands/crear-skill.md +0 -77
- package/vendor/theus-arsenal/commands/enrutar-modelo.md +0 -30
- package/vendor/theus-arsenal/commands/epica-desbloquear.md +0 -17
- package/vendor/theus-arsenal/commands/epica-descomponer.md +0 -22
- package/vendor/theus-arsenal/commands/epica-publicar.md +0 -21
- package/vendor/theus-arsenal/commands/epica-reclamar.md +0 -25
- package/vendor/theus-arsenal/commands/epica-revisar.md +0 -22
- package/vendor/theus-arsenal/commands/epica-sincronizar.md +0 -21
- package/vendor/theus-arsenal/commands/epica-validar.md +0 -21
- package/vendor/theus-arsenal/commands/evolucionar.md +0 -176
- package/vendor/theus-arsenal/commands/funcionalidad.md +0 -49
- package/vendor/theus-arsenal/commands/guardar-sesion.md +0 -278
- package/vendor/theus-arsenal/commands/harness-build.md +0 -111
- package/vendor/theus-arsenal/commands/harness-diseno.md +0 -46
- package/vendor/theus-arsenal/commands/informe-costos.md +0 -90
- package/vendor/theus-arsenal/commands/iniciar-bucle.md +0 -36
- package/vendor/theus-arsenal/commands/instintos-estado.md +0 -53
- package/vendor/theus-arsenal/commands/instintos-exportar.md +0 -69
- package/vendor/theus-arsenal/commands/instintos-importar.md +0 -120
- package/vendor/theus-arsenal/commands/lienzo-planes.md +0 -48
- package/vendor/theus-arsenal/commands/limpiar-refactor.md +0 -84
- package/vendor/theus-arsenal/commands/modo-seguro.md +0 -96
- package/vendor/theus-arsenal/commands/orq-agregar-funcionalidad.md +0 -36
- package/vendor/theus-arsenal/commands/orq-cambiar-funcionalidad.md +0 -38
- package/vendor/theus-arsenal/commands/orq-construir-mvp.md +0 -36
- package/vendor/theus-arsenal/commands/orq-corregir-defecto.md +0 -40
- package/vendor/theus-arsenal/commands/orq-refinar-codigo.md +0 -44
- package/vendor/theus-arsenal/commands/orq-revisar.md +0 -135
- package/vendor/theus-arsenal/commands/plan-prd.md +0 -160
- package/vendor/theus-arsenal/commands/plan.md +0 -198
- package/vendor/theus-arsenal/commands/podar.md +0 -31
- package/vendor/theus-arsenal/commands/pr.md +0 -181
- package/vendor/theus-arsenal/commands/promover.md +0 -41
- package/vendor/theus-arsenal/commands/proyectos.md +0 -33
- package/vendor/theus-arsenal/commands/prp-commit.md +0 -112
- package/vendor/theus-arsenal/commands/prp-implementar.md +0 -385
- package/vendor/theus-arsenal/commands/prp-planificar.md +0 -501
- package/vendor/theus-arsenal/commands/reanudar-sesion.md +0 -160
- package/vendor/theus-arsenal/commands/revisar-codigo.md +0 -289
- package/vendor/theus-arsenal/commands/revisar-cpp.md +0 -132
- package/vendor/theus-arsenal/commands/revisar-fastapi.md +0 -39
- package/vendor/theus-arsenal/commands/revisar-flutter.md +0 -115
- package/vendor/theus-arsenal/commands/revisar-go.md +0 -148
- package/vendor/theus-arsenal/commands/revisar-kotlin.md +0 -140
- package/vendor/theus-arsenal/commands/revisar-pr.md +0 -37
- package/vendor/theus-arsenal/commands/revisar-python.md +0 -297
- package/vendor/theus-arsenal/commands/revisar-react.md +0 -170
- package/vendor/theus-arsenal/commands/revisar-rust.md +0 -142
- package/vendor/theus-arsenal/commands/revisar-vue.md +0 -174
- package/vendor/theus-arsenal/commands/salud-skills.md +0 -52
- package/vendor/theus-arsenal/commands/sesiones.md +0 -345
- package/vendor/theus-arsenal/commands/test-cpp.md +0 -252
- package/vendor/theus-arsenal/commands/test-flutter.md +0 -143
- package/vendor/theus-arsenal/commands/test-go.md +0 -268
- package/vendor/theus-arsenal/commands/test-kotlin.md +0 -312
- package/vendor/theus-arsenal/commands/test-react.md +0 -265
- package/vendor/theus-arsenal/commands/test-rust.md +0 -309
- package/vendor/theus-arsenal/commands/vigilancia-canary.md +0 -43
- package/vendor/theus-arsenal/hooks/hooks.json +0 -274
- package/vendor/theus-arsenal/package.json +0 -6
- package/vendor/theus-arsenal/rules/angular/coding-style.md +0 -182
- package/vendor/theus-arsenal/rules/angular/hooks.md +0 -25
- package/vendor/theus-arsenal/rules/angular/patterns.md +0 -249
- package/vendor/theus-arsenal/rules/angular/security.md +0 -87
- package/vendor/theus-arsenal/rules/angular/testing.md +0 -164
- package/vendor/theus-arsenal/rules/arkts/coding-style.md +0 -153
- package/vendor/theus-arsenal/rules/arkts/hooks.md +0 -80
- package/vendor/theus-arsenal/rules/arkts/patterns.md +0 -236
- package/vendor/theus-arsenal/rules/arkts/security.md +0 -141
- package/vendor/theus-arsenal/rules/arkts/testing.md +0 -126
- package/vendor/theus-arsenal/rules/common/agents.md +0 -49
- package/vendor/theus-arsenal/rules/common/code-review.md +0 -123
- package/vendor/theus-arsenal/rules/common/coding-style.md +0 -90
- package/vendor/theus-arsenal/rules/common/development-workflow.md +0 -44
- package/vendor/theus-arsenal/rules/common/git-workflow.md +0 -24
- package/vendor/theus-arsenal/rules/common/hooks.md +0 -32
- package/vendor/theus-arsenal/rules/common/patterns.md +0 -31
- package/vendor/theus-arsenal/rules/common/performance.md +0 -53
- package/vendor/theus-arsenal/rules/common/security.md +0 -29
- package/vendor/theus-arsenal/rules/common/testing.md +0 -57
- package/vendor/theus-arsenal/rules/cpp/coding-style.md +0 -44
- package/vendor/theus-arsenal/rules/cpp/hooks.md +0 -39
- package/vendor/theus-arsenal/rules/cpp/patterns.md +0 -51
- package/vendor/theus-arsenal/rules/cpp/security.md +0 -51
- package/vendor/theus-arsenal/rules/cpp/testing.md +0 -44
- package/vendor/theus-arsenal/rules/csharp/coding-style.md +0 -72
- package/vendor/theus-arsenal/rules/csharp/hooks.md +0 -25
- package/vendor/theus-arsenal/rules/csharp/patterns.md +0 -50
- package/vendor/theus-arsenal/rules/csharp/security.md +0 -58
- package/vendor/theus-arsenal/rules/csharp/testing.md +0 -50
- package/vendor/theus-arsenal/rules/dart/coding-style.md +0 -159
- package/vendor/theus-arsenal/rules/dart/hooks.md +0 -68
- package/vendor/theus-arsenal/rules/dart/patterns.md +0 -261
- package/vendor/theus-arsenal/rules/dart/security.md +0 -135
- package/vendor/theus-arsenal/rules/dart/testing.md +0 -215
- package/vendor/theus-arsenal/rules/fsharp/coding-style.md +0 -112
- package/vendor/theus-arsenal/rules/fsharp/hooks.md +0 -26
- package/vendor/theus-arsenal/rules/fsharp/patterns.md +0 -111
- package/vendor/theus-arsenal/rules/fsharp/security.md +0 -76
- package/vendor/theus-arsenal/rules/fsharp/testing.md +0 -62
- package/vendor/theus-arsenal/rules/golang/coding-style.md +0 -32
- package/vendor/theus-arsenal/rules/golang/hooks.md +0 -17
- package/vendor/theus-arsenal/rules/golang/patterns.md +0 -45
- package/vendor/theus-arsenal/rules/golang/security.md +0 -34
- package/vendor/theus-arsenal/rules/golang/testing.md +0 -31
- package/vendor/theus-arsenal/rules/java/coding-style.md +0 -114
- package/vendor/theus-arsenal/rules/java/hooks.md +0 -18
- package/vendor/theus-arsenal/rules/java/patterns.md +0 -147
- package/vendor/theus-arsenal/rules/java/security.md +0 -101
- package/vendor/theus-arsenal/rules/java/testing.md +0 -133
- package/vendor/theus-arsenal/rules/kotlin/coding-style.md +0 -86
- package/vendor/theus-arsenal/rules/kotlin/hooks.md +0 -17
- package/vendor/theus-arsenal/rules/kotlin/patterns.md +0 -146
- package/vendor/theus-arsenal/rules/kotlin/security.md +0 -82
- package/vendor/theus-arsenal/rules/kotlin/testing.md +0 -128
- package/vendor/theus-arsenal/rules/nuxt/coding-style.md +0 -47
- package/vendor/theus-arsenal/rules/nuxt/hooks.md +0 -39
- package/vendor/theus-arsenal/rules/nuxt/patterns.md +0 -54
- package/vendor/theus-arsenal/rules/nuxt/security.md +0 -48
- package/vendor/theus-arsenal/rules/nuxt/testing.md +0 -49
- package/vendor/theus-arsenal/rules/php/coding-style.md +0 -40
- package/vendor/theus-arsenal/rules/php/hooks.md +0 -24
- package/vendor/theus-arsenal/rules/php/patterns.md +0 -33
- package/vendor/theus-arsenal/rules/php/security.md +0 -37
- package/vendor/theus-arsenal/rules/php/testing.md +0 -38
- package/vendor/theus-arsenal/rules/python/coding-style.md +0 -42
- package/vendor/theus-arsenal/rules/python/fastapi.md +0 -58
- package/vendor/theus-arsenal/rules/python/hooks.md +0 -19
- package/vendor/theus-arsenal/rules/python/patterns.md +0 -39
- package/vendor/theus-arsenal/rules/python/security.md +0 -30
- package/vendor/theus-arsenal/rules/python/testing.md +0 -38
- package/vendor/theus-arsenal/rules/react/coding-style.md +0 -109
- package/vendor/theus-arsenal/rules/react/hooks.md +0 -187
- package/vendor/theus-arsenal/rules/react/patterns.md +0 -194
- package/vendor/theus-arsenal/rules/react/security.md +0 -180
- package/vendor/theus-arsenal/rules/react/testing.md +0 -208
- package/vendor/theus-arsenal/rules/react-native/accessibility.md +0 -55
- package/vendor/theus-arsenal/rules/react-native/coding-style.md +0 -71
- package/vendor/theus-arsenal/rules/react-native/hooks.md +0 -28
- package/vendor/theus-arsenal/rules/react-native/patterns.md +0 -88
- package/vendor/theus-arsenal/rules/react-native/performance.md +0 -45
- package/vendor/theus-arsenal/rules/react-native/production-readiness.md +0 -51
- package/vendor/theus-arsenal/rules/react-native/security.md +0 -43
- package/vendor/theus-arsenal/rules/react-native/testing.md +0 -52
- package/vendor/theus-arsenal/rules/ruby/coding-style.md +0 -46
- package/vendor/theus-arsenal/rules/ruby/hooks.md +0 -37
- package/vendor/theus-arsenal/rules/ruby/patterns.md +0 -44
- package/vendor/theus-arsenal/rules/ruby/security.md +0 -51
- package/vendor/theus-arsenal/rules/ruby/testing.md +0 -51
- package/vendor/theus-arsenal/rules/rust/coding-style.md +0 -151
- package/vendor/theus-arsenal/rules/rust/hooks.md +0 -16
- package/vendor/theus-arsenal/rules/rust/patterns.md +0 -168
- package/vendor/theus-arsenal/rules/rust/security.md +0 -141
- package/vendor/theus-arsenal/rules/rust/testing.md +0 -154
- package/vendor/theus-arsenal/rules/swift/coding-style.md +0 -47
- package/vendor/theus-arsenal/rules/swift/hooks.md +0 -20
- package/vendor/theus-arsenal/rules/swift/patterns.md +0 -66
- package/vendor/theus-arsenal/rules/swift/security.md +0 -33
- package/vendor/theus-arsenal/rules/swift/testing.md +0 -45
- package/vendor/theus-arsenal/rules/typescript/coding-style.md +0 -199
- package/vendor/theus-arsenal/rules/typescript/hooks.md +0 -22
- package/vendor/theus-arsenal/rules/typescript/patterns.md +0 -52
- package/vendor/theus-arsenal/rules/typescript/security.md +0 -28
- package/vendor/theus-arsenal/rules/typescript/testing.md +0 -18
- package/vendor/theus-arsenal/rules/vue/coding-style.md +0 -54
- package/vendor/theus-arsenal/rules/vue/hooks.md +0 -45
- package/vendor/theus-arsenal/rules/vue/patterns.md +0 -56
- package/vendor/theus-arsenal/rules/vue/security.md +0 -46
- package/vendor/theus-arsenal/rules/vue/testing.md +0 -53
- package/vendor/theus-arsenal/rules/web/coding-style.md +0 -108
- package/vendor/theus-arsenal/rules/web/design-quality.md +0 -75
- package/vendor/theus-arsenal/rules/web/hooks.md +0 -141
- package/vendor/theus-arsenal/rules/web/patterns.md +0 -91
- package/vendor/theus-arsenal/rules/web/performance.md +0 -76
- package/vendor/theus-arsenal/rules/web/security.md +0 -69
- package/vendor/theus-arsenal/rules/web/testing.md +0 -67
- package/vendor/theus-arsenal/scripts/hooks/bash-hook-dispatcher.js +0 -192
- package/vendor/theus-arsenal/scripts/hooks/block-no-verify.js +0 -556
- package/vendor/theus-arsenal/scripts/hooks/check-console-log.js +0 -92
- package/vendor/theus-arsenal/scripts/hooks/check-hook-enabled.js +0 -16
- package/vendor/theus-arsenal/scripts/hooks/config-protection.js +0 -171
- package/vendor/theus-arsenal/scripts/hooks/design-quality-check.js +0 -132
- package/vendor/theus-arsenal/scripts/hooks/doc-file-warning.js +0 -110
- package/vendor/theus-arsenal/scripts/hooks/modo-seguro.js +0 -158
- package/vendor/theus-arsenal/scripts/hooks/notificacion-escritorio.js +0 -208
- package/vendor/theus-arsenal/scripts/hooks/plugin-hook-bootstrap.js +0 -280
- package/vendor/theus-arsenal/scripts/hooks/post-bash-command-log.js +0 -87
- package/vendor/theus-arsenal/scripts/hooks/post-bash-dispatcher.js +0 -24
- package/vendor/theus-arsenal/scripts/hooks/post-bash-pr-created.js +0 -60
- package/vendor/theus-arsenal/scripts/hooks/post-edit-accumulator.js +0 -94
- package/vendor/theus-arsenal/scripts/hooks/post-edit-console-warn.js +0 -54
- package/vendor/theus-arsenal/scripts/hooks/post-edit-format.js +0 -110
- package/vendor/theus-arsenal/scripts/hooks/post-edit-typecheck.js +0 -98
- package/vendor/theus-arsenal/scripts/hooks/pre-bash-commit-quality.js +0 -447
- package/vendor/theus-arsenal/scripts/hooks/pre-bash-dev-server-block.js +0 -232
- package/vendor/theus-arsenal/scripts/hooks/pre-bash-dispatcher.js +0 -27
- package/vendor/theus-arsenal/scripts/hooks/pre-compact.js +0 -101
- package/vendor/theus-arsenal/scripts/hooks/pretooluse-visible-output.js +0 -49
- package/vendor/theus-arsenal/scripts/hooks/quality-gate.js +0 -168
- package/vendor/theus-arsenal/scripts/hooks/run-with-flags.js +0 -271
- package/vendor/theus-arsenal/scripts/hooks/salud-mcp.js +0 -749
- package/vendor/theus-arsenal/scripts/hooks/seguimiento-costos.js +0 -161
- package/vendor/theus-arsenal/scripts/hooks/session-end.js +0 -340
- package/vendor/theus-arsenal/scripts/hooks/session-start.js +0 -906
- package/vendor/theus-arsenal/scripts/hooks/stop-format-typecheck.js +0 -263
- package/vendor/theus-arsenal/scripts/hooks/suggest-compact.js +0 -286
- package/vendor/theus-arsenal/scripts/hooks/theus-context-monitor.js +0 -289
- package/vendor/theus-arsenal/scripts/hooks/verificacion-factual.js +0 -1278
- package/vendor/theus-arsenal/scripts/lib/agent-data-home.js +0 -96
- package/vendor/theus-arsenal/scripts/lib/cost-estimate.js +0 -66
- package/vendor/theus-arsenal/scripts/lib/hook-flags.js +0 -79
- package/vendor/theus-arsenal/scripts/lib/llm-summary.js +0 -185
- package/vendor/theus-arsenal/scripts/lib/loopback-guard.js +0 -54
- package/vendor/theus-arsenal/scripts/lib/observer-sessions.js +0 -237
- package/vendor/theus-arsenal/scripts/lib/package-manager.d.ts +0 -120
- package/vendor/theus-arsenal/scripts/lib/package-manager.js +0 -434
- package/vendor/theus-arsenal/scripts/lib/path-safety.js +0 -86
- package/vendor/theus-arsenal/scripts/lib/plan-canvas/markdown.js +0 -285
- package/vendor/theus-arsenal/scripts/lib/plan-canvas/sdk.js +0 -238
- package/vendor/theus-arsenal/scripts/lib/plan-canvas/server.js +0 -534
- package/vendor/theus-arsenal/scripts/lib/plan-canvas/sessions.js +0 -274
- package/vendor/theus-arsenal/scripts/lib/plan-canvas/ui.js +0 -546
- package/vendor/theus-arsenal/scripts/lib/project-detect.js +0 -451
- package/vendor/theus-arsenal/scripts/lib/resolve-formatter.js +0 -193
- package/vendor/theus-arsenal/scripts/lib/resolve-theus-root.js +0 -101
- package/vendor/theus-arsenal/scripts/lib/session-aliases.d.ts +0 -138
- package/vendor/theus-arsenal/scripts/lib/session-aliases.js +0 -483
- package/vendor/theus-arsenal/scripts/lib/session-bridge.js +0 -155
- package/vendor/theus-arsenal/scripts/lib/session-manager.d.ts +0 -137
- package/vendor/theus-arsenal/scripts/lib/session-manager.js +0 -555
- package/vendor/theus-arsenal/scripts/lib/shell-split.js +0 -86
- package/vendor/theus-arsenal/scripts/lib/shell-substitution.js +0 -501
- package/vendor/theus-arsenal/scripts/lib/skill-evolution/dashboard.js +0 -414
- package/vendor/theus-arsenal/scripts/lib/skill-evolution/health.js +0 -285
- package/vendor/theus-arsenal/scripts/lib/skill-evolution/index.js +0 -22
- package/vendor/theus-arsenal/scripts/lib/skill-evolution/provenance.js +0 -200
- package/vendor/theus-arsenal/scripts/lib/skill-evolution/tracker.js +0 -157
- package/vendor/theus-arsenal/scripts/lib/skill-evolution/versioning.js +0 -246
- package/vendor/theus-arsenal/scripts/lib/transcript-context.js +0 -232
- package/vendor/theus-arsenal/scripts/lib/utils.d.ts +0 -205
- package/vendor/theus-arsenal/scripts/lib/utils.js +0 -658
- package/vendor/theus-arsenal/scripts/plan-canvas.js +0 -341
- package/vendor/theus-arsenal/scripts/setup-package-manager.js +0 -46
- package/vendor/theus-arsenal/scripts/skills-health.js +0 -143
- package/vendor/theus-arsenal/skills/a11y-frontend/SKILL.md +0 -444
- package/vendor/theus-arsenal/skills/accesibilidad/SKILL.md +0 -144
- package/vendor/theus-arsenal/skills/acelerador-throughput-datos/SKILL.md +0 -72
- package/vendor/theus-arsenal/skills/agente-scraper-datos/SKILL.md +0 -771
- package/vendor/theus-arsenal/skills/analisis-plataformas-competencia/SKILL.md +0 -137
- package/vendor/theus-arsenal/skills/api-x/SKILL.md +0 -233
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/SKILL.md +0 -335
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/agents/observer-loop.sh +0 -374
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/agents/observer.md +0 -188
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/agents/session-guardian.sh +0 -153
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/agents/start-observer.sh +0 -253
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/config.json +0 -8
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/hooks/observe.sh +0 -594
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/scripts/detect-project.sh +0 -325
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/scripts/instinct-cli.py +0 -1918
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/scripts/lib/arsenal-dir.sh +0 -23
- package/vendor/theus-arsenal/skills/aprendizaje-continuo-v2/scripts/test_parse_instinct.py +0 -1420
- package/vendor/theus-arsenal/skills/arquitectura-hexagonal/SKILL.md +0 -275
- package/vendor/theus-arsenal/skills/arquitectura-limpia-android/SKILL.md +0 -340
- package/vendor/theus-arsenal/skills/arquitectura-pipelines-recomendacion/SKILL.md +0 -112
- package/vendor/theus-arsenal/skills/atlas-datos-prediccion/SKILL.md +0 -53
- package/vendor/theus-arsenal/skills/auditoria-arquitectura-agentes/SKILL.md +0 -257
- package/vendor/theus-arsenal/skills/auditoria-produccion/SKILL.md +0 -180
- package/vendor/theus-arsenal/skills/auditoria-rutas-clic/SKILL.md +0 -243
- package/vendor/theus-arsenal/skills/autoevaluacion-agente/SKILL.md +0 -180
- package/vendor/theus-arsenal/skills/autoevaluacion-agente/examples/high-score-example.md +0 -89
- package/vendor/theus-arsenal/skills/autoevaluacion-agente/examples/low-score-example.md +0 -87
- package/vendor/theus-arsenal/skills/autoevaluacion-agente/references/evaluation-criteria.md +0 -71
- package/vendor/theus-arsenal/skills/autoevaluacion-agente/references/hook-integration.md +0 -90
- package/vendor/theus-arsenal/skills/autoevaluacion-agente/scripts/evaluate.py +0 -409
- package/vendor/theus-arsenal/skills/autoevaluacion-agente/templates/evaluation-report.md +0 -86
- package/vendor/theus-arsenal/skills/automatizacion-ssh-netmiko/SKILL.md +0 -176
- package/vendor/theus-arsenal/skills/base-datos-pubmed/SKILL.md +0 -174
- package/vendor/theus-arsenal/skills/base-datos-uspto/SKILL.md +0 -173
- package/vendor/theus-arsenal/skills/benchmark/SKILL.md +0 -92
- package/vendor/theus-arsenal/skills/bitacora-crecimiento/SKILL.md +0 -120
- package/vendor/theus-arsenal/skills/bucle-verificacion/SKILL.md +0 -125
- package/vendor/theus-arsenal/skills/buscar-primero/SKILL.md +0 -184
- package/vendor/theus-arsenal/skills/busqueda-exa/SKILL.md +0 -111
- package/vendor/theus-arsenal/skills/cache-por-hash/SKILL.md +0 -160
- package/vendor/theus-arsenal/skills/campana-marketing/SKILL.md +0 -118
- package/vendor/theus-arsenal/skills/capacidad-producto/SKILL.md +0 -138
- package/vendor/theus-arsenal/skills/cazador-vulnerabilidades/SKILL.md +0 -104
- package/vendor/theus-arsenal/skills/compactacion-estrategica/SKILL.md +0 -115
- package/vendor/theus-arsenal/skills/comparador-baskets-prediccion/SKILL.md +0 -69
- package/vendor/theus-arsenal/skills/compras-energia/SKILL.md +0 -206
- package/vendor/theus-arsenal/skills/concurrencia-swift-6-2/SKILL.md +0 -224
- package/vendor/theus-arsenal/skills/consejo/SKILL.md +0 -201
- package/vendor/theus-arsenal/skills/construccion-harness-agentes/SKILL.md +0 -78
- package/vendor/theus-arsenal/skills/constructor-conectores/SKILL.md +0 -119
- package/vendor/theus-arsenal/skills/constructor-equipos/SKILL.md +0 -169
- package/vendor/theus-arsenal/skills/consulta-documentacion/SKILL.md +0 -92
- package/vendor/theus-arsenal/skills/consultas-genomicas-gget/SKILL.md +0 -170
- package/vendor/theus-arsenal/skills/contacto-inversores/SKILL.md +0 -95
- package/vendor/theus-arsenal/skills/corrutinas-flows-kotlin/SKILL.md +0 -284
- package/vendor/theus-arsenal/skills/creacion-video-remotion/SKILL.md +0 -46
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/3d.md +0 -86
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/animations.md +0 -29
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/assets/charts-bar-chart.tsx +0 -173
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/assets/text-animations-typewriter.tsx +0 -100
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/assets/text-animations-word-highlight.tsx +0 -108
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/assets.md +0 -78
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/audio.md +0 -172
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/calculate-metadata.md +0 -104
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/can-decode.md +0 -75
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/charts.md +0 -58
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/compositions.md +0 -146
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/display-captions.md +0 -126
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/extract-frames.md +0 -229
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/fonts.md +0 -152
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/get-audio-duration.md +0 -58
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/get-video-dimensions.md +0 -68
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/get-video-duration.md +0 -58
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/gifs.md +0 -138
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/images.md +0 -130
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/import-srt-captions.md +0 -67
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/lottie.md +0 -67
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/measuring-dom-nodes.md +0 -34
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/measuring-text.md +0 -143
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/sequencing.md +0 -106
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/tailwind.md +0 -11
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/text-animations.md +0 -20
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/timing.md +0 -179
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/transcribe-captions.md +0 -19
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/transitions.md +0 -122
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/trimming.md +0 -52
- package/vendor/theus-arsenal/skills/creacion-video-remotion/rules/videos.md +0 -171
- package/vendor/theus-arsenal/skills/cumplimiento-aduanero/SKILL.md +0 -270
- package/vendor/theus-arsenal/skills/cumplimiento-hipaa/SKILL.md +0 -78
- package/vendor/theus-arsenal/skills/datos-sensibles-salud/SKILL.md +0 -149
- package/vendor/theus-arsenal/skills/decimales-tokens-evm/SKILL.md +0 -128
- package/vendor/theus-arsenal/skills/decisiones-arquitectura/SKILL.md +0 -206
- package/vendor/theus-arsenal/skills/demos-ui/SKILL.md +0 -469
- package/vendor/theus-arsenal/skills/desarrollo-por-intencion/SKILL.md +0 -369
- package/vendor/theus-arsenal/skills/descubrimiento-marca/SKILL.md +0 -149
- package/vendor/theus-arsenal/skills/descubrimiento-marca/references/10_proposito-porque.md +0 -42
- package/vendor/theus-arsenal/skills/descubrimiento-marca/references/20_posicionamiento.md +0 -47
- package/vendor/theus-arsenal/skills/descubrimiento-marca/references/30_audiencia-nicho.md +0 -53
- package/vendor/theus-arsenal/skills/descubrimiento-marca/references/40_personalidad-arquetipo.md +0 -61
- package/vendor/theus-arsenal/skills/descubrimiento-marca/references/50_voz-tono.md +0 -62
- package/vendor/theus-arsenal/skills/descubrimiento-marca/references/60_narrativa-historia.md +0 -54
- package/vendor/theus-arsenal/skills/descubrimiento-marca/references/70_tension-fundador.md +0 -50
- package/vendor/theus-arsenal/skills/descubrimiento-marca/references/90_SINTESIS.md +0 -133
- package/vendor/theus-arsenal/skills/descubrimiento-plugins-laravel/SKILL.md +0 -230
- package/vendor/theus-arsenal/skills/destilado-reglas/SKILL.md +0 -263
- package/vendor/theus-arsenal/skills/destilado-reglas/scripts/scan-rules.sh +0 -58
- package/vendor/theus-arsenal/skills/destilado-reglas/scripts/scan-skills.sh +0 -129
- package/vendor/theus-arsenal/skills/diagnostico-bgp/SKILL.md +0 -170
- package/vendor/theus-arsenal/skills/direccion-estetica/SKILL.md +0 -220
- package/vendor/theus-arsenal/skills/direccion-estetica/references/taxonomia-generos.md +0 -89
- package/vendor/theus-arsenal/skills/diseno-api/SKILL.md +0 -522
- package/vendor/theus-arsenal/skills/diseno-liquid-glass/SKILL.md +0 -282
- package/vendor/theus-arsenal/skills/e2e-escritorio-windows/SKILL.md +0 -879
- package/vendor/theus-arsenal/skills/edicion-video/SKILL.md +0 -289
- package/vendor/theus-arsenal/skills/enfoque-producto/SKILL.md +0 -91
- package/vendor/theus-arsenal/skills/entornos-flox/SKILL.md +0 -493
- package/vendor/theus-arsenal/skills/estandares-codigo/SKILL.md +0 -548
- package/vendor/theus-arsenal/skills/estandares-cpp/SKILL.md +0 -727
- package/vendor/theus-arsenal/skills/estandares-java/SKILL.md +0 -389
- package/vendor/theus-arsenal/skills/estilo-legado/SKILL.md +0 -149
- package/vendor/theus-arsenal/skills/estructura-informes-competencia/SKILL.md +0 -95
- package/vendor/theus-arsenal/skills/evaluacion-academica/SKILL.md +0 -165
- package/vendor/theus-arsenal/skills/explorador-skills/SKILL.md +0 -129
- package/vendor/theus-arsenal/skills/flujo-git/SKILL.md +0 -714
- package/vendor/theus-arsenal/skills/flujo-ingenieria-ml/SKILL.md +0 -343
- package/vendor/theus-arsenal/skills/flujo-tdd/SKILL.md +0 -584
- package/vendor/theus-arsenal/skills/forja-personas/SKILL.md +0 -298
- package/vendor/theus-arsenal/skills/forja-personas/gacha.py +0 -224
- package/vendor/theus-arsenal/skills/forja-personas/gacha.sh +0 -5
- package/vendor/theus-arsenal/skills/forja-personas/references/avatar-style.md +0 -123
- package/vendor/theus-arsenal/skills/forja-personas/references/boundary-rules.md +0 -53
- package/vendor/theus-arsenal/skills/forja-personas/references/error-handling.md +0 -53
- package/vendor/theus-arsenal/skills/forja-personas/references/identity-tension.md +0 -48
- package/vendor/theus-arsenal/skills/forja-personas/references/naming-system.md +0 -39
- package/vendor/theus-arsenal/skills/forja-personas/references/output-template.md +0 -166
- package/vendor/theus-arsenal/skills/fundamentos-animacion/SKILL.md +0 -296
- package/vendor/theus-arsenal/skills/generacion-iconos-ios/SKILL.md +0 -156
- package/vendor/theus-arsenal/skills/generacion-iconos-ios/scripts/generate_icons.swift +0 -258
- package/vendor/theus-arsenal/skills/generacion-iconos-ios/scripts/iconify_gen.sh +0 -235
- package/vendor/theus-arsenal/skills/gestion-excepciones-logisticas/SKILL.md +0 -208
- package/vendor/theus-arsenal/skills/gestion-transportistas/SKILL.md +0 -204
- package/vendor/theus-arsenal/skills/guia-adopcion-ml/SKILL.md +0 -64
- package/vendor/theus-arsenal/skills/harness-agente-autonomo/SKILL.md +0 -252
- package/vendor/theus-arsenal/skills/harness-eval-clinico/SKILL.md +0 -212
- package/vendor/theus-arsenal/skills/harness-evaluacion/SKILL.md +0 -296
- package/vendor/theus-arsenal/skills/harness-generador-evaluador/SKILL.md +0 -254
- package/vendor/theus-arsenal/skills/homelab-diseno-red/SKILL.md +0 -120
- package/vendor/theus-arsenal/skills/homelab-dns-pihole/SKILL.md +0 -264
- package/vendor/theus-arsenal/skills/homelab-preparacion-red/SKILL.md +0 -144
- package/vendor/theus-arsenal/skills/homelab-segmentacion-vlan/SKILL.md +0 -307
- package/vendor/theus-arsenal/skills/homelab-vpn-wireguard/SKILL.md +0 -299
- package/vendor/theus-arsenal/skills/ingenieria-agentica/SKILL.md +0 -96
- package/vendor/theus-arsenal/skills/inspeccion-animaciones-blender/SKILL.md +0 -164
- package/vendor/theus-arsenal/skills/instaladores-python/SKILL.md +0 -820
- package/vendor/theus-arsenal/skills/integracion-jira/SKILL.md +0 -307
- package/vendor/theus-arsenal/skills/inteligencia-leads/SKILL.md +0 -312
- package/vendor/theus-arsenal/skills/inteligencia-leads/agents/agente-enriquecimiento.md +0 -87
- package/vendor/theus-arsenal/skills/inteligencia-leads/agents/mapeador-mutuos.md +0 -76
- package/vendor/theus-arsenal/skills/inteligencia-leads/agents/puntuador-senales.md +0 -60
- package/vendor/theus-arsenal/skills/inteligencia-leads/agents/redactor-outreach.md +0 -102
- package/vendor/theus-arsenal/skills/inteligencia-mercados-prediccion/SKILL.md +0 -53
- package/vendor/theus-arsenal/skills/introspeccion-agente/SKILL.md +0 -150
- package/vendor/theus-arsenal/skills/inventario-skills/SKILL.md +0 -194
- package/vendor/theus-arsenal/skills/inventario-skills/scripts/quick-diff.sh +0 -87
- package/vendor/theus-arsenal/skills/inventario-skills/scripts/save-results.sh +0 -57
- package/vendor/theus-arsenal/skills/inventario-skills/scripts/scan.sh +0 -170
- package/vendor/theus-arsenal/skills/investigacion-mercado/SKILL.md +0 -87
- package/vendor/theus-arsenal/skills/investigacion-oraculos-prediccion/SKILL.md +0 -59
- package/vendor/theus-arsenal/skills/investigacion-profunda/SKILL.md +0 -161
- package/vendor/theus-arsenal/skills/keccak256-nodejs/SKILL.md +0 -100
- package/vendor/theus-arsenal/skills/libro-decisiones-recursivo/SKILL.md +0 -82
- package/vendor/theus-arsenal/skills/lienzo-planes/SKILL.md +0 -111
- package/vendor/theus-arsenal/skills/limpieza-configuracion/SKILL.md +0 -116
- package/vendor/theus-arsenal/skills/logistica-inversa-devoluciones/SKILL.md +0 -241
- package/vendor/theus-arsenal/skills/manejo-errores/SKILL.md +0 -375
- package/vendor/theus-arsenal/skills/materiales-inversores/SKILL.md +0 -101
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/SKILL.md +0 -170
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/commands/forget.mjs +0 -44
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/commands/info.mjs +0 -24
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/commands/init.mjs +0 -152
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/commands/list.mjs +0 -40
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/commands/migrate.mjs +0 -214
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/commands/resume.mjs +0 -36
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/commands/save.mjs +0 -210
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/commands/shared.mjs +0 -402
- package/vendor/theus-arsenal/skills/memoria-por-proyecto/hooks/session-start.mjs +0 -238
- package/vendor/theus-arsenal/skills/metodologia-benchmarks/SKILL.md +0 -131
- package/vendor/theus-arsenal/skills/migraciones-bd/SKILL.md +0 -430
- package/vendor/theus-arsenal/skills/modelos-fundacionales-en-dispositivo/SKILL.md +0 -244
- package/vendor/theus-arsenal/skills/modo-seguro/SKILL.md +0 -86
- package/vendor/theus-arsenal/skills/motion-avanzado/SKILL.md +0 -592
- package/vendor/theus-arsenal/skills/motor-de-contenido/SKILL.md +0 -136
- package/vendor/theus-arsenal/skills/no-conformidades-calidad/SKILL.md +0 -249
- package/vendor/theus-arsenal/skills/onboarding-codigo/SKILL.md +0 -244
- package/vendor/theus-arsenal/skills/operaciones-agentes-empresariales/SKILL.md +0 -51
- package/vendor/theus-arsenal/skills/operaciones-conocimiento/SKILL.md +0 -159
- package/vendor/theus-arsenal/skills/operaciones-correo/SKILL.md +0 -123
- package/vendor/theus-arsenal/skills/operaciones-facturacion/SKILL.md +0 -142
- package/vendor/theus-arsenal/skills/operaciones-flujo-proyectos/SKILL.md +0 -110
- package/vendor/theus-arsenal/skills/operaciones-github/SKILL.md +0 -143
- package/vendor/theus-arsenal/skills/operaciones-google-workspace/SKILL.md +0 -100
- package/vendor/theus-arsenal/skills/operaciones-investigacion/SKILL.md +0 -124
- package/vendor/theus-arsenal/skills/operaciones-mensajes/SKILL.md +0 -103
- package/vendor/theus-arsenal/skills/operaciones-terminal/SKILL.md +0 -110
- package/vendor/theus-arsenal/skills/optimizacion-benchmark/SKILL.md +0 -70
- package/vendor/theus-arsenal/skills/optimizacion-costos-llm/SKILL.md +0 -182
- package/vendor/theus-arsenal/skills/optimizador-conexiones/SKILL.md +0 -191
- package/vendor/theus-arsenal/skills/optimizador-paralelismo/SKILL.md +0 -75
- package/vendor/theus-arsenal/skills/optimizador-prompts/SKILL.md +0 -374
- package/vendor/theus-arsenal/skills/orq-agregar-funcionalidad/SKILL.md +0 -37
- package/vendor/theus-arsenal/skills/orq-cambiar-funcionalidad/SKILL.md +0 -36
- package/vendor/theus-arsenal/skills/orq-construir-mvp/SKILL.md +0 -38
- package/vendor/theus-arsenal/skills/orq-corregir-defecto/SKILL.md +0 -36
- package/vendor/theus-arsenal/skills/orq-pipeline/SKILL.md +0 -134
- package/vendor/theus-arsenal/skills/orq-refinar-codigo/SKILL.md +0 -49
- package/vendor/theus-arsenal/skills/orquestacion-equipos-agentes/SKILL.md +0 -117
- package/vendor/theus-arsenal/skills/pagos-agentes-x402/SKILL.md +0 -237
- package/vendor/theus-arsenal/skills/patrones-angular/SKILL.md +0 -153
- package/vendor/theus-arsenal/skills/patrones-angular/references/angular-animations.md +0 -160
- package/vendor/theus-arsenal/skills/patrones-angular/references/angular-aria.md +0 -410
- package/vendor/theus-arsenal/skills/patrones-angular/references/cli.md +0 -86
- package/vendor/theus-arsenal/skills/patrones-angular/references/component-harnesses.md +0 -59
- package/vendor/theus-arsenal/skills/patrones-angular/references/component-styling.md +0 -91
- package/vendor/theus-arsenal/skills/patrones-angular/references/components.md +0 -117
- package/vendor/theus-arsenal/skills/patrones-angular/references/creating-services.md +0 -97
- package/vendor/theus-arsenal/skills/patrones-angular/references/data-resolvers.md +0 -69
- package/vendor/theus-arsenal/skills/patrones-angular/references/define-routes.md +0 -67
- package/vendor/theus-arsenal/skills/patrones-angular/references/defining-providers.md +0 -72
- package/vendor/theus-arsenal/skills/patrones-angular/references/di-fundamentals.md +0 -120
- package/vendor/theus-arsenal/skills/patrones-angular/references/e2e-testing.md +0 -56
- package/vendor/theus-arsenal/skills/patrones-angular/references/effects.md +0 -83
- package/vendor/theus-arsenal/skills/patrones-angular/references/hierarchical-injectors.md +0 -43
- package/vendor/theus-arsenal/skills/patrones-angular/references/host-elements.md +0 -80
- package/vendor/theus-arsenal/skills/patrones-angular/references/injection-context.md +0 -63
- package/vendor/theus-arsenal/skills/patrones-angular/references/inputs.md +0 -101
- package/vendor/theus-arsenal/skills/patrones-angular/references/linked-signal.md +0 -59
- package/vendor/theus-arsenal/skills/patrones-angular/references/loading-strategies.md +0 -61
- package/vendor/theus-arsenal/skills/patrones-angular/references/mcp.md +0 -67
- package/vendor/theus-arsenal/skills/patrones-angular/references/navigate-to-routes.md +0 -69
- package/vendor/theus-arsenal/skills/patrones-angular/references/outputs.md +0 -86
- package/vendor/theus-arsenal/skills/patrones-angular/references/reactive-forms.md +0 -122
- package/vendor/theus-arsenal/skills/patrones-angular/references/rendering-strategies.md +0 -44
- package/vendor/theus-arsenal/skills/patrones-angular/references/resource.md +0 -77
- package/vendor/theus-arsenal/skills/patrones-angular/references/route-animations.md +0 -56
- package/vendor/theus-arsenal/skills/patrones-angular/references/route-guards.md +0 -52
- package/vendor/theus-arsenal/skills/patrones-angular/references/router-lifecycle.md +0 -45
- package/vendor/theus-arsenal/skills/patrones-angular/references/router-testing.md +0 -87
- package/vendor/theus-arsenal/skills/patrones-angular/references/show-routes-with-outlets.md +0 -68
- package/vendor/theus-arsenal/skills/patrones-angular/references/signal-forms.md +0 -795
- package/vendor/theus-arsenal/skills/patrones-angular/references/signals-overview.md +0 -94
- package/vendor/theus-arsenal/skills/patrones-angular/references/tailwind-css.md +0 -69
- package/vendor/theus-arsenal/skills/patrones-angular/references/template-driven-forms.md +0 -114
- package/vendor/theus-arsenal/skills/patrones-angular/references/testing-fundamentals.md +0 -65
- package/vendor/theus-arsenal/skills/patrones-animacion/SKILL.md +0 -432
- package/vendor/theus-arsenal/skills/patrones-backend/SKILL.md +0 -560
- package/vendor/theus-arsenal/skills/patrones-cdss-clinico/SKILL.md +0 -249
- package/vendor/theus-arsenal/skills/patrones-cisco-ios/SKILL.md +0 -148
- package/vendor/theus-arsenal/skills/patrones-clickhouse/SKILL.md +0 -443
- package/vendor/theus-arsenal/skills/patrones-compose-multiplatform/SKILL.md +0 -298
- package/vendor/theus-arsenal/skills/patrones-dart-flutter/SKILL.md +0 -566
- package/vendor/theus-arsenal/skills/patrones-despliegue/SKILL.md +0 -426
- package/vendor/theus-arsenal/skills/patrones-django/SKILL.md +0 -739
- package/vendor/theus-arsenal/skills/patrones-django-celery/SKILL.md +0 -456
- package/vendor/theus-arsenal/skills/patrones-docker/SKILL.md +0 -363
- package/vendor/theus-arsenal/skills/patrones-dotnet/SKILL.md +0 -321
- package/vendor/theus-arsenal/skills/patrones-emr-clinico/SKILL.md +0 -168
- package/vendor/theus-arsenal/skills/patrones-fastapi/SKILL.md +0 -513
- package/vendor/theus-arsenal/skills/patrones-frontend/SKILL.md +0 -655
- package/vendor/theus-arsenal/skills/patrones-go/SKILL.md +0 -674
- package/vendor/theus-arsenal/skills/patrones-jpa/SKILL.md +0 -156
- package/vendor/theus-arsenal/skills/patrones-kotlin/SKILL.md +0 -717
- package/vendor/theus-arsenal/skills/patrones-kotlin-exposed/SKILL.md +0 -724
- package/vendor/theus-arsenal/skills/patrones-kotlin-ktor/SKILL.md +0 -694
- package/vendor/theus-arsenal/skills/patrones-kubernetes/SKILL.md +0 -754
- package/vendor/theus-arsenal/skills/patrones-laravel/SKILL.md +0 -414
- package/vendor/theus-arsenal/skills/patrones-mcp/SKILL.md +0 -66
- package/vendor/theus-arsenal/skills/patrones-mysql/SKILL.md +0 -420
- package/vendor/theus-arsenal/skills/patrones-nestjs/SKILL.md +0 -229
- package/vendor/theus-arsenal/skills/patrones-nextjs/SKILL.md +0 -56
- package/vendor/theus-arsenal/skills/patrones-nuxt4/SKILL.md +0 -99
- package/vendor/theus-arsenal/skills/patrones-perl/SKILL.md +0 -508
- package/vendor/theus-arsenal/skills/patrones-postgres/SKILL.md +0 -141
- package/vendor/theus-arsenal/skills/patrones-prisma/SKILL.md +0 -399
- package/vendor/theus-arsenal/skills/patrones-python/SKILL.md +0 -750
- package/vendor/theus-arsenal/skills/patrones-pytorch/SKILL.md +0 -400
- package/vendor/theus-arsenal/skills/patrones-quarkus/SKILL.md +0 -729
- package/vendor/theus-arsenal/skills/patrones-react/SKILL.md +0 -339
- package/vendor/theus-arsenal/skills/patrones-react-native/SKILL.md +0 -325
- package/vendor/theus-arsenal/skills/patrones-redis/SKILL.md +0 -401
- package/vendor/theus-arsenal/skills/patrones-rust/SKILL.md +0 -504
- package/vendor/theus-arsenal/skills/patrones-springboot/SKILL.md +0 -314
- package/vendor/theus-arsenal/skills/patrones-swiftui/SKILL.md +0 -261
- package/vendor/theus-arsenal/skills/patrones-tinystruct/SKILL.md +0 -281
- package/vendor/theus-arsenal/skills/patrones-tinystruct/references/architecture.md +0 -90
- package/vendor/theus-arsenal/skills/patrones-tinystruct/references/data-handling.md +0 -60
- package/vendor/theus-arsenal/skills/patrones-tinystruct/references/database.md +0 -99
- package/vendor/theus-arsenal/skills/patrones-tinystruct/references/routing.md +0 -64
- package/vendor/theus-arsenal/skills/patrones-tinystruct/references/system-usage.md +0 -97
- package/vendor/theus-arsenal/skills/patrones-tinystruct/references/testing.md +0 -72
- package/vendor/theus-arsenal/skills/patrones-vite/SKILL.md +0 -449
- package/vendor/theus-arsenal/skills/patrones-vue/SKILL.md +0 -469
- package/vendor/theus-arsenal/skills/persistencia-actores-swift/SKILL.md +0 -148
- package/vendor/theus-arsenal/skills/pipeline-opensource/SKILL.md +0 -254
- package/vendor/theus-arsenal/skills/planificacion-demanda-inventario/SKILL.md +0 -236
- package/vendor/theus-arsenal/skills/planificador-trades-prediccion/SKILL.md +0 -60
- package/vendor/theus-arsenal/skills/plano-tecnico/SKILL.md +0 -102
- package/vendor/theus-arsenal/skills/presentaciones-html/SKILL.md +0 -189
- package/vendor/theus-arsenal/skills/presentaciones-html/STYLE_PRESETS.md +0 -330
- package/vendor/theus-arsenal/skills/presentaciones-html/animation-patterns.md +0 -122
- package/vendor/theus-arsenal/skills/presentaciones-html/html-template.md +0 -419
- package/vendor/theus-arsenal/skills/presentaciones-html/scripts/export-pdf.sh +0 -418
- package/vendor/theus-arsenal/skills/presentaciones-html/scripts/extract-pptx.py +0 -96
- package/vendor/theus-arsenal/skills/presentaciones-html/viewport-base.css +0 -153
- package/vendor/theus-arsenal/skills/presupuesto-contexto/SKILL.md +0 -263
- package/vendor/theus-arsenal/skills/procesamiento-documentos-nutrient/SKILL.md +0 -168
- package/vendor/theus-arsenal/skills/programacion-produccion/SKILL.md +0 -220
- package/vendor/theus-arsenal/skills/publicacion-cruzada/SKILL.md +0 -117
- package/vendor/theus-arsenal/skills/publicador-social/SKILL.md +0 -121
- package/vendor/theus-arsenal/skills/puerta-de-entrega/SKILL.md +0 -164
- package/vendor/theus-arsenal/skills/puerta-de-entrega/hooks/puerta-entrega.py +0 -242
- package/vendor/theus-arsenal/skills/pulir-interfaces/SKILL.md +0 -126
- package/vendor/theus-arsenal/skills/qa-navegador/SKILL.md +0 -103
- package/vendor/theus-arsenal/skills/ranking-grafo-social/SKILL.md +0 -153
- package/vendor/theus-arsenal/skills/recuperacion-iterativa/SKILL.md +0 -236
- package/vendor/theus-arsenal/skills/redaccion-articulos/SKILL.md +0 -81
- package/vendor/theus-arsenal/skills/regex-vs-llm/SKILL.md +0 -221
- package/vendor/theus-arsenal/skills/regresion-codigo-ia/SKILL.md +0 -388
- package/vendor/theus-arsenal/skills/rendimiento-react/SKILL.md +0 -567
- package/vendor/theus-arsenal/skills/revision-codigo-flutter-dart/SKILL.md +0 -442
- package/vendor/theus-arsenal/skills/revision-literatura-cientifica/SKILL.md +0 -191
- package/vendor/theus-arsenal/skills/revision-multiperspectiva/SKILL.md +0 -302
- package/vendor/theus-arsenal/skills/revision-riesgo-prediccion/SKILL.md +0 -56
- package/vendor/theus-arsenal/skills/revision-seguridad/SKILL.md +0 -505
- package/vendor/theus-arsenal/skills/revision-seguridad/cloud-infrastructure-security.md +0 -361
- package/vendor/theus-arsenal/skills/runtime-bun/SKILL.md +0 -83
- package/vendor/theus-arsenal/skills/salud-codigo-codescene/SKILL.md +0 -163
- package/vendor/theus-arsenal/skills/salud-interfaces-red/SKILL.md +0 -143
- package/vendor/theus-arsenal/skills/seguimiento-costos/SKILL.md +0 -122
- package/vendor/theus-arsenal/skills/seguridad-agentes-trading/SKILL.md +0 -145
- package/vendor/theus-arsenal/skills/seguridad-amm-defi/SKILL.md +0 -166
- package/vendor/theus-arsenal/skills/seguridad-django/SKILL.md +0 -648
- package/vendor/theus-arsenal/skills/seguridad-laravel/SKILL.md +0 -945
- package/vendor/theus-arsenal/skills/seguridad-perl/SKILL.md +0 -507
- package/vendor/theus-arsenal/skills/seguridad-quarkus/SKILL.md +0 -474
- package/vendor/theus-arsenal/skills/seguridad-springboot/SKILL.md +0 -279
- package/vendor/theus-arsenal/skills/seo/SKILL.md +0 -166
- package/vendor/theus-arsenal/skills/sistema-diseno/SKILL.md +0 -81
- package/vendor/theus-arsenal/skills/sistema-operativo-agentico/SKILL.md +0 -375
- package/vendor/theus-arsenal/skills/sistemas-latencia-critica/SKILL.md +0 -65
- package/vendor/theus-arsenal/skills/tdd-django/SKILL.md +0 -735
- package/vendor/theus-arsenal/skills/tdd-laravel/SKILL.md +0 -673
- package/vendor/theus-arsenal/skills/tdd-quarkus/SKILL.md +0 -817
- package/vendor/theus-arsenal/skills/tdd-springboot/SKILL.md +0 -165
- package/vendor/theus-arsenal/skills/testing-cpp/SKILL.md +0 -328
- package/vendor/theus-arsenal/skills/testing-csharp/SKILL.md +0 -321
- package/vendor/theus-arsenal/skills/testing-e2e/SKILL.md +0 -325
- package/vendor/theus-arsenal/skills/testing-go/SKILL.md +0 -719
- package/vendor/theus-arsenal/skills/testing-kotlin/SKILL.md +0 -829
- package/vendor/theus-arsenal/skills/testing-perl/SKILL.md +0 -479
- package/vendor/theus-arsenal/skills/testing-protocolos-di-swift/SKILL.md +0 -196
- package/vendor/theus-arsenal/skills/testing-python/SKILL.md +0 -815
- package/vendor/theus-arsenal/skills/testing-react/SKILL.md +0 -421
- package/vendor/theus-arsenal/skills/testing-rust/SKILL.md +0 -506
- package/vendor/theus-arsenal/skills/tours-de-codigo/SKILL.md +0 -252
- package/vendor/theus-arsenal/skills/traduccion-documentos-visado/SKILL.md +0 -121
- package/vendor/theus-arsenal/skills/ui-a-vue/SKILL.md +0 -141
- package/vendor/theus-arsenal/skills/validacion-config-red/SKILL.md +0 -211
- package/vendor/theus-arsenal/skills/verificacion-django/SKILL.md +0 -470
- package/vendor/theus-arsenal/skills/verificacion-factual/SKILL.md +0 -142
- package/vendor/theus-arsenal/skills/verificacion-laravel/SKILL.md +0 -184
- package/vendor/theus-arsenal/skills/verificacion-quarkus/SKILL.md +0 -485
- package/vendor/theus-arsenal/skills/verificacion-springboot/SKILL.md +0 -237
- package/vendor/theus-arsenal/skills/videodb/SKILL.md +0 -374
- package/vendor/theus-arsenal/skills/videodb/reference/api-reference.md +0 -550
- package/vendor/theus-arsenal/skills/videodb/reference/capture-reference.md +0 -407
- package/vendor/theus-arsenal/skills/videodb/reference/capture.md +0 -101
- package/vendor/theus-arsenal/skills/videodb/reference/editor.md +0 -443
- package/vendor/theus-arsenal/skills/videodb/reference/generative.md +0 -331
- package/vendor/theus-arsenal/skills/videodb/reference/rtstream-reference.md +0 -564
- package/vendor/theus-arsenal/skills/videodb/reference/rtstream.md +0 -65
- package/vendor/theus-arsenal/skills/videodb/reference/search.md +0 -230
- package/vendor/theus-arsenal/skills/videodb/reference/streaming.md +0 -406
- package/vendor/theus-arsenal/skills/videodb/reference/use-cases.md +0 -118
- package/vendor/theus-arsenal/skills/videodb/scripts/ws_listener.py +0 -279
- package/vendor/theus-arsenal/skills/videos-manim/SKILL.md +0 -91
- package/vendor/theus-arsenal/skills/videos-manim/assets/network_graph_scene.py +0 -52
- package/vendor/theus-arsenal/skills/vigilancia-canary/SKILL.md +0 -108
- package/vendor/theus-arsenal/skills/voz-de-marca/SKILL.md +0 -106
|
@@ -0,0 +1,2522 @@
|
|
|
1
|
+
"""Inspect/clean AI provenance metadata in non-raster containers.
|
|
2
|
+
|
|
3
|
+
Formats: SVG, PDF (best-effort), DOCX, ODT, HTML, Markdown frontmatter, EPUB.
|
|
4
|
+
Stdlib-first; PDF prefers optional exiftool/c2patool when present.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import base64
|
|
10
|
+
import io
|
|
11
|
+
import os
|
|
12
|
+
import posixpath
|
|
13
|
+
import re
|
|
14
|
+
import subprocess
|
|
15
|
+
import tempfile
|
|
16
|
+
import time
|
|
17
|
+
import urllib.parse
|
|
18
|
+
import zipfile
|
|
19
|
+
import zlib
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from pathlib import Path
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from common import (
|
|
25
|
+
c2patool_probe_note,
|
|
26
|
+
classify_finding_confidence,
|
|
27
|
+
safe_arg,
|
|
28
|
+
safe_write_bytes,
|
|
29
|
+
safe_write_text,
|
|
30
|
+
subprocess_creationflags,
|
|
31
|
+
subprocess_preexec_fn,
|
|
32
|
+
which,
|
|
33
|
+
)
|
|
34
|
+
from image_meta import (
|
|
35
|
+
AI_META_HINTS,
|
|
36
|
+
C2PA_MARKERS,
|
|
37
|
+
inspect_bmp,
|
|
38
|
+
inspect_gif,
|
|
39
|
+
inspect_isobmff,
|
|
40
|
+
inspect_jpeg,
|
|
41
|
+
inspect_png,
|
|
42
|
+
inspect_tiff,
|
|
43
|
+
inspect_webp,
|
|
44
|
+
run_optional_tools,
|
|
45
|
+
strip_bmp,
|
|
46
|
+
strip_gif,
|
|
47
|
+
strip_isobmff,
|
|
48
|
+
strip_jpeg,
|
|
49
|
+
strip_png,
|
|
50
|
+
strip_tiff,
|
|
51
|
+
strip_webp,
|
|
52
|
+
)
|
|
53
|
+
from image_meta import (
|
|
54
|
+
detect_format as detect_image_format,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class ZipBudgetExceeded(Exception):
|
|
59
|
+
"""A zip's declared decompressed size exceeds the processing cap.
|
|
60
|
+
|
|
61
|
+
Kept separate from the parse errors below so a refused zip bomb keeps
|
|
62
|
+
propagating (as it already does out of the clean_* helpers) instead of
|
|
63
|
+
being reported as an unparseable container.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# A corrupt, truncated, encrypted, or unsupported-compression zip surfaces as
|
|
68
|
+
# more than just BadZipFile: reading a member can raise NotImplementedError
|
|
69
|
+
# (unknown compression/version), RuntimeError (encrypted), zlib.error (bad
|
|
70
|
+
# deflate stream), EOFError (truncated), OSError (invalid stream), or
|
|
71
|
+
# ValueError (e.g. a negative seek).
|
|
72
|
+
_ZIP_PARSE_ERRORS = (
|
|
73
|
+
zipfile.BadZipFile,
|
|
74
|
+
zipfile.LargeZipFile,
|
|
75
|
+
NotImplementedError,
|
|
76
|
+
RuntimeError,
|
|
77
|
+
EOFError,
|
|
78
|
+
OSError,
|
|
79
|
+
ValueError,
|
|
80
|
+
zlib.error,
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
# Frontmatter / meta keys that often carry AI provenance
|
|
84
|
+
AI_FRONTMATTER_KEYS = frozenset(
|
|
85
|
+
{
|
|
86
|
+
"generator",
|
|
87
|
+
"ai",
|
|
88
|
+
"ai_generated",
|
|
89
|
+
"ai-generated",
|
|
90
|
+
"claude",
|
|
91
|
+
"anthropic",
|
|
92
|
+
"openai",
|
|
93
|
+
"gemini",
|
|
94
|
+
"synthid",
|
|
95
|
+
"c2pa",
|
|
96
|
+
"content_credentials",
|
|
97
|
+
"contentcredentials",
|
|
98
|
+
"provenance",
|
|
99
|
+
"digital_source_type",
|
|
100
|
+
"digitalsourcetype",
|
|
101
|
+
"created_with",
|
|
102
|
+
"createdwith",
|
|
103
|
+
"model",
|
|
104
|
+
"llm",
|
|
105
|
+
}
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
AI_META_NAME_RE = re.compile(
|
|
109
|
+
r"generator|ai[-_ ]?generated|claude|anthropic|openai|gemini|synthid|"
|
|
110
|
+
r"c2pa|content.?credential|provenance|digital.?source|aigc",
|
|
111
|
+
re.I,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
SVG_DROP_TAGS = frozenset(
|
|
115
|
+
{
|
|
116
|
+
"{http://www.w3.org/2000/svg}metadata",
|
|
117
|
+
"metadata",
|
|
118
|
+
"{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF",
|
|
119
|
+
"{adobe:ns:meta/}xmpmeta",
|
|
120
|
+
}
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass
|
|
125
|
+
class ContainerInspectReport:
|
|
126
|
+
path: str
|
|
127
|
+
format: str
|
|
128
|
+
has_c2pa: bool
|
|
129
|
+
has_ai_metadata: bool
|
|
130
|
+
findings: list[str] = field(default_factory=list)
|
|
131
|
+
tools: dict[str, Any] = field(default_factory=dict)
|
|
132
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
133
|
+
notes: list[str] = field(default_factory=list)
|
|
134
|
+
# Layer A (invisible/format Unicode) scan of the text body, populated only
|
|
135
|
+
# for the formats clean_container() actually scrubs. Without it, inspect
|
|
136
|
+
# reported a markdown/html file carrying invisible carriers as clean while
|
|
137
|
+
# clean then went on to remove them.
|
|
138
|
+
layer_a_total: int = 0
|
|
139
|
+
layer_a_hits: list[dict] = field(default_factory=list)
|
|
140
|
+
|
|
141
|
+
def to_dict(self) -> dict:
|
|
142
|
+
return {
|
|
143
|
+
"path": self.path,
|
|
144
|
+
"format": self.format,
|
|
145
|
+
"has_c2pa": self.has_c2pa,
|
|
146
|
+
"has_ai_metadata": self.has_ai_metadata,
|
|
147
|
+
"findings": self.findings,
|
|
148
|
+
"findings_confidence": [classify_finding_confidence(f) for f in self.findings],
|
|
149
|
+
"tools": self.tools,
|
|
150
|
+
"details": self.details,
|
|
151
|
+
"notes": self.notes,
|
|
152
|
+
# Same key as TextInspectReport so every caller — including the
|
|
153
|
+
# HTTP server's `suspicious` flag — reads both report kinds alike.
|
|
154
|
+
"suspicious_total": self.layer_a_total,
|
|
155
|
+
"layer_a_hits": self.layer_a_hits,
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def detect_container_format(path: Path, data: bytes | None = None) -> str:
|
|
160
|
+
ext = path.suffix.lower()
|
|
161
|
+
if ext in (".svg",):
|
|
162
|
+
return "svg"
|
|
163
|
+
if ext in (".pdf",):
|
|
164
|
+
return "pdf"
|
|
165
|
+
if ext in (".docx",):
|
|
166
|
+
return "docx"
|
|
167
|
+
if ext in (".xlsx",):
|
|
168
|
+
return "xlsx"
|
|
169
|
+
if ext in (".pptx",):
|
|
170
|
+
return "pptx"
|
|
171
|
+
if ext in (".odt",):
|
|
172
|
+
return "odt"
|
|
173
|
+
if ext in (".epub",):
|
|
174
|
+
return "epub"
|
|
175
|
+
if ext in (".html", ".htm"):
|
|
176
|
+
return "html"
|
|
177
|
+
if ext in (".md", ".markdown", ".mdx"):
|
|
178
|
+
return "markdown"
|
|
179
|
+
if data is not None:
|
|
180
|
+
if data[:4] == b"%PDF":
|
|
181
|
+
return "pdf"
|
|
182
|
+
if data[:100].lstrip().startswith(b"<") and b"svg" in data[:500].lower():
|
|
183
|
+
return "svg"
|
|
184
|
+
if data[:2] == b"PK":
|
|
185
|
+
# zip-based; sniff
|
|
186
|
+
try:
|
|
187
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
|
188
|
+
names = set(zf.namelist())
|
|
189
|
+
if "word/document.xml" in names:
|
|
190
|
+
return "docx"
|
|
191
|
+
if "xl/workbook.xml" in names:
|
|
192
|
+
return "xlsx"
|
|
193
|
+
if "ppt/presentation.xml" in names:
|
|
194
|
+
return "pptx"
|
|
195
|
+
if "content.xml" in names and "meta.xml" in names:
|
|
196
|
+
return "odt"
|
|
197
|
+
if "META-INF/container.xml" in names and any(n.endswith(".opf") for n in names):
|
|
198
|
+
return "epub"
|
|
199
|
+
except _ZIP_PARSE_ERRORS:
|
|
200
|
+
pass
|
|
201
|
+
return "unknown"
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _blob_hits(blob: bytes) -> tuple[bool, bool, list[str]]:
|
|
205
|
+
lower = blob.lower()
|
|
206
|
+
findings: list[str] = []
|
|
207
|
+
has_c2pa = False
|
|
208
|
+
has_ai = False
|
|
209
|
+
for n in C2PA_MARKERS:
|
|
210
|
+
if n.lower() in lower:
|
|
211
|
+
has_c2pa = True
|
|
212
|
+
findings.append(f"marker:{n.decode('ascii', errors='replace')}")
|
|
213
|
+
for n in AI_META_HINTS:
|
|
214
|
+
if n.lower() in lower:
|
|
215
|
+
has_ai = True
|
|
216
|
+
label = n.decode("ascii", errors="replace")
|
|
217
|
+
if label not in {f.split(":", 1)[-1] for f in findings}:
|
|
218
|
+
findings.append(f"ai:{label}")
|
|
219
|
+
return has_c2pa, has_ai or has_c2pa, findings[:30]
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
RE_DATA_IMAGE_URI = re.compile(
|
|
223
|
+
r"data:image\/(?P<mime>[a-zA-Z0-9\+\-\.]+)(?P<params>;[^\s\"'\)<>]+)?,(?P<payload>[A-Za-z0-9+/=\s%]+)",
|
|
224
|
+
re.I,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _inspect_embedded_data_uris(text: str) -> tuple[bool, bool, list[str]]:
|
|
229
|
+
has_c2pa = False
|
|
230
|
+
has_ai = False
|
|
231
|
+
findings: list[str] = []
|
|
232
|
+
|
|
233
|
+
for m in RE_DATA_IMAGE_URI.finditer(text):
|
|
234
|
+
mime = m.group("mime").lower()
|
|
235
|
+
params = (m.group("params") or "").lower()
|
|
236
|
+
payload = m.group("payload")
|
|
237
|
+
is_b64 = "base64" in params
|
|
238
|
+
|
|
239
|
+
try:
|
|
240
|
+
if is_b64:
|
|
241
|
+
raw_b64 = re.sub(r"\s+", "", payload)
|
|
242
|
+
pad = len(raw_b64) % 4
|
|
243
|
+
if pad:
|
|
244
|
+
raw_b64 += "=" * (4 - pad)
|
|
245
|
+
data = base64.b64decode(raw_b64)
|
|
246
|
+
else:
|
|
247
|
+
data = urllib.parse.unquote_to_bytes(payload)
|
|
248
|
+
except Exception: # noqa: S112 - malformed data URI; skip to next match
|
|
249
|
+
continue
|
|
250
|
+
|
|
251
|
+
if not data:
|
|
252
|
+
continue
|
|
253
|
+
|
|
254
|
+
fmt = detect_image_format(data)
|
|
255
|
+
if fmt == "png":
|
|
256
|
+
sub_c2pa, sub_ai, sub_findings = inspect_png(data)
|
|
257
|
+
elif fmt == "jpeg":
|
|
258
|
+
sub_c2pa, sub_ai, sub_findings = inspect_jpeg(data)
|
|
259
|
+
elif fmt == "webp":
|
|
260
|
+
sub_c2pa, sub_ai, sub_findings = inspect_webp(data)
|
|
261
|
+
elif fmt in ("avif", "heic"):
|
|
262
|
+
sub_c2pa, sub_ai, sub_findings = inspect_isobmff(data, fmt)
|
|
263
|
+
elif "svg" in mime or data.lstrip().startswith(b"<"):
|
|
264
|
+
sub_c2pa, sub_ai, sub_findings, _ = inspect_svg(data)
|
|
265
|
+
else:
|
|
266
|
+
sub_c2pa, sub_ai, sub_findings = _blob_hits(data)
|
|
267
|
+
|
|
268
|
+
if sub_c2pa:
|
|
269
|
+
has_c2pa = True
|
|
270
|
+
if sub_ai or sub_c2pa:
|
|
271
|
+
has_ai = True
|
|
272
|
+
for f in sub_findings:
|
|
273
|
+
findings.append(f"embedded data:image/{mime}: {f}")
|
|
274
|
+
|
|
275
|
+
return has_c2pa, has_ai, findings
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
def _clean_embedded_data_uris(
|
|
279
|
+
text: str, *, strip_all_metadata: bool = True
|
|
280
|
+
) -> tuple[str, list[str]]:
|
|
281
|
+
actions: list[str] = []
|
|
282
|
+
|
|
283
|
+
def _replace_uri(m: re.Match[str]) -> str:
|
|
284
|
+
full_match = m.group(0)
|
|
285
|
+
mime = m.group("mime")
|
|
286
|
+
params = m.group("params") or ""
|
|
287
|
+
payload = m.group("payload")
|
|
288
|
+
is_b64 = "base64" in params.lower()
|
|
289
|
+
|
|
290
|
+
try:
|
|
291
|
+
if is_b64:
|
|
292
|
+
raw_b64 = re.sub(r"\s+", "", payload)
|
|
293
|
+
pad = len(raw_b64) % 4
|
|
294
|
+
if pad:
|
|
295
|
+
raw_b64 += "=" * (4 - pad)
|
|
296
|
+
data = base64.b64decode(raw_b64)
|
|
297
|
+
else:
|
|
298
|
+
data = urllib.parse.unquote_to_bytes(payload)
|
|
299
|
+
except Exception:
|
|
300
|
+
return full_match
|
|
301
|
+
|
|
302
|
+
if not data:
|
|
303
|
+
return full_match
|
|
304
|
+
|
|
305
|
+
fmt = detect_image_format(data)
|
|
306
|
+
sub_actions: list[str] = []
|
|
307
|
+
cleaned_bytes = data
|
|
308
|
+
|
|
309
|
+
try:
|
|
310
|
+
if fmt == "png":
|
|
311
|
+
cleaned_bytes, sub_actions = strip_png(data, strip_all_text=strip_all_metadata)
|
|
312
|
+
elif fmt == "jpeg":
|
|
313
|
+
cleaned_bytes, sub_actions = strip_jpeg(data, strip_all_app=strip_all_metadata)
|
|
314
|
+
elif fmt == "webp":
|
|
315
|
+
cleaned_bytes, sub_actions = strip_webp(data, strip_all_metadata=strip_all_metadata)
|
|
316
|
+
elif fmt in ("avif", "heic"):
|
|
317
|
+
cleaned_bytes, sub_actions = strip_isobmff(
|
|
318
|
+
data, fmt, strip_all_metadata=strip_all_metadata
|
|
319
|
+
)
|
|
320
|
+
elif "svg" in mime.lower() or data.lstrip().startswith(b"<"):
|
|
321
|
+
cleaned_bytes, sub_actions = clean_svg(data)
|
|
322
|
+
except Exception:
|
|
323
|
+
return full_match
|
|
324
|
+
|
|
325
|
+
if not any("drop" in a.lower() for a in sub_actions) or cleaned_bytes == data:
|
|
326
|
+
return full_match
|
|
327
|
+
|
|
328
|
+
actions.append(f"cleaned embedded data:image/{mime} ({', '.join(sub_actions[:2])})")
|
|
329
|
+
|
|
330
|
+
if is_b64:
|
|
331
|
+
new_b64 = base64.b64encode(cleaned_bytes).decode("ascii")
|
|
332
|
+
return f"data:image/{mime}{params},{new_b64}"
|
|
333
|
+
else:
|
|
334
|
+
new_payload = urllib.parse.quote_from_bytes(cleaned_bytes)
|
|
335
|
+
return f"data:image/{mime}{params},{new_payload}"
|
|
336
|
+
|
|
337
|
+
out = RE_DATA_IMAGE_URI.sub(_replace_uri, text)
|
|
338
|
+
return out, actions
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
# ---------------------------------------------------------------------------
|
|
342
|
+
# Linear tag-block scanning
|
|
343
|
+
# ---------------------------------------------------------------------------
|
|
344
|
+
#
|
|
345
|
+
# The lazy ".*?</close>" idiom is quadratic on adversarial input: with many
|
|
346
|
+
# opening tags and no closing tag, the engine rescans to end-of-input from
|
|
347
|
+
# every candidate start position (CPython's "re" holds the GIL, so one such
|
|
348
|
+
# request stalls the whole single-process service for its duration). The
|
|
349
|
+
# helpers below locate every opening and closing tag once and pair them with a
|
|
350
|
+
# forward pointer - O(n) total regardless of input shape - while preserving
|
|
351
|
+
# the exact match semantics of re.subn with a lazy ".*?" (the first closing
|
|
352
|
+
# tag at/after each opening tag's end, blocks never overlapping).
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _iter_tag_blocks(text, open_re, close_re):
|
|
356
|
+
"""Yield (open_start, open_end, close_start, close_end) for each block.
|
|
357
|
+
|
|
358
|
+
Works on str and bytes alike. Linear in len(text): closing tags are
|
|
359
|
+
collected once with finditer and consumed by a forward pointer, so an
|
|
360
|
+
unbounded run of unclosed opening tags costs one pass, not one rescan per
|
|
361
|
+
opening tag.
|
|
362
|
+
"""
|
|
363
|
+
closes = list(close_re.finditer(text))
|
|
364
|
+
ci = 0
|
|
365
|
+
last_end = 0
|
|
366
|
+
for om in open_re.finditer(text):
|
|
367
|
+
if om.start() < last_end:
|
|
368
|
+
continue
|
|
369
|
+
while ci < len(closes) and closes[ci].start() < om.end():
|
|
370
|
+
ci += 1
|
|
371
|
+
if ci >= len(closes):
|
|
372
|
+
return
|
|
373
|
+
cm = closes[ci]
|
|
374
|
+
yield om.start(), om.end(), cm.start(), cm.end()
|
|
375
|
+
last_end = cm.end()
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _drop_tag_blocks(text, open_re, close_re):
|
|
379
|
+
"""Remove every open...close block. Return (text, count). Linear time."""
|
|
380
|
+
out = []
|
|
381
|
+
last = 0
|
|
382
|
+
count = 0
|
|
383
|
+
for os_, _oe, _cs, ce in _iter_tag_blocks(text, open_re, close_re):
|
|
384
|
+
out.append(text[last:os_])
|
|
385
|
+
last = ce
|
|
386
|
+
count += 1
|
|
387
|
+
if not count:
|
|
388
|
+
return text, 0
|
|
389
|
+
out.append(text[last:])
|
|
390
|
+
return text[:0].join(out), count
|
|
391
|
+
|
|
392
|
+
|
|
393
|
+
def _drop_blocks_if(text, open_re, close_re, predicate):
|
|
394
|
+
"""Drop blocks whose full text satisfies predicate. Return (text, count).
|
|
395
|
+
|
|
396
|
+
Blocks failing the predicate are kept verbatim; predicate is called with
|
|
397
|
+
the whole block (open tag through closing tag), matching the
|
|
398
|
+
callback-based re.sub sites this replaces.
|
|
399
|
+
"""
|
|
400
|
+
out = []
|
|
401
|
+
last = 0
|
|
402
|
+
count = 0
|
|
403
|
+
for os_, _oe, _cs, ce in _iter_tag_blocks(text, open_re, close_re):
|
|
404
|
+
if not predicate(text[os_:ce]):
|
|
405
|
+
continue
|
|
406
|
+
out.append(text[last:os_])
|
|
407
|
+
last = ce
|
|
408
|
+
count += 1
|
|
409
|
+
if not count:
|
|
410
|
+
return text, 0
|
|
411
|
+
out.append(text[last:])
|
|
412
|
+
return text[:0].join(out), count
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
# ---------------------------------------------------------------------------
|
|
416
|
+
# Markdown frontmatter
|
|
417
|
+
# ---------------------------------------------------------------------------
|
|
418
|
+
|
|
419
|
+
_FM_RE = re.compile(r"\A---\r?\n(.*?)\r?\n---\r?\n?", re.DOTALL)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _parse_simple_yaml_keys(block: str) -> list[tuple[str, str, int]]:
|
|
423
|
+
"""Return list of (key, full_line, line_index) for top-level keys only."""
|
|
424
|
+
rows: list[tuple[str, str, int]] = []
|
|
425
|
+
for i, line in enumerate(block.splitlines()):
|
|
426
|
+
if not line.strip() or line.strip().startswith("#"):
|
|
427
|
+
continue
|
|
428
|
+
if line[0] in (" ", "\t", "-"):
|
|
429
|
+
continue # nested / list — leave alone
|
|
430
|
+
m = re.match(r"^([A-Za-z0-9_.-]+)\s*:", line)
|
|
431
|
+
if m:
|
|
432
|
+
rows.append((m.group(1), line, i))
|
|
433
|
+
return rows
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def inspect_markdown(text: str) -> tuple[bool, bool, list[str], dict]:
|
|
437
|
+
findings: list[str] = []
|
|
438
|
+
has_ai = False
|
|
439
|
+
has_fm = False
|
|
440
|
+
keys = []
|
|
441
|
+
m = _FM_RE.match(text)
|
|
442
|
+
if m:
|
|
443
|
+
has_fm = True
|
|
444
|
+
block = m.group(1)
|
|
445
|
+
for key, _line, _i in _parse_simple_yaml_keys(block):
|
|
446
|
+
keys.append(key)
|
|
447
|
+
if key.lower() in AI_FRONTMATTER_KEYS or AI_META_NAME_RE.search(key):
|
|
448
|
+
has_ai = True
|
|
449
|
+
findings.append(f"frontmatter key: {key}")
|
|
450
|
+
# also check value
|
|
451
|
+
val = _line.split(":", 1)[1] if ":" in _line else ""
|
|
452
|
+
if AI_META_NAME_RE.search(val):
|
|
453
|
+
has_ai = True
|
|
454
|
+
findings.append(f"frontmatter value hit on {key}")
|
|
455
|
+
|
|
456
|
+
uri_c2pa, uri_ai, uri_findings = _inspect_embedded_data_uris(text)
|
|
457
|
+
if uri_c2pa:
|
|
458
|
+
has_ai = True
|
|
459
|
+
if uri_ai:
|
|
460
|
+
has_ai = True
|
|
461
|
+
findings.extend(uri_findings)
|
|
462
|
+
|
|
463
|
+
c2pa = uri_c2pa or any("c2pa" in f.lower() or "content" in f.lower() for f in findings)
|
|
464
|
+
return c2pa, has_ai, findings, {"has_frontmatter": has_fm, "keys": keys}
|
|
465
|
+
|
|
466
|
+
|
|
467
|
+
def clean_markdown(text: str) -> tuple[str, list[str]]:
|
|
468
|
+
actions: list[str] = []
|
|
469
|
+
m = _FM_RE.match(text)
|
|
470
|
+
if m:
|
|
471
|
+
block = m.group(1)
|
|
472
|
+
body = text[m.end() :]
|
|
473
|
+
kept: list[str] = []
|
|
474
|
+
dropping = False # inside the nested block of a dropped top-level key
|
|
475
|
+
for line in block.splitlines():
|
|
476
|
+
stripped = line.strip()
|
|
477
|
+
|
|
478
|
+
# Blank lines and comments belong to whichever block we are inside.
|
|
479
|
+
if not stripped or stripped.startswith("#"):
|
|
480
|
+
if not dropping:
|
|
481
|
+
kept.append(line)
|
|
482
|
+
continue
|
|
483
|
+
|
|
484
|
+
# Continuation lines (nested mappings, list items) follow their parent.
|
|
485
|
+
if line[0] in (" ", "\t", "-"):
|
|
486
|
+
if not dropping:
|
|
487
|
+
kept.append(line)
|
|
488
|
+
continue
|
|
489
|
+
|
|
490
|
+
km = re.match(r"^([A-Za-z0-9_.-]+)\s*:", line)
|
|
491
|
+
if not km:
|
|
492
|
+
dropping = False
|
|
493
|
+
kept.append(line)
|
|
494
|
+
continue
|
|
495
|
+
|
|
496
|
+
key = km.group(1)
|
|
497
|
+
val = line.split(":", 1)[1] if ":" in line else ""
|
|
498
|
+
if key.lower() in AI_FRONTMATTER_KEYS or AI_META_NAME_RE.search(key):
|
|
499
|
+
actions.append(f"drop frontmatter key: {key}")
|
|
500
|
+
dropping = True
|
|
501
|
+
continue
|
|
502
|
+
if AI_META_NAME_RE.search(val):
|
|
503
|
+
actions.append(f"drop frontmatter key (value hit): {key}")
|
|
504
|
+
dropping = True
|
|
505
|
+
continue
|
|
506
|
+
|
|
507
|
+
dropping = False
|
|
508
|
+
kept.append(line)
|
|
509
|
+
new_block = "\n".join(kept).strip("\n")
|
|
510
|
+
if new_block:
|
|
511
|
+
out = f"---\n{new_block}\n---\n{body}"
|
|
512
|
+
else:
|
|
513
|
+
out = body.lstrip("\n")
|
|
514
|
+
actions.append("removed empty frontmatter block")
|
|
515
|
+
else:
|
|
516
|
+
out = text
|
|
517
|
+
|
|
518
|
+
out, uri_actions = _clean_embedded_data_uris(out)
|
|
519
|
+
if uri_actions:
|
|
520
|
+
actions.extend(uri_actions)
|
|
521
|
+
|
|
522
|
+
if not actions:
|
|
523
|
+
actions.append("no AI frontmatter keys or embedded data URIs removed")
|
|
524
|
+
return out, actions
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
# ---------------------------------------------------------------------------
|
|
528
|
+
# HTML
|
|
529
|
+
# ---------------------------------------------------------------------------
|
|
530
|
+
|
|
531
|
+
_META_TAG_RE = re.compile(
|
|
532
|
+
r"<meta\b[^>]*>",
|
|
533
|
+
re.I,
|
|
534
|
+
)
|
|
535
|
+
_META_ATTR_RE = re.compile(
|
|
536
|
+
r"""(name|property|content|generator)\s*=\s*["']([^"']*)["']""",
|
|
537
|
+
re.I,
|
|
538
|
+
)
|
|
539
|
+
|
|
540
|
+
# Known AI vendor names for the "generator" meta tag. A plain CMS generator
|
|
541
|
+
# (WordPress, Elementor) is CMS provenance, not AI-generator metadata.
|
|
542
|
+
_GENERATOR_AI_RE = re.compile(
|
|
543
|
+
r"claude|anthropic|openai|chatgpt|gemini|synthid|copilot|midjourney|dall.?e|stable.?diffusion",
|
|
544
|
+
re.I,
|
|
545
|
+
)
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def _meta_attrs(tag: str) -> dict[str, str]:
|
|
549
|
+
return {name.lower(): value for name, value in _META_ATTR_RE.findall(tag)}
|
|
550
|
+
|
|
551
|
+
|
|
552
|
+
def _is_cms_generator_meta(tag: str) -> bool:
|
|
553
|
+
"""Return True for a generator meta tag that is CMS provenance, not AI."""
|
|
554
|
+
attrs = _meta_attrs(tag)
|
|
555
|
+
name_or_prop = (
|
|
556
|
+
attrs.get("name") or attrs.get("property") or attrs.get("generator") or ""
|
|
557
|
+
).lower()
|
|
558
|
+
if name_or_prop != "generator":
|
|
559
|
+
return False
|
|
560
|
+
return not (_GENERATOR_AI_RE.search(attrs.get("content", "")) or _GENERATOR_AI_RE.search(tag))
|
|
561
|
+
|
|
562
|
+
|
|
563
|
+
_JSONLD_OPEN_RE = re.compile(
|
|
564
|
+
r"""<script\b[^>]*type\s*=\s*["']application/ld\+json["'][^>]*>""",
|
|
565
|
+
re.I,
|
|
566
|
+
)
|
|
567
|
+
_JSONLD_CLOSE_RE = re.compile(r"</script>", re.I)
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def inspect_html(text: str) -> tuple[bool, bool, list[str], dict]:
|
|
571
|
+
findings: list[str] = []
|
|
572
|
+
has_ai = False
|
|
573
|
+
has_c2pa = False
|
|
574
|
+
for tag in _META_TAG_RE.findall(text):
|
|
575
|
+
if re.search(r"c2pa|content.?credential", tag, re.I):
|
|
576
|
+
has_c2pa = True
|
|
577
|
+
if _is_cms_generator_meta(tag):
|
|
578
|
+
findings.append(f"info: cms generator: {tag[:120]}")
|
|
579
|
+
continue
|
|
580
|
+
if AI_META_NAME_RE.search(tag) or any(
|
|
581
|
+
h.decode("ascii", "ignore").lower() in tag.lower() for h in AI_META_HINTS[:12]
|
|
582
|
+
):
|
|
583
|
+
has_ai = True
|
|
584
|
+
findings.append(f"meta: {tag[:120]}")
|
|
585
|
+
for os_, _oe, _cs, ce in _iter_tag_blocks(text, _JSONLD_OPEN_RE, _JSONLD_CLOSE_RE):
|
|
586
|
+
blob = text[os_:ce]
|
|
587
|
+
if AI_META_NAME_RE.search(blob) or re.search(
|
|
588
|
+
r"DigitalSourceType|trainedAlgorithmicMedia|SoftwareAgent", blob, re.I
|
|
589
|
+
):
|
|
590
|
+
has_ai = True
|
|
591
|
+
findings.append("json-ld provenance-like block")
|
|
592
|
+
if re.search(r"c2pa|contentcredential", blob, re.I):
|
|
593
|
+
has_c2pa = True
|
|
594
|
+
# data-ai* attributes
|
|
595
|
+
for m in re.finditer(r"\bdata-ai[\w-]*\s*=\s*[\"'][^\"']*[\"']", text, re.I):
|
|
596
|
+
has_ai = True
|
|
597
|
+
findings.append(f"attr: {m.group(0)[:80]}")
|
|
598
|
+
|
|
599
|
+
uri_c2pa, uri_ai, uri_findings = _inspect_embedded_data_uris(text)
|
|
600
|
+
if uri_c2pa:
|
|
601
|
+
has_c2pa = True
|
|
602
|
+
if uri_ai:
|
|
603
|
+
has_ai = True
|
|
604
|
+
findings.extend(uri_findings)
|
|
605
|
+
|
|
606
|
+
return has_c2pa, has_ai, findings, {}
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def clean_html(text: str) -> tuple[str, list[str]]:
|
|
610
|
+
actions: list[str] = []
|
|
611
|
+
|
|
612
|
+
def _meta_sub(m: re.Match[str]) -> str:
|
|
613
|
+
tag = m.group(0)
|
|
614
|
+
if _is_cms_generator_meta(tag):
|
|
615
|
+
return tag
|
|
616
|
+
if AI_META_NAME_RE.search(tag) or re.search(
|
|
617
|
+
r"generator|claude|anthropic|openai|gemini|synthid|c2pa|aigc", tag, re.I
|
|
618
|
+
):
|
|
619
|
+
actions.append(f"drop meta: {tag[:80]}")
|
|
620
|
+
return ""
|
|
621
|
+
return tag
|
|
622
|
+
|
|
623
|
+
out = _META_TAG_RE.sub(_meta_sub, text)
|
|
624
|
+
|
|
625
|
+
def _jsonld_is_ai(blob: str) -> bool:
|
|
626
|
+
return AI_META_NAME_RE.search(blob) or re.search(
|
|
627
|
+
r"DigitalSourceType|trainedAlgorithmicMedia|SoftwareAgent", blob, re.I
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
new, n = _drop_blocks_if(out, _JSONLD_OPEN_RE, _JSONLD_CLOSE_RE, _jsonld_is_ai)
|
|
631
|
+
if n:
|
|
632
|
+
actions.extend(["drop json-ld provenance-like script"] * n)
|
|
633
|
+
out = new
|
|
634
|
+
out2, n = re.subn(r"\sdata-ai[\w-]*\s*=\s*[\"'][^\"']*[\"']", "", out, flags=re.I)
|
|
635
|
+
if n:
|
|
636
|
+
actions.append(f"drop data-ai* attributes x{n}")
|
|
637
|
+
out = out2
|
|
638
|
+
|
|
639
|
+
out, uri_actions = _clean_embedded_data_uris(out)
|
|
640
|
+
if uri_actions:
|
|
641
|
+
actions.extend(uri_actions)
|
|
642
|
+
|
|
643
|
+
if not actions:
|
|
644
|
+
actions.append("no HTML AI meta removed")
|
|
645
|
+
return out, actions
|
|
646
|
+
|
|
647
|
+
|
|
648
|
+
# ---------------------------------------------------------------------------
|
|
649
|
+
# SVG
|
|
650
|
+
# ---------------------------------------------------------------------------
|
|
651
|
+
|
|
652
|
+
# Opening/closing tag patterns for the linear block scans below. Kept as
|
|
653
|
+
# separate open/close halves: the lazy ".*?</close>" form is quadratic when
|
|
654
|
+
# many opening tags have no closing tag (see _iter_tag_blocks).
|
|
655
|
+
_SVG_METADATA_OPEN_RE = re.compile(r"<metadata\b[^>]*>", re.I)
|
|
656
|
+
_SVG_METADATA_CLOSE_RE = re.compile(r"</metadata\s*>", re.I)
|
|
657
|
+
_SVG_XMPMETA_OPEN_RE = re.compile(r"<x:xmpmeta\b[^>]*>", re.I)
|
|
658
|
+
_SVG_XMPMETA_CLOSE_RE = re.compile(r"</x:xmpmeta\s*>", re.I)
|
|
659
|
+
_SVG_COMMENT_OPEN_RE = re.compile(r"<!--")
|
|
660
|
+
# HTML comment end tags are "-->" or "--!>" (CodeQL: bad HTML filtering
|
|
661
|
+
# regexp otherwise). XML/SVG only emits "-->", but accepting both keeps the
|
|
662
|
+
# scrub effective on HTML-flavoured inputs.
|
|
663
|
+
_SVG_COMMENT_CLOSE_RE = re.compile(r"--!?>")
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def inspect_svg(data: bytes) -> tuple[bool, bool, list[str], dict]:
|
|
667
|
+
findings: list[str] = []
|
|
668
|
+
has_c2pa, has_ai, hits = _blob_hits(data)
|
|
669
|
+
findings.extend(hits)
|
|
670
|
+
try:
|
|
671
|
+
text = data.decode("utf-8", errors="replace")
|
|
672
|
+
if re.search(r"<metadata[\s>]", text, re.I):
|
|
673
|
+
findings.append("svg <metadata> present")
|
|
674
|
+
has_ai = True # often XMP; treat as inspect signal
|
|
675
|
+
if re.search(r"xmpmeta|rdf:RDF|contentcredentials", text, re.I):
|
|
676
|
+
has_ai = True
|
|
677
|
+
findings.append("XMP/RDF-like content in SVG")
|
|
678
|
+
if re.search(r"c2pa|jumbf", text, re.I):
|
|
679
|
+
has_c2pa = True
|
|
680
|
+
|
|
681
|
+
uri_c2pa, uri_ai, uri_findings = _inspect_embedded_data_uris(text)
|
|
682
|
+
if uri_c2pa:
|
|
683
|
+
has_c2pa = True
|
|
684
|
+
if uri_ai:
|
|
685
|
+
has_ai = True
|
|
686
|
+
findings.extend(uri_findings)
|
|
687
|
+
except Exception as e:
|
|
688
|
+
findings.append(f"svg decode note: {e}")
|
|
689
|
+
return has_c2pa, has_ai or has_c2pa, findings, {}
|
|
690
|
+
|
|
691
|
+
|
|
692
|
+
def clean_svg(data: bytes) -> tuple[bytes, list[str]]:
|
|
693
|
+
actions: list[str] = []
|
|
694
|
+
text = data.decode("utf-8", errors="surrogateescape")
|
|
695
|
+
# Drop metadata blocks (linear scan - lazy .*? is quadratic on unclosed tags)
|
|
696
|
+
new, n = _drop_tag_blocks(text, _SVG_METADATA_OPEN_RE, _SVG_METADATA_CLOSE_RE)
|
|
697
|
+
if n:
|
|
698
|
+
actions.append(f"drop <metadata> x{n}")
|
|
699
|
+
text = new
|
|
700
|
+
# Drop adobe xmp packets
|
|
701
|
+
new, n = _drop_tag_blocks(text, _SVG_XMPMETA_OPEN_RE, _SVG_XMPMETA_CLOSE_RE)
|
|
702
|
+
if n:
|
|
703
|
+
actions.append(f"drop xmpmeta x{n}")
|
|
704
|
+
text = new
|
|
705
|
+
|
|
706
|
+
# Drop comments that look like provenance (linear scan)
|
|
707
|
+
def _cmt(block: str) -> bool:
|
|
708
|
+
return bool(AI_META_NAME_RE.search(block))
|
|
709
|
+
|
|
710
|
+
new, n = _drop_blocks_if(text, _SVG_COMMENT_OPEN_RE, _SVG_COMMENT_CLOSE_RE, _cmt)
|
|
711
|
+
if n:
|
|
712
|
+
actions.extend(["drop SVG comment with AI markers"] * n)
|
|
713
|
+
text = new
|
|
714
|
+
|
|
715
|
+
# Clean embedded data URIs
|
|
716
|
+
text, uri_actions = _clean_embedded_data_uris(text)
|
|
717
|
+
if uri_actions:
|
|
718
|
+
actions.extend(uri_actions)
|
|
719
|
+
|
|
720
|
+
if not actions:
|
|
721
|
+
# still strip generator attribute on root if present
|
|
722
|
+
new, n = re.subn(
|
|
723
|
+
r'\s(inkscape:version|sodipodi:docname|generator)\s*=\s*"[^"]*"',
|
|
724
|
+
"",
|
|
725
|
+
text,
|
|
726
|
+
flags=re.I,
|
|
727
|
+
)
|
|
728
|
+
if n:
|
|
729
|
+
actions.append(f"drop generator-like attrs x{n}")
|
|
730
|
+
text = new
|
|
731
|
+
if not actions:
|
|
732
|
+
actions.append("no SVG metadata removed")
|
|
733
|
+
return text.encode("utf-8", errors="surrogateescape"), actions
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
# ---------------------------------------------------------------------------
|
|
737
|
+
# DOCX / ODT (zip + XML)
|
|
738
|
+
# ---------------------------------------------------------------------------
|
|
739
|
+
|
|
740
|
+
# Open/close tag halves for the linear metadata-block scans in clean_odt.
|
|
741
|
+
_ODT_GENERATOR_OPEN_RE = re.compile(r"<meta:generator\b[^>]*>", re.I)
|
|
742
|
+
_ODT_GENERATOR_CLOSE_RE = re.compile(r"</meta:generator\s*>", re.I)
|
|
743
|
+
_DC_CREATOR_OPEN_RE = re.compile(r"<dc:creator\b[^>]*>", re.I)
|
|
744
|
+
_DC_CREATOR_CLOSE_RE = re.compile(r"</dc:creator\s*>", re.I)
|
|
745
|
+
|
|
746
|
+
DOCX_META_PARTS = (
|
|
747
|
+
"docProps/core.xml",
|
|
748
|
+
"docProps/app.xml",
|
|
749
|
+
"docProps/custom.xml",
|
|
750
|
+
)
|
|
751
|
+
DOCX_CUSTOM_PREFIXES = (
|
|
752
|
+
"customXml/",
|
|
753
|
+
"docProps/",
|
|
754
|
+
)
|
|
755
|
+
|
|
756
|
+
# Provenance fields in docProps/core.xml and docProps/app.xml that always come
|
|
757
|
+
# out empty. dc:title is deliberately not listed: it is the document's own
|
|
758
|
+
# heading, not provenance.
|
|
759
|
+
DOCX_SCRUB_FIELDS = (
|
|
760
|
+
("dc:creator", "dc:creator"),
|
|
761
|
+
("cp:lastModifiedBy", "cp:lastModifiedBy"),
|
|
762
|
+
("dc:description", "dc:description"),
|
|
763
|
+
("cp:keywords", "cp:keywords"),
|
|
764
|
+
("dc:subject", "dc:subject"),
|
|
765
|
+
("cp:category", "cp:category"),
|
|
766
|
+
("Application", "Application"),
|
|
767
|
+
("AppVersion", "AppVersion"),
|
|
768
|
+
("Company", "Company"),
|
|
769
|
+
("Manager", "Manager"),
|
|
770
|
+
)
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def _zip_namelist(data: bytes) -> list[str]:
|
|
774
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
|
775
|
+
return zf.namelist()
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
MAX_ZIP_DECOMPRESSED_BYTES = 128 * 1024 * 1024
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def _check_zip_budget(info: zipfile.ZipInfo, budget: list[int]) -> None:
|
|
782
|
+
"""Fast-path zip-bomb guard on the declared member size.
|
|
783
|
+
|
|
784
|
+
A single member whose *declared* size already exceeds the cap is
|
|
785
|
+
rejected before any decompression. The authoritative accounting lives in
|
|
786
|
+
_read_zip_member, which charges **actual** decompressed bytes to the
|
|
787
|
+
shared budget: ZipInfo.file_size comes from the archive central
|
|
788
|
+
directory and is attacker-controlled, so trusting it for the cumulative
|
|
789
|
+
limit would let a crafted archive declare a tiny size and still expand
|
|
790
|
+
to gigabytes.
|
|
791
|
+
"""
|
|
792
|
+
if info.file_size > MAX_ZIP_DECOMPRESSED_BYTES:
|
|
793
|
+
raise ZipBudgetExceeded(
|
|
794
|
+
"zip decompressed size exceeds cap "
|
|
795
|
+
f"({MAX_ZIP_DECOMPRESSED_BYTES} bytes); refusing to process"
|
|
796
|
+
)
|
|
797
|
+
|
|
798
|
+
|
|
799
|
+
def _read_zip_member(zf: zipfile.ZipFile, info: zipfile.ZipInfo, budget: list[int]) -> bytes:
|
|
800
|
+
"""Read one zip member, charging real decompressed bytes to the budget.
|
|
801
|
+
|
|
802
|
+
Streams the member in chunks so the cumulative cap is enforced on bytes
|
|
803
|
+
actually produced, not on the declared ``file_size``; raises
|
|
804
|
+
``ZipBudgetExceeded`` the moment the cap is crossed, before the whole
|
|
805
|
+
member is buffered.
|
|
806
|
+
"""
|
|
807
|
+
_check_zip_budget(info, budget)
|
|
808
|
+
with zf.open(info) as stream:
|
|
809
|
+
chunks: list[bytes] = []
|
|
810
|
+
while True:
|
|
811
|
+
chunk = stream.read(1 << 16)
|
|
812
|
+
if not chunk:
|
|
813
|
+
break
|
|
814
|
+
budget[0] += len(chunk)
|
|
815
|
+
if budget[0] > MAX_ZIP_DECOMPRESSED_BYTES:
|
|
816
|
+
raise ZipBudgetExceeded(
|
|
817
|
+
"zip decompressed size exceeds cap "
|
|
818
|
+
f"({MAX_ZIP_DECOMPRESSED_BYTES} bytes); refusing to process"
|
|
819
|
+
)
|
|
820
|
+
chunks.append(chunk)
|
|
821
|
+
return b"".join(chunks)
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def _is_docx_meta_part(name: str) -> bool:
|
|
825
|
+
"""Return True for DOCX/XLSX/PPTX parts that carry provenance, not visible content."""
|
|
826
|
+
return name.startswith(("docProps/", "customXml/"))
|
|
827
|
+
|
|
828
|
+
|
|
829
|
+
def _inspect_ooxml_zip(data: bytes, fmt: str) -> tuple[bool, bool, list[str], dict]:
|
|
830
|
+
findings: list[str] = []
|
|
831
|
+
has_c2pa = False
|
|
832
|
+
has_ai = False
|
|
833
|
+
parts: list[str] = []
|
|
834
|
+
budget = [0]
|
|
835
|
+
try:
|
|
836
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
|
837
|
+
parts = zf.namelist()
|
|
838
|
+
for info in zf.infolist():
|
|
839
|
+
_check_zip_budget(info, budget)
|
|
840
|
+
name = info.filename
|
|
841
|
+
# Check media parts for C2PA/AI metadata
|
|
842
|
+
if re.search(
|
|
843
|
+
r"^(?:word|xl|ppt)/media/.+\.(png|jpe?g|webp|avif|heic|gif|bmp|tiff?|svg)$",
|
|
844
|
+
name,
|
|
845
|
+
re.I,
|
|
846
|
+
):
|
|
847
|
+
raw = _read_zip_member(zf, info, budget)
|
|
848
|
+
img_fmt = detect_image_format(raw)
|
|
849
|
+
sub_c2pa, sub_ai, sub_findings = False, False, []
|
|
850
|
+
if img_fmt == "png":
|
|
851
|
+
sub_c2pa, sub_ai, sub_findings = inspect_png(raw)
|
|
852
|
+
elif img_fmt == "jpeg":
|
|
853
|
+
sub_c2pa, sub_ai, sub_findings = inspect_jpeg(raw)
|
|
854
|
+
elif img_fmt == "webp":
|
|
855
|
+
sub_c2pa, sub_ai, sub_findings = inspect_webp(raw)
|
|
856
|
+
elif img_fmt in ("avif", "heic"):
|
|
857
|
+
sub_c2pa, sub_ai, sub_findings = inspect_isobmff(raw, img_fmt)
|
|
858
|
+
elif img_fmt == "gif":
|
|
859
|
+
sub_c2pa, sub_ai, sub_findings = inspect_gif(raw)
|
|
860
|
+
elif img_fmt == "tiff":
|
|
861
|
+
sub_c2pa, sub_ai, sub_findings = inspect_tiff(raw)
|
|
862
|
+
elif img_fmt == "bmp":
|
|
863
|
+
sub_c2pa, sub_ai, sub_findings = inspect_bmp(raw)
|
|
864
|
+
elif name.lower().endswith(".svg") or raw.lstrip().startswith(b"<"):
|
|
865
|
+
sub_c2pa, sub_ai, sub_findings, _ = inspect_svg(raw)
|
|
866
|
+
if sub_c2pa:
|
|
867
|
+
has_c2pa = True
|
|
868
|
+
if sub_ai or sub_c2pa:
|
|
869
|
+
has_ai = True
|
|
870
|
+
for sf in sub_findings:
|
|
871
|
+
findings.append(f"{name}: {sf}")
|
|
872
|
+
continue
|
|
873
|
+
|
|
874
|
+
# Only metadata/provenance parts carry AI markers. The visible
|
|
875
|
+
# body (word/*.xml, xl/*.xml, ppt/*.xml) may legitimately mention
|
|
876
|
+
# vendor names such as "Claude" without being AI-generated metadata.
|
|
877
|
+
if not _is_docx_meta_part(name):
|
|
878
|
+
continue
|
|
879
|
+
raw = _read_zip_member(zf, info, budget)
|
|
880
|
+
c2, ai, hits = _blob_hits(raw)
|
|
881
|
+
if c2 or ai:
|
|
882
|
+
if c2:
|
|
883
|
+
has_c2pa = True
|
|
884
|
+
if ai:
|
|
885
|
+
has_ai = True
|
|
886
|
+
findings.append(f"{name}: {', '.join(hits[:6])}")
|
|
887
|
+
# always flag customXml presence lightly
|
|
888
|
+
custom = [n for n in parts if n.startswith("customXml/")]
|
|
889
|
+
if custom:
|
|
890
|
+
findings.append(f"customXml parts: {len(custom)}")
|
|
891
|
+
except _ZIP_PARSE_ERRORS as exc:
|
|
892
|
+
# A member that failed to read must not discard the evidence already
|
|
893
|
+
# collected from earlier members, nor read as "opened and found
|
|
894
|
+
# clean": keep the flags/findings, append a partial-read note, and
|
|
895
|
+
# only a wholly-garbage container (nothing accumulated) keeps the old
|
|
896
|
+
# not-a-valid-zip shape (#164).
|
|
897
|
+
if findings:
|
|
898
|
+
findings.append(
|
|
899
|
+
f"partial read of {fmt.upper()} zip ({exc.__class__.__name__}); "
|
|
900
|
+
"evidence above survives, later members were not scanned"
|
|
901
|
+
)
|
|
902
|
+
return has_c2pa, has_ai or has_c2pa, findings, {"parts": len(parts)}
|
|
903
|
+
return False, False, [f"not a valid {fmt.upper()} zip"], {}
|
|
904
|
+
return has_c2pa, has_ai or has_c2pa, findings, {"parts": len(parts)}
|
|
905
|
+
|
|
906
|
+
|
|
907
|
+
def inspect_docx(data: bytes) -> tuple[bool, bool, list[str], dict]:
|
|
908
|
+
return _inspect_ooxml_zip(data, "docx")
|
|
909
|
+
|
|
910
|
+
|
|
911
|
+
def inspect_xlsx(data: bytes) -> tuple[bool, bool, list[str], dict]:
|
|
912
|
+
return _inspect_ooxml_zip(data, "xlsx")
|
|
913
|
+
|
|
914
|
+
|
|
915
|
+
def inspect_pptx(data: bytes) -> tuple[bool, bool, list[str], dict]:
|
|
916
|
+
return _inspect_ooxml_zip(data, "pptx")
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
_XML_CHAR_REF_RE = re.compile(r"&#(?:x([0-9A-Fa-f]+)|([0-9]+));|&(amp|lt|gt|quot|apos);")
|
|
920
|
+
_XML_NAMED_ENTITIES = {"amp": "&", "lt": "<", "gt": ">", "quot": '"', "apos": "'"}
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
def _decode_xml_entities(s: str) -> str:
|
|
924
|
+
"""Decode what a conforming XML parser would resolve: numeric character
|
|
925
|
+
references and the five predefined entities. A watermark carrier encoded as
|
|
926
|
+
``​`` otherwise reaches Layer A as nine ASCII characters and survives
|
|
927
|
+
cleaning, because the parser in Word/Writer decodes the entity afterwards
|
|
928
|
+
(#129). Anything a parser would leave literal stays literal here.
|
|
929
|
+
"""
|
|
930
|
+
|
|
931
|
+
def _sub(m: re.Match[str]) -> str:
|
|
932
|
+
hex_digits, decimal, named = m.group(1), m.group(2), m.group(3)
|
|
933
|
+
if named:
|
|
934
|
+
return _XML_NAMED_ENTITIES[named]
|
|
935
|
+
codepoint = int(hex_digits, 16) if hex_digits else int(decimal)
|
|
936
|
+
if codepoint == 0 or codepoint > 0x10FFFF or 0xD800 <= codepoint <= 0xDFFF:
|
|
937
|
+
return m.group(0) # not a character; leave for the real parser to reject
|
|
938
|
+
try:
|
|
939
|
+
return chr(codepoint)
|
|
940
|
+
except ValueError:
|
|
941
|
+
return m.group(0)
|
|
942
|
+
|
|
943
|
+
return _XML_CHAR_REF_RE.sub(_sub, s)
|
|
944
|
+
|
|
945
|
+
|
|
946
|
+
def _reencode_xml_text(s: str) -> str:
|
|
947
|
+
"""Escape text content the way a serializer would: ``&``, ``<`` and ``>``.
|
|
948
|
+
|
|
949
|
+
``>`` is legal literally in text content but re-escaping it is a semantic
|
|
950
|
+
no-op, and ``<``/``&`` must be escaped anyway — so a decode/clean/re-encode
|
|
951
|
+
round trip is stable for any well-formed input.
|
|
952
|
+
"""
|
|
953
|
+
return s.replace("&", "&").replace("<", "<").replace(">", ">")
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def _scrub_text_runs(
|
|
957
|
+
xml_text: str, open_re: re.Pattern[str], close_re: re.Pattern[str]
|
|
958
|
+
) -> tuple[str, int, int]:
|
|
959
|
+
"""Run Layer A over the text runs delimited by open_re/close_re.
|
|
960
|
+
|
|
961
|
+
XML character references are decoded first (a carrier written as
|
|
962
|
+
``​`` otherwise reaches Layer A as nine ASCII characters and
|
|
963
|
+
survives cleaning, only to be resurrected by the parser in Word/Writer —
|
|
964
|
+
#129) and the surviving text is re-encoded on the way out, so the round
|
|
965
|
+
trip is stable for any well-formed input.
|
|
966
|
+
|
|
967
|
+
Shared by the DOCX/XLSX/PPTX body scrubs. Linear scan (see
|
|
968
|
+
_iter_tag_blocks) - the previous "(<tag>)(.*?)(</tag>)" lazy pattern was
|
|
969
|
+
quadratic on a run of unclosed opening tags.
|
|
970
|
+
"""
|
|
971
|
+
from text_unicode import clean_text # local import to avoid cycles
|
|
972
|
+
|
|
973
|
+
removed = 0
|
|
974
|
+
replaced = 0
|
|
975
|
+
out = []
|
|
976
|
+
last = 0
|
|
977
|
+
for os_, oe, cs_, ce in _iter_tag_blocks(xml_text, open_re, close_re):
|
|
978
|
+
open_tag = xml_text[os_:oe]
|
|
979
|
+
inner = xml_text[oe:cs_]
|
|
980
|
+
close_tag = xml_text[cs_:ce]
|
|
981
|
+
new_inner, stats = clean_text(_decode_xml_entities(inner))
|
|
982
|
+
if not (stats["removed_count"] or stats["replaced_count"]):
|
|
983
|
+
continue
|
|
984
|
+
removed += stats["removed_count"]
|
|
985
|
+
replaced += stats["replaced_count"]
|
|
986
|
+
if (new_inner[:1].isspace() or new_inner[-1:].isspace()) and "xml:space" not in open_tag:
|
|
987
|
+
open_tag = open_tag[:-1] + ' xml:space="preserve">'
|
|
988
|
+
out.append(xml_text[last:os_])
|
|
989
|
+
out.append(open_tag + _reencode_xml_text(new_inner) + close_tag)
|
|
990
|
+
last = ce
|
|
991
|
+
out.append(xml_text[last:])
|
|
992
|
+
return "".join(out), removed, replaced
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def _scrub_docx_text(xml_text: str) -> tuple[str, int, int]:
|
|
996
|
+
"""Run Layer A over the ``<w:t>`` text runs of a DOCX part.
|
|
997
|
+
|
|
998
|
+
Only ``w:t`` nodes are touched: field codes (``w:instrText``), run/paragraph
|
|
999
|
+
properties and the surrounding XML are left byte-identical. If leading or
|
|
1000
|
+
trailing whitespace survives the clean, the node keeps
|
|
1001
|
+
``xml:space="preserve"`` so Word does not trim it.
|
|
1002
|
+
"""
|
|
1003
|
+
return _scrub_text_runs(xml_text, re.compile(r"<w:t\b[^>]*>"), re.compile(r"</w:t>"))
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
def _scrub_xlsx_text(xml_text: str) -> tuple[str, int, int]:
|
|
1007
|
+
"""Run Layer A over the ``<t>`` text elements of an XLSX part."""
|
|
1008
|
+
return _scrub_text_runs(xml_text, re.compile(r"<t\b[^>]*>"), re.compile(r"</t>"))
|
|
1009
|
+
|
|
1010
|
+
|
|
1011
|
+
def _scrub_pptx_text(xml_text: str) -> tuple[str, int, int]:
|
|
1012
|
+
"""Run Layer A over the ``<a:t>`` text elements of a PPTX part."""
|
|
1013
|
+
return _scrub_text_runs(xml_text, re.compile(r"<a:t\b[^>]*>"), re.compile(r"</a:t>"))
|
|
1014
|
+
|
|
1015
|
+
|
|
1016
|
+
def _scrub_odt_text(xml_text: str) -> tuple[str, int, int]:
|
|
1017
|
+
"""Run Layer A over ODF paragraph text (``text:p`` content, incl. spans).
|
|
1018
|
+
|
|
1019
|
+
``text:span``/``text:tab``/``text:s`` children live inside the paragraph,
|
|
1020
|
+
so cleaning the paragraph content covers the visible text. The markup
|
|
1021
|
+
itself is untouched: entities are decoded and re-encoded per text segment
|
|
1022
|
+
only — a whole-paragraph round trip would escape the nested markup.
|
|
1023
|
+
Linear scan (see _iter_tag_blocks).
|
|
1024
|
+
"""
|
|
1025
|
+
from text_unicode import clean_text # local import to avoid cycles
|
|
1026
|
+
|
|
1027
|
+
removed = 0
|
|
1028
|
+
replaced = 0
|
|
1029
|
+
out = []
|
|
1030
|
+
last = 0
|
|
1031
|
+
for os_, oe, cs_, ce in _iter_tag_blocks(
|
|
1032
|
+
xml_text, re.compile(r"<text:p\b[^>]*>"), re.compile(r"</text:p>")
|
|
1033
|
+
):
|
|
1034
|
+
open_tag = xml_text[os_:oe]
|
|
1035
|
+
inner = xml_text[oe:cs_]
|
|
1036
|
+
close_tag = xml_text[cs_:ce]
|
|
1037
|
+
parts = re.split(r"(<[^>]+>)", inner)
|
|
1038
|
+
new_parts: list[str] = []
|
|
1039
|
+
changed = False
|
|
1040
|
+
for segment in parts:
|
|
1041
|
+
if not segment or segment.startswith("<"):
|
|
1042
|
+
new_parts.append(segment)
|
|
1043
|
+
continue
|
|
1044
|
+
new_segment, stats = clean_text(_decode_xml_entities(segment))
|
|
1045
|
+
if stats["removed_count"] or stats["replaced_count"]:
|
|
1046
|
+
removed += stats["removed_count"]
|
|
1047
|
+
replaced += stats["replaced_count"]
|
|
1048
|
+
changed = True
|
|
1049
|
+
new_parts.append(_reencode_xml_text(new_segment))
|
|
1050
|
+
else:
|
|
1051
|
+
new_parts.append(segment)
|
|
1052
|
+
if not changed:
|
|
1053
|
+
continue
|
|
1054
|
+
new_para = "".join(new_parts)
|
|
1055
|
+
if (new_para[:1].isspace() or new_para[-1:].isspace()) and "xml:space" not in open_tag:
|
|
1056
|
+
open_tag = open_tag[:-1] + ' xml:space="preserve">'
|
|
1057
|
+
out.append(xml_text[last:os_])
|
|
1058
|
+
out.append(open_tag + new_para + close_tag)
|
|
1059
|
+
last = ce
|
|
1060
|
+
out.append(xml_text[last:])
|
|
1061
|
+
return "".join(out), removed, replaced
|
|
1062
|
+
|
|
1063
|
+
|
|
1064
|
+
def _prune_dangling_relationships(
|
|
1065
|
+
rels_name: str, raw: bytes, kept_names: set[str]
|
|
1066
|
+
) -> tuple[bytes, int]:
|
|
1067
|
+
"""Drop <Relationship> entries whose internal target part no longer exists.
|
|
1068
|
+
|
|
1069
|
+
Removing a part (e.g. a customXml tree) must also remove the relationships
|
|
1070
|
+
that point at it, or the package is malformed: python-docx refuses to open
|
|
1071
|
+
it and Word offers to repair it. External relationships (``TargetMode``)
|
|
1072
|
+
and the package root (``Target="/"``) are left alone. ``rels_name`` is the
|
|
1073
|
+
archive member like ``word/_rels/document.xml.rels``; ``kept_names`` is the
|
|
1074
|
+
set of archive members that survive cleaning.
|
|
1075
|
+
"""
|
|
1076
|
+
base = posixpath.dirname(posixpath.dirname(rels_name))
|
|
1077
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1078
|
+
dropped = [0]
|
|
1079
|
+
|
|
1080
|
+
def _target_attr(tag: str) -> str:
|
|
1081
|
+
# XML allows single- or double-quoted attribute values; matching only
|
|
1082
|
+
# double quotes made every single-quoted Relationship look like an
|
|
1083
|
+
# empty target, which resolved to the package base and got pruned —
|
|
1084
|
+
# corrupting packages from tools that emit single quotes (#130).
|
|
1085
|
+
m = re.search(r"\bTarget\s*=\s*([\"'])(.*?)\1", tag, re.I)
|
|
1086
|
+
return m.group(2) if m else ""
|
|
1087
|
+
|
|
1088
|
+
def _drop(m: re.Match[str]) -> str:
|
|
1089
|
+
tag = m.group(0)
|
|
1090
|
+
if re.search(r"\bTargetMode\s*=", tag, re.I):
|
|
1091
|
+
return tag # external (http / mailto / ...) — never pruned
|
|
1092
|
+
target = _target_attr(tag)
|
|
1093
|
+
if target.startswith("/"):
|
|
1094
|
+
resolved = posixpath.normpath(target.lstrip("/"))
|
|
1095
|
+
else:
|
|
1096
|
+
resolved = posixpath.normpath(posixpath.join(base, target))
|
|
1097
|
+
if resolved in ("", "."):
|
|
1098
|
+
return tag # points at the package root
|
|
1099
|
+
if resolved in kept_names:
|
|
1100
|
+
return tag
|
|
1101
|
+
dropped[0] += 1
|
|
1102
|
+
return ""
|
|
1103
|
+
|
|
1104
|
+
new = re.sub(r"<Relationship\b[^>]*/>", _drop, text, flags=re.I)
|
|
1105
|
+
return new.encode("utf-8"), dropped[0]
|
|
1106
|
+
|
|
1107
|
+
|
|
1108
|
+
def _prune_odt_manifest_entries(raw: bytes, dropped: set[str]) -> tuple[bytes, int]:
|
|
1109
|
+
"""Remove manifest file-entry elements pointing at dropped parts.
|
|
1110
|
+
|
|
1111
|
+
ODF packages list every member in META-INF/manifest.xml; dropping a part
|
|
1112
|
+
(e.g. one carrying AI/C2PA markers) without removing its entry makes
|
|
1113
|
+
readers flag the package as damaged. full-path is matched
|
|
1114
|
+
attribute-order-independently, mirroring _prune_dangling_relationships.
|
|
1115
|
+
The package-root entry (full-path="/") and entries for surviving parts
|
|
1116
|
+
are left untouched.
|
|
1117
|
+
"""
|
|
1118
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1119
|
+
removed = [0]
|
|
1120
|
+
|
|
1121
|
+
def _full_path(tag: str) -> str:
|
|
1122
|
+
# Same single/double-quote tolerance as _target_attr (#130).
|
|
1123
|
+
m = re.search(r"\bfull-path\s*=\s*([\"'])(.*?)\1", tag, re.I)
|
|
1124
|
+
return m.group(2) if m else ""
|
|
1125
|
+
|
|
1126
|
+
def _drop(m: re.Match[str]) -> str:
|
|
1127
|
+
tag = m.group(0)
|
|
1128
|
+
path = posixpath.normpath(_full_path(tag).lstrip("/"))
|
|
1129
|
+
if path in ("", "."):
|
|
1130
|
+
return tag # package root — never pruned
|
|
1131
|
+
if path in dropped:
|
|
1132
|
+
removed[0] += 1
|
|
1133
|
+
return ""
|
|
1134
|
+
return tag
|
|
1135
|
+
|
|
1136
|
+
new = re.sub(r"<manifest:file-entry\b[^>]*/>", _drop, text, flags=re.I)
|
|
1137
|
+
return new.encode("utf-8"), removed[0]
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
def _prune_opf_manifest(raw: bytes, opf_name: str, dropped: set[str]) -> tuple[bytes, int]:
|
|
1141
|
+
"""Remove OPF <item> entries pointing at dropped parts (and spine refs).
|
|
1142
|
+
|
|
1143
|
+
The package document lists every content part; a dropped part that is
|
|
1144
|
+
still referenced makes EPUB readers reject the book. href values are
|
|
1145
|
+
relative to the package document directory, and <itemref> spine entries
|
|
1146
|
+
for removed items are pruned too so no idref dangles.
|
|
1147
|
+
"""
|
|
1148
|
+
base = posixpath.dirname(opf_name)
|
|
1149
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1150
|
+
removed = [0]
|
|
1151
|
+
removed_ids: set[str] = set()
|
|
1152
|
+
|
|
1153
|
+
def _attr(tag: str, name: str) -> str:
|
|
1154
|
+
m = re.search(rf'\b{name}\s*=\s*"([^"]*)"', tag, re.I)
|
|
1155
|
+
return m.group(1) if m else ""
|
|
1156
|
+
|
|
1157
|
+
def _drop_item(m: re.Match[str]) -> str:
|
|
1158
|
+
tag = m.group(0)
|
|
1159
|
+
href = _attr(tag, "href")
|
|
1160
|
+
if not href:
|
|
1161
|
+
return tag
|
|
1162
|
+
resolved = posixpath.normpath(posixpath.join(base, href))
|
|
1163
|
+
if resolved in dropped:
|
|
1164
|
+
removed[0] += 1
|
|
1165
|
+
item_id = _attr(tag, "id")
|
|
1166
|
+
if item_id:
|
|
1167
|
+
removed_ids.add(item_id)
|
|
1168
|
+
return ""
|
|
1169
|
+
return tag
|
|
1170
|
+
|
|
1171
|
+
new = re.sub(r"<item\b[^>]*/>", _drop_item, text, flags=re.I)
|
|
1172
|
+
|
|
1173
|
+
if removed_ids:
|
|
1174
|
+
|
|
1175
|
+
def _drop_itemref(m: re.Match[str]) -> str:
|
|
1176
|
+
tag = m.group(0)
|
|
1177
|
+
if _attr(tag, "idref") in removed_ids:
|
|
1178
|
+
removed[0] += 1
|
|
1179
|
+
return ""
|
|
1180
|
+
return tag
|
|
1181
|
+
|
|
1182
|
+
new = re.sub(r"<itemref\b[^>]*/>", _drop_itemref, new, flags=re.I)
|
|
1183
|
+
return new.encode("utf-8"), removed[0]
|
|
1184
|
+
|
|
1185
|
+
|
|
1186
|
+
def _scrub_ooxml_zip(
|
|
1187
|
+
data: bytes, fmt: str, *, also_layer_a_text: bool = True
|
|
1188
|
+
) -> tuple[bytes, list[str]]:
|
|
1189
|
+
actions: list[str] = []
|
|
1190
|
+
budget = [0]
|
|
1191
|
+
layer_removed = 0
|
|
1192
|
+
layer_replaced = 0
|
|
1193
|
+
kept: list[tuple[zipfile.ZipInfo, bytes]] = []
|
|
1194
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zin:
|
|
1195
|
+
for info in zin.infolist():
|
|
1196
|
+
_check_zip_budget(info, budget)
|
|
1197
|
+
name = info.filename
|
|
1198
|
+
raw = _read_zip_member(zin, info, budget)
|
|
1199
|
+
|
|
1200
|
+
# 1. Clean embedded media (PNG, JPEG, WebP, AVIF, HEIC, GIF, BMP, TIFF, SVG)
|
|
1201
|
+
if re.search(
|
|
1202
|
+
r"^(?:word|xl|ppt)/media/.+\.(png|jpe?g|webp|avif|heic|gif|bmp|tiff?|svg)$",
|
|
1203
|
+
name,
|
|
1204
|
+
re.I,
|
|
1205
|
+
):
|
|
1206
|
+
img_fmt = detect_image_format(raw)
|
|
1207
|
+
sub_actions: list[str] = []
|
|
1208
|
+
cleaned_bytes = raw
|
|
1209
|
+
try:
|
|
1210
|
+
if img_fmt == "png":
|
|
1211
|
+
cleaned_bytes, sub_actions = strip_png(raw, strip_all_text=True)
|
|
1212
|
+
elif img_fmt == "jpeg":
|
|
1213
|
+
cleaned_bytes, sub_actions = strip_jpeg(raw, strip_all_app=True)
|
|
1214
|
+
elif img_fmt == "webp":
|
|
1215
|
+
cleaned_bytes, sub_actions = strip_webp(raw, strip_all_metadata=True)
|
|
1216
|
+
elif img_fmt in ("avif", "heic"):
|
|
1217
|
+
cleaned_bytes, sub_actions = strip_isobmff(
|
|
1218
|
+
raw, img_fmt, strip_all_metadata=True
|
|
1219
|
+
)
|
|
1220
|
+
elif img_fmt == "gif":
|
|
1221
|
+
cleaned_bytes, sub_actions = strip_gif(raw, strip_all_metadata=True)
|
|
1222
|
+
elif img_fmt == "bmp":
|
|
1223
|
+
cleaned_bytes, sub_actions = strip_bmp(raw, strip_all_metadata=True)
|
|
1224
|
+
elif img_fmt == "tiff":
|
|
1225
|
+
cleaned_bytes, sub_actions = strip_tiff(raw, strip_all_metadata=True)
|
|
1226
|
+
elif name.lower().endswith(".svg") or raw.lstrip().startswith(b"<"):
|
|
1227
|
+
cleaned_bytes, sub_actions = clean_svg(raw)
|
|
1228
|
+
except Exception: # noqa: S110 - malformed embedded media; keep original part
|
|
1229
|
+
pass
|
|
1230
|
+
if any("drop" in a.lower() for a in sub_actions) and cleaned_bytes != raw:
|
|
1231
|
+
actions.append(f"clean embedded media in {name} ({', '.join(sub_actions[:2])})")
|
|
1232
|
+
raw = cleaned_bytes
|
|
1233
|
+
kept.append((info, raw))
|
|
1234
|
+
continue
|
|
1235
|
+
|
|
1236
|
+
# 2. Drop customXml trees
|
|
1237
|
+
if name.startswith("customXml/"):
|
|
1238
|
+
actions.append(f"drop part {name}")
|
|
1239
|
+
continue
|
|
1240
|
+
|
|
1241
|
+
# 3. docProps/ provenance
|
|
1242
|
+
if name in DOCX_META_PARTS or name.startswith("docProps/"):
|
|
1243
|
+
if name.endswith("custom.xml"):
|
|
1244
|
+
actions.append(f"drop part {name}")
|
|
1245
|
+
continue
|
|
1246
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1247
|
+
new = text
|
|
1248
|
+
for tag, label in DOCX_SCRUB_FIELDS:
|
|
1249
|
+
open_re = re.compile(rf"<{tag}\b[^>]*>", re.I)
|
|
1250
|
+
close_re = re.compile(rf"</{tag}>", re.I)
|
|
1251
|
+
out = []
|
|
1252
|
+
last = 0
|
|
1253
|
+
n = 0
|
|
1254
|
+
for os_, oe, cs_, ce in _iter_tag_blocks(new, open_re, close_re):
|
|
1255
|
+
out.append(new[last:os_])
|
|
1256
|
+
out.append(new[os_:oe] + new[cs_:ce])
|
|
1257
|
+
last = ce
|
|
1258
|
+
if new[oe:cs_]:
|
|
1259
|
+
n += 1
|
|
1260
|
+
actions.append(f"scrub {name} field {label}")
|
|
1261
|
+
if n:
|
|
1262
|
+
out.append(new[last:])
|
|
1263
|
+
new = "".join(out)
|
|
1264
|
+
raw = new.encode("utf-8")
|
|
1265
|
+
|
|
1266
|
+
# 4. [Content_Types].xml overrides
|
|
1267
|
+
if name == "[Content_Types].xml":
|
|
1268
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1269
|
+
new, n = re.subn(
|
|
1270
|
+
r'<Override\b[^>]*PartName="/customXml/[^"]*"[^>]*/>',
|
|
1271
|
+
"",
|
|
1272
|
+
text,
|
|
1273
|
+
)
|
|
1274
|
+
if n:
|
|
1275
|
+
actions.append(f"drop Content_Types customXml overrides x{n}")
|
|
1276
|
+
raw = new.encode("utf-8")
|
|
1277
|
+
new, n = re.subn(
|
|
1278
|
+
r'<Override\b[^>]*PartName="/docProps/custom\.xml"[^>]*/>',
|
|
1279
|
+
"",
|
|
1280
|
+
raw.decode("utf-8", errors="replace"),
|
|
1281
|
+
)
|
|
1282
|
+
if n:
|
|
1283
|
+
actions.append(f"drop Content_Types custom.xml override x{n}")
|
|
1284
|
+
raw = new.encode("utf-8")
|
|
1285
|
+
|
|
1286
|
+
# 5. Layer A text runs
|
|
1287
|
+
if also_layer_a_text and name.endswith(".xml"):
|
|
1288
|
+
if fmt == "docx" and name.startswith("word/"):
|
|
1289
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1290
|
+
new, r, rp = _scrub_docx_text(text)
|
|
1291
|
+
if r or rp:
|
|
1292
|
+
layer_removed += r
|
|
1293
|
+
layer_replaced += rp
|
|
1294
|
+
raw = new.encode("utf-8")
|
|
1295
|
+
elif fmt == "xlsx" and name.startswith("xl/"):
|
|
1296
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1297
|
+
new, r, rp = _scrub_xlsx_text(text)
|
|
1298
|
+
if r or rp:
|
|
1299
|
+
layer_removed += r
|
|
1300
|
+
layer_replaced += rp
|
|
1301
|
+
raw = new.encode("utf-8")
|
|
1302
|
+
elif fmt == "pptx" and name.startswith("ppt/"):
|
|
1303
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1304
|
+
new, r, rp = _scrub_pptx_text(text)
|
|
1305
|
+
if r or rp:
|
|
1306
|
+
layer_removed += r
|
|
1307
|
+
layer_replaced += rp
|
|
1308
|
+
raw = new.encode("utf-8")
|
|
1309
|
+
|
|
1310
|
+
kept.append((info, raw))
|
|
1311
|
+
|
|
1312
|
+
kept_names = {info.filename for info, _ in kept}
|
|
1313
|
+
final: list[tuple[zipfile.ZipInfo, bytes]] = []
|
|
1314
|
+
for info, raw in kept:
|
|
1315
|
+
part_raw = raw
|
|
1316
|
+
if info.filename.endswith(".rels"):
|
|
1317
|
+
part_raw, n = _prune_dangling_relationships(info.filename, raw, kept_names)
|
|
1318
|
+
if n:
|
|
1319
|
+
actions.append(f"prune dangling relationships x{n} in {info.filename}")
|
|
1320
|
+
final.append((info, part_raw))
|
|
1321
|
+
|
|
1322
|
+
out_buf = io.BytesIO()
|
|
1323
|
+
with zipfile.ZipFile(out_buf, "w", compression=zipfile.ZIP_DEFLATED) as zout:
|
|
1324
|
+
for info, raw in final:
|
|
1325
|
+
zout.writestr(info, raw)
|
|
1326
|
+
if layer_removed or layer_replaced:
|
|
1327
|
+
actions.append(f"layer A text: removed={layer_removed} replaced={layer_replaced}")
|
|
1328
|
+
if not actions:
|
|
1329
|
+
actions.append(f"no {fmt.upper()} metadata parts removed")
|
|
1330
|
+
return out_buf.getvalue(), actions
|
|
1331
|
+
|
|
1332
|
+
|
|
1333
|
+
def clean_docx(data: bytes, *, also_layer_a_text: bool = True) -> tuple[bytes, list[str]]:
|
|
1334
|
+
return _scrub_ooxml_zip(data, "docx", also_layer_a_text=also_layer_a_text)
|
|
1335
|
+
|
|
1336
|
+
|
|
1337
|
+
def clean_xlsx(data: bytes, *, also_layer_a_text: bool = True) -> tuple[bytes, list[str]]:
|
|
1338
|
+
return _scrub_ooxml_zip(data, "xlsx", also_layer_a_text=also_layer_a_text)
|
|
1339
|
+
|
|
1340
|
+
|
|
1341
|
+
def clean_pptx(data: bytes, *, also_layer_a_text: bool = True) -> tuple[bytes, list[str]]:
|
|
1342
|
+
return _scrub_ooxml_zip(data, "pptx", also_layer_a_text=also_layer_a_text)
|
|
1343
|
+
|
|
1344
|
+
|
|
1345
|
+
def inspect_odt(data: bytes) -> tuple[bool, bool, list[str], dict]:
|
|
1346
|
+
findings: list[str] = []
|
|
1347
|
+
has_c2pa = False
|
|
1348
|
+
has_ai = False
|
|
1349
|
+
budget = [0]
|
|
1350
|
+
try:
|
|
1351
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
|
1352
|
+
for info in zf.infolist():
|
|
1353
|
+
_check_zip_budget(info, budget)
|
|
1354
|
+
raw = _read_zip_member(zf, info, budget)
|
|
1355
|
+
c2, ai, hits = _blob_hits(raw)
|
|
1356
|
+
if c2 or ai:
|
|
1357
|
+
if c2:
|
|
1358
|
+
has_c2pa = True
|
|
1359
|
+
if ai:
|
|
1360
|
+
has_ai = True
|
|
1361
|
+
findings.append(f"{info.filename}: {', '.join(hits[:6])}")
|
|
1362
|
+
if "meta.xml" in zf.namelist():
|
|
1363
|
+
meta = _read_zip_member(zf, zf.getinfo("meta.xml"), budget).decode(
|
|
1364
|
+
"utf-8", errors="replace"
|
|
1365
|
+
)
|
|
1366
|
+
if re.search(r"generator|claude|openai|anthropic|gemini", meta, re.I):
|
|
1367
|
+
has_ai = True
|
|
1368
|
+
findings.append("meta.xml generator-like fields")
|
|
1369
|
+
except _ZIP_PARSE_ERRORS as exc:
|
|
1370
|
+
if findings:
|
|
1371
|
+
findings.append(
|
|
1372
|
+
f"partial read of ODT zip ({exc.__class__.__name__}); "
|
|
1373
|
+
"evidence above survives, later members were not scanned"
|
|
1374
|
+
)
|
|
1375
|
+
return has_c2pa, has_ai or has_c2pa, findings, {}
|
|
1376
|
+
return False, False, ["not a valid ODT zip"], {}
|
|
1377
|
+
return has_c2pa, has_ai or has_c2pa, findings, {}
|
|
1378
|
+
|
|
1379
|
+
|
|
1380
|
+
def clean_odt(data: bytes, *, also_layer_a_text: bool = True) -> tuple[bytes, list[str]]:
|
|
1381
|
+
actions: list[str] = []
|
|
1382
|
+
budget = [0]
|
|
1383
|
+
layer_removed = 0
|
|
1384
|
+
layer_replaced = 0
|
|
1385
|
+
kept: list[tuple[zipfile.ZipInfo, bytes]] = []
|
|
1386
|
+
dropped: set[str] = set()
|
|
1387
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zin:
|
|
1388
|
+
for info in zin.infolist():
|
|
1389
|
+
_check_zip_budget(info, budget)
|
|
1390
|
+
name = info.filename
|
|
1391
|
+
raw = _read_zip_member(zin, info, budget)
|
|
1392
|
+
if name == "meta.xml":
|
|
1393
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1394
|
+
# Drop meta:generator blocks (linear scan - lazy .*? is
|
|
1395
|
+
# quadratic on unclosed tags, see _iter_tag_blocks)
|
|
1396
|
+
new, n = _drop_tag_blocks(text, _ODT_GENERATOR_OPEN_RE, _ODT_GENERATOR_CLOSE_RE)
|
|
1397
|
+
if n:
|
|
1398
|
+
actions.append("drop meta:generator")
|
|
1399
|
+
text = new
|
|
1400
|
+
|
|
1401
|
+
# scrub creator-like if AI (linear scan)
|
|
1402
|
+
def _is_ai_creator(block: str) -> bool:
|
|
1403
|
+
return bool(AI_META_NAME_RE.search(block))
|
|
1404
|
+
|
|
1405
|
+
new, n = _drop_blocks_if(
|
|
1406
|
+
text, _DC_CREATOR_OPEN_RE, _DC_CREATOR_CLOSE_RE, _is_ai_creator
|
|
1407
|
+
)
|
|
1408
|
+
if n:
|
|
1409
|
+
actions.extend(["scrub creator-like meta"] * n)
|
|
1410
|
+
text = new
|
|
1411
|
+
raw = text.encode("utf-8")
|
|
1412
|
+
else:
|
|
1413
|
+
c2, ai, _ = _blob_hits(raw)
|
|
1414
|
+
if (c2 or ai) and name not in (
|
|
1415
|
+
"content.xml",
|
|
1416
|
+
"styles.xml",
|
|
1417
|
+
"mimetype",
|
|
1418
|
+
"META-INF/manifest.xml",
|
|
1419
|
+
):
|
|
1420
|
+
actions.append(f"drop part {name} (AI/C2PA markers)")
|
|
1421
|
+
dropped.add(name)
|
|
1422
|
+
continue
|
|
1423
|
+
# Layer A over the visible paragraph text of the body part.
|
|
1424
|
+
if also_layer_a_text and name == "content.xml":
|
|
1425
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1426
|
+
new, r, rp = _scrub_odt_text(text)
|
|
1427
|
+
if r or rp:
|
|
1428
|
+
layer_removed += r
|
|
1429
|
+
layer_replaced += rp
|
|
1430
|
+
raw = new.encode("utf-8")
|
|
1431
|
+
kept.append((info, raw))
|
|
1432
|
+
|
|
1433
|
+
# Two-pass: rewrite META-INF/manifest.xml now that the dropped set is
|
|
1434
|
+
# known. The manifest precedes the dropped parts in the archive, so a
|
|
1435
|
+
# single pass cannot remove entries for parts dropped later in the loop.
|
|
1436
|
+
if dropped:
|
|
1437
|
+
rewritten: list[tuple[zipfile.ZipInfo, bytes]] = []
|
|
1438
|
+
for info, part_raw in kept:
|
|
1439
|
+
out_raw = part_raw
|
|
1440
|
+
if info.filename == "META-INF/manifest.xml":
|
|
1441
|
+
pruned, n = _prune_odt_manifest_entries(part_raw, dropped)
|
|
1442
|
+
if n:
|
|
1443
|
+
actions.append(f"drop manifest entries x{n}")
|
|
1444
|
+
out_raw = pruned
|
|
1445
|
+
rewritten.append((info, out_raw))
|
|
1446
|
+
kept = rewritten
|
|
1447
|
+
|
|
1448
|
+
out_buf = io.BytesIO()
|
|
1449
|
+
with zipfile.ZipFile(out_buf, "w", compression=zipfile.ZIP_DEFLATED) as zout:
|
|
1450
|
+
for info, raw in kept:
|
|
1451
|
+
zout.writestr(info, raw)
|
|
1452
|
+
if layer_removed or layer_replaced:
|
|
1453
|
+
actions.append(f"layer A text: removed={layer_removed} replaced={layer_replaced}")
|
|
1454
|
+
if not actions:
|
|
1455
|
+
actions.append("no ODT metadata removed")
|
|
1456
|
+
return out_buf.getvalue(), actions
|
|
1457
|
+
|
|
1458
|
+
|
|
1459
|
+
# ---------------------------------------------------------------------------
|
|
1460
|
+
# EPUB
|
|
1461
|
+
# ---------------------------------------------------------------------------
|
|
1462
|
+
|
|
1463
|
+
_EPUB_MEDIA_RE = re.compile(r"\.(png|jpe?g|webp|avif|heic|gif|bmp|tiff?|svg)$", re.I)
|
|
1464
|
+
|
|
1465
|
+
|
|
1466
|
+
def _epub_content_part(name: str) -> bool:
|
|
1467
|
+
"""True for parts that carry visible content or structure and must never be dropped.
|
|
1468
|
+
|
|
1469
|
+
XHTML/HTML, CSS, JS, navigation (ncx), the package document (opf), raster
|
|
1470
|
+
and vector media, fonts, the 'mimetype' part, relationship files, and the
|
|
1471
|
+
OCF control files are content or structure. Everything else (META-INF
|
|
1472
|
+
custom parts, stray XML at the archive root) is a candidate for dropping
|
|
1473
|
+
when it carries AI/C2PA markers.
|
|
1474
|
+
"""
|
|
1475
|
+
low = name.lower()
|
|
1476
|
+
return bool(
|
|
1477
|
+
low == "mimetype"
|
|
1478
|
+
or low.endswith(".rels")
|
|
1479
|
+
or low
|
|
1480
|
+
in (
|
|
1481
|
+
"meta-inf/container.xml",
|
|
1482
|
+
"meta-inf/encryption.xml",
|
|
1483
|
+
"meta-inf/signatures.xml",
|
|
1484
|
+
"meta-inf/rights.xml",
|
|
1485
|
+
)
|
|
1486
|
+
or re.search(
|
|
1487
|
+
r"\.(xhtml|html?|css|js|ncx|opf|svg|png|jpe?g|webp|avif|heic|gif|bmp|tiff?|ttf|otf|woff2?)$",
|
|
1488
|
+
low,
|
|
1489
|
+
)
|
|
1490
|
+
)
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
def _epub_encrypted_parts(data: bytes) -> set[str]:
|
|
1494
|
+
"""Return part names listed as encrypted in META-INF/encryption.xml (OCF)."""
|
|
1495
|
+
try:
|
|
1496
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
|
1497
|
+
if "META-INF/encryption.xml" not in zf.namelist():
|
|
1498
|
+
return set()
|
|
1499
|
+
budget: list[int] = [0]
|
|
1500
|
+
xml = _read_zip_member(zf, zf.getinfo("META-INF/encryption.xml"), budget).decode(
|
|
1501
|
+
"utf-8", errors="replace"
|
|
1502
|
+
)
|
|
1503
|
+
except (zipfile.BadZipFile, KeyError):
|
|
1504
|
+
return set()
|
|
1505
|
+
names: set[str] = set()
|
|
1506
|
+
for uri in re.findall(r'CipherReference\s+URI="([^"]+)"', xml, re.I):
|
|
1507
|
+
# The URI is relative to META-INF/ per OCF, but producers are not
|
|
1508
|
+
# consistent, so accept every plausible normalization.
|
|
1509
|
+
names.add(posixpath.normpath(posixpath.join("META-INF", uri)))
|
|
1510
|
+
names.add(posixpath.normpath(uri))
|
|
1511
|
+
if uri.startswith("../"):
|
|
1512
|
+
names.add(posixpath.normpath(uri[3:]))
|
|
1513
|
+
return names
|
|
1514
|
+
|
|
1515
|
+
|
|
1516
|
+
def inspect_epub(data: bytes) -> tuple[bool, bool, list[str], dict]:
|
|
1517
|
+
findings: list[str] = []
|
|
1518
|
+
has_c2pa = False
|
|
1519
|
+
has_ai = False
|
|
1520
|
+
budget = [0]
|
|
1521
|
+
encrypted = _epub_encrypted_parts(data)
|
|
1522
|
+
names: list[str] = []
|
|
1523
|
+
try:
|
|
1524
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
|
1525
|
+
names = zf.namelist()
|
|
1526
|
+
for info in zf.infolist():
|
|
1527
|
+
_check_zip_budget(info, budget)
|
|
1528
|
+
name = info.filename
|
|
1529
|
+
if name in encrypted:
|
|
1530
|
+
findings.append(f"{name}: encrypted content (skipped)")
|
|
1531
|
+
continue
|
|
1532
|
+
raw = _read_zip_member(zf, info, budget)
|
|
1533
|
+
if name.lower().endswith((".xhtml", ".html", ".htm")):
|
|
1534
|
+
text = raw.decode("utf-8", errors="surrogateescape")
|
|
1535
|
+
c2, ai, sub, _ = inspect_html(text)
|
|
1536
|
+
if c2:
|
|
1537
|
+
has_c2pa = True
|
|
1538
|
+
if ai:
|
|
1539
|
+
has_ai = True
|
|
1540
|
+
for f in sub:
|
|
1541
|
+
findings.append(f"{name}: {f}")
|
|
1542
|
+
continue
|
|
1543
|
+
if name.lower().endswith(".opf"):
|
|
1544
|
+
text = raw.decode("utf-8", errors="surrogateescape")
|
|
1545
|
+
if AI_META_NAME_RE.search(text):
|
|
1546
|
+
has_ai = True
|
|
1547
|
+
findings.append(f"{name}: AI-ish metadata in package document")
|
|
1548
|
+
c2, ai, hits = _blob_hits(raw)
|
|
1549
|
+
if c2 or ai:
|
|
1550
|
+
has_c2pa = has_c2pa or c2
|
|
1551
|
+
has_ai = has_ai or ai
|
|
1552
|
+
findings.append(f"{name}: {', '.join(hits[:6])}")
|
|
1553
|
+
continue
|
|
1554
|
+
c2, ai, hits = _blob_hits(raw)
|
|
1555
|
+
if c2 or ai:
|
|
1556
|
+
has_c2pa = has_c2pa or c2
|
|
1557
|
+
has_ai = has_ai or ai
|
|
1558
|
+
findings.append(f"{name}: {', '.join(hits[:6])}")
|
|
1559
|
+
except _ZIP_PARSE_ERRORS as exc:
|
|
1560
|
+
# EPUB previously caught only BadZipFile, so a zlib error propagated
|
|
1561
|
+
# (crashing the scan) while a CRC failure discarded everything —
|
|
1562
|
+
# neither is right. Same partial-read contract as the OOXML/ODT
|
|
1563
|
+
# inspectors (#164).
|
|
1564
|
+
if findings:
|
|
1565
|
+
findings.append(
|
|
1566
|
+
f"partial read of EPUB zip ({exc.__class__.__name__}); "
|
|
1567
|
+
"evidence above survives, later members were not scanned"
|
|
1568
|
+
)
|
|
1569
|
+
return has_c2pa, has_ai or has_c2pa, findings, {"parts": len(names)}
|
|
1570
|
+
return False, False, ["not a valid EPUB zip"], {}
|
|
1571
|
+
return has_c2pa, has_ai or has_c2pa, findings, {"parts": len(names)}
|
|
1572
|
+
|
|
1573
|
+
|
|
1574
|
+
# Open/close tag halves for the linear OPF scans in _scrub_epub_opf.
|
|
1575
|
+
_OPF_META_OPEN_RE = re.compile(r"<meta\b[^>]*>", re.I)
|
|
1576
|
+
_OPF_META_CLOSE_RE = re.compile(r"</meta\s*>", re.I)
|
|
1577
|
+
_DC_FIELDS_OPEN_RE = re.compile(
|
|
1578
|
+
r"<(dc:(?:creator|contributor|publisher|description|rights|source))\b[^>]*>",
|
|
1579
|
+
re.I,
|
|
1580
|
+
)
|
|
1581
|
+
_DC_FIELDS_CLOSE_RE = re.compile(
|
|
1582
|
+
r"</(dc:(?:creator|contributor|publisher|description|rights|source))\s*>",
|
|
1583
|
+
re.I,
|
|
1584
|
+
)
|
|
1585
|
+
|
|
1586
|
+
|
|
1587
|
+
def _scrub_epub_opf(text: str) -> tuple[str, list[str]]:
|
|
1588
|
+
"""Scrub AI-ish metadata from the EPUB package document (OPF)."""
|
|
1589
|
+
actions: list[str] = []
|
|
1590
|
+
|
|
1591
|
+
def _meta(m: re.Match[str]) -> str:
|
|
1592
|
+
tag = m.group(0)
|
|
1593
|
+
if AI_META_NAME_RE.search(tag):
|
|
1594
|
+
actions.append("drop OPF meta tag")
|
|
1595
|
+
return ""
|
|
1596
|
+
return tag
|
|
1597
|
+
|
|
1598
|
+
new = re.sub(r"<meta\b[^>]*/>", _meta, text, flags=re.I)
|
|
1599
|
+
|
|
1600
|
+
# Block-form <meta>...</meta> (linear scan; the lazy .*? form is
|
|
1601
|
+
# quadratic on unclosed opening tags, see _iter_tag_blocks)
|
|
1602
|
+
def _meta_block_is_ai(block: str) -> bool:
|
|
1603
|
+
if AI_META_NAME_RE.search(block):
|
|
1604
|
+
actions.append("drop OPF meta tag")
|
|
1605
|
+
return True
|
|
1606
|
+
return False
|
|
1607
|
+
|
|
1608
|
+
# The predicate itself records the per-block action, so the count is unused.
|
|
1609
|
+
new, _n = _drop_blocks_if(new, _OPF_META_OPEN_RE, _OPF_META_CLOSE_RE, _meta_block_is_ai)
|
|
1610
|
+
|
|
1611
|
+
# dc:... provenance fields. The original pattern paired each opening tag
|
|
1612
|
+
# with its backreferenced closing tag via a lazy .*? - quadratic on
|
|
1613
|
+
# unclosed openings. Pair opens with same-name closes directly: linear,
|
|
1614
|
+
# exact same match semantics (first same-name close at/after the open).
|
|
1615
|
+
out = []
|
|
1616
|
+
last = 0
|
|
1617
|
+
dc_closes: dict[str, list[re.Match[str]]] = {}
|
|
1618
|
+
for m in re.finditer(_DC_FIELDS_CLOSE_RE, new):
|
|
1619
|
+
dc_closes.setdefault(m.group(1), []).append(m)
|
|
1620
|
+
ptr: dict[str, int] = {name: 0 for name in dc_closes}
|
|
1621
|
+
for om in re.finditer(_DC_FIELDS_OPEN_RE, new):
|
|
1622
|
+
name = om.group(1)
|
|
1623
|
+
if om.start() < last:
|
|
1624
|
+
continue
|
|
1625
|
+
closes = dc_closes.get(name, ())
|
|
1626
|
+
i = ptr.get(name, 0)
|
|
1627
|
+
while i < len(closes) and closes[i].start() < om.end():
|
|
1628
|
+
i += 1
|
|
1629
|
+
ptr[name] = i
|
|
1630
|
+
if i >= len(closes):
|
|
1631
|
+
continue # no same-name close - block never matches (kept)
|
|
1632
|
+
cm = closes[i]
|
|
1633
|
+
block = new[om.start() : cm.end()]
|
|
1634
|
+
if AI_META_NAME_RE.search(block):
|
|
1635
|
+
out.append(new[last : om.start()])
|
|
1636
|
+
out.append(f"<{name}/>")
|
|
1637
|
+
last = cm.end()
|
|
1638
|
+
actions.append(f"scrub {name} (AI vendor name)")
|
|
1639
|
+
if last:
|
|
1640
|
+
out.append(new[last:])
|
|
1641
|
+
new = "".join(out)
|
|
1642
|
+
|
|
1643
|
+
if not actions:
|
|
1644
|
+
actions.append("no OPF metadata removed")
|
|
1645
|
+
return new, actions
|
|
1646
|
+
|
|
1647
|
+
|
|
1648
|
+
def clean_epub(data: bytes, *, also_layer_a_text: bool = True) -> tuple[bytes, list[str]]:
|
|
1649
|
+
"""Rewrite the EPUB: scrub OPF metadata, XHTML meta/JSON-LD, and Layer A.
|
|
1650
|
+
|
|
1651
|
+
Embedded raster/SVG media get their own metadata stripped; structural and
|
|
1652
|
+
OCF control parts (mimetype, container.xml, encryption.xml, relationships,
|
|
1653
|
+
fonts) are kept so the book stays readable. Parts listed in
|
|
1654
|
+
META-INF/encryption.xml are copied verbatim — their bytes are ciphertext,
|
|
1655
|
+
so any rewrite would corrupt them. Non-content parts carrying AI/C2PA
|
|
1656
|
+
markers are dropped, and the OPF manifest is pruned afterwards so no
|
|
1657
|
+
dropped part stays referenced.
|
|
1658
|
+
"""
|
|
1659
|
+
from text_unicode import clean_text # local import to avoid cycles
|
|
1660
|
+
|
|
1661
|
+
actions: list[str] = []
|
|
1662
|
+
budget = [0]
|
|
1663
|
+
layer_removed = 0
|
|
1664
|
+
layer_replaced = 0
|
|
1665
|
+
encrypted = _epub_encrypted_parts(data)
|
|
1666
|
+
kept: list[tuple[zipfile.ZipInfo, bytes]] = []
|
|
1667
|
+
dropped: set[str] = set()
|
|
1668
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zin:
|
|
1669
|
+
for info in zin.infolist():
|
|
1670
|
+
_check_zip_budget(info, budget)
|
|
1671
|
+
name = info.filename
|
|
1672
|
+
raw = _read_zip_member(zin, info, budget)
|
|
1673
|
+
low = name.lower()
|
|
1674
|
+
|
|
1675
|
+
# Encrypted parts are opaque ciphertext: pass through untouched.
|
|
1676
|
+
if name in encrypted:
|
|
1677
|
+
kept.append((info, raw))
|
|
1678
|
+
continue
|
|
1679
|
+
|
|
1680
|
+
# 1. Embedded raster / vector media: strip metadata
|
|
1681
|
+
if _EPUB_MEDIA_RE.search(low):
|
|
1682
|
+
img_fmt = detect_image_format(raw)
|
|
1683
|
+
sub_actions: list[str] = []
|
|
1684
|
+
cleaned = raw
|
|
1685
|
+
try:
|
|
1686
|
+
if img_fmt == "png":
|
|
1687
|
+
cleaned, sub_actions = strip_png(raw, strip_all_text=True)
|
|
1688
|
+
elif img_fmt == "jpeg":
|
|
1689
|
+
cleaned, sub_actions = strip_jpeg(raw, strip_all_app=True)
|
|
1690
|
+
elif img_fmt == "webp":
|
|
1691
|
+
cleaned, sub_actions = strip_webp(raw, strip_all_metadata=True)
|
|
1692
|
+
elif img_fmt in ("avif", "heic"):
|
|
1693
|
+
cleaned, sub_actions = strip_isobmff(raw, img_fmt, strip_all_metadata=True)
|
|
1694
|
+
elif img_fmt == "gif":
|
|
1695
|
+
cleaned, sub_actions = strip_gif(raw, strip_all_metadata=True)
|
|
1696
|
+
elif img_fmt == "bmp":
|
|
1697
|
+
cleaned, sub_actions = strip_bmp(raw, strip_all_metadata=True)
|
|
1698
|
+
elif img_fmt == "tiff":
|
|
1699
|
+
cleaned, sub_actions = strip_tiff(raw, strip_all_metadata=True)
|
|
1700
|
+
elif low.endswith(".svg") or raw.lstrip().startswith(b"<"):
|
|
1701
|
+
cleaned, sub_actions = clean_svg(raw)
|
|
1702
|
+
except Exception: # noqa: S110 - malformed embedded media; keep original
|
|
1703
|
+
pass
|
|
1704
|
+
if any("drop" in a.lower() for a in sub_actions) and cleaned != raw:
|
|
1705
|
+
actions.append(f"clean embedded media in {name} ({', '.join(sub_actions[:2])})")
|
|
1706
|
+
raw = cleaned
|
|
1707
|
+
kept.append((info, raw))
|
|
1708
|
+
continue
|
|
1709
|
+
|
|
1710
|
+
# 2. XHTML content: strip AI meta/JSON-LD, then Layer A
|
|
1711
|
+
if low.endswith((".xhtml", ".html", ".htm")):
|
|
1712
|
+
text = raw.decode("utf-8", errors="surrogateescape")
|
|
1713
|
+
text, sub_actions = clean_html(text)
|
|
1714
|
+
if sub_actions and sub_actions != ["no HTML AI meta removed"]:
|
|
1715
|
+
actions.append(f"{name}: {', '.join(sub_actions[:2])}")
|
|
1716
|
+
if also_layer_a_text:
|
|
1717
|
+
text2, stats = clean_text(text)
|
|
1718
|
+
if stats["removed_count"] or stats["replaced_count"]:
|
|
1719
|
+
layer_removed += stats["removed_count"]
|
|
1720
|
+
layer_replaced += stats["replaced_count"]
|
|
1721
|
+
text = text2
|
|
1722
|
+
raw = text.encode("utf-8", errors="surrogateescape")
|
|
1723
|
+
kept.append((info, raw))
|
|
1724
|
+
continue
|
|
1725
|
+
|
|
1726
|
+
# 3. Package document (OPF): scrub AI-ish metadata
|
|
1727
|
+
if low.endswith(".opf"):
|
|
1728
|
+
text = raw.decode("utf-8", errors="surrogateescape")
|
|
1729
|
+
new_text, sub_actions = _scrub_epub_opf(text)
|
|
1730
|
+
if sub_actions and sub_actions != ["no OPF metadata removed"]:
|
|
1731
|
+
actions.extend(f"{name}: {a}" for a in sub_actions)
|
|
1732
|
+
raw = new_text.encode("utf-8", errors="surrogateescape")
|
|
1733
|
+
kept.append((info, raw))
|
|
1734
|
+
continue
|
|
1735
|
+
|
|
1736
|
+
# 4. Other parts: drop non-content parts carrying AI/C2PA markers
|
|
1737
|
+
c2, ai, _hits = _blob_hits(raw)
|
|
1738
|
+
if (c2 or ai) and not _epub_content_part(name):
|
|
1739
|
+
actions.append(f"drop part {name} (AI/C2PA markers)")
|
|
1740
|
+
dropped.add(name)
|
|
1741
|
+
continue
|
|
1742
|
+
|
|
1743
|
+
# 5. mimetype must stay first and stored — it already is, since we
|
|
1744
|
+
# write in the original entry order; force stored compression.
|
|
1745
|
+
if low == "mimetype":
|
|
1746
|
+
info.compress_type = zipfile.ZIP_STORED
|
|
1747
|
+
kept.append((info, raw))
|
|
1748
|
+
|
|
1749
|
+
# Two-pass: prune the OPF manifest now that the dropped set is known.
|
|
1750
|
+
if dropped:
|
|
1751
|
+
rewritten: list[tuple[zipfile.ZipInfo, bytes]] = []
|
|
1752
|
+
for info, part_raw in kept:
|
|
1753
|
+
out_raw = part_raw
|
|
1754
|
+
if info.filename.lower().endswith(".opf"):
|
|
1755
|
+
pruned, n = _prune_opf_manifest(part_raw, info.filename, dropped)
|
|
1756
|
+
if n:
|
|
1757
|
+
actions.append(f"prune OPF manifest entries x{n}")
|
|
1758
|
+
out_raw = pruned
|
|
1759
|
+
rewritten.append((info, out_raw))
|
|
1760
|
+
kept = rewritten
|
|
1761
|
+
|
|
1762
|
+
out_buf = io.BytesIO()
|
|
1763
|
+
with zipfile.ZipFile(out_buf, "w", compression=zipfile.ZIP_DEFLATED) as zout:
|
|
1764
|
+
for info, raw in kept:
|
|
1765
|
+
zout.writestr(info, raw)
|
|
1766
|
+
|
|
1767
|
+
if layer_removed or layer_replaced:
|
|
1768
|
+
actions.append(f"layer A text: removed={layer_removed} replaced={layer_replaced}")
|
|
1769
|
+
if not actions:
|
|
1770
|
+
actions.append("no EPUB metadata removed")
|
|
1771
|
+
return out_buf.getvalue(), actions
|
|
1772
|
+
|
|
1773
|
+
|
|
1774
|
+
# ---------------------------------------------------------------------------
|
|
1775
|
+
# PDF
|
|
1776
|
+
# ---------------------------------------------------------------------------
|
|
1777
|
+
|
|
1778
|
+
# Open/close halves for the linear XMP packet scan below. The lazy
|
|
1779
|
+
# ".*?</\?xpacket end..." form was quadratic on a flood of "<?xpacket begin"
|
|
1780
|
+
# markers with no end marker.
|
|
1781
|
+
_XMP_PACKET_OPEN_RE = re.compile(rb"<\?xpacket begin", re.I)
|
|
1782
|
+
_XMP_PACKET_CLOSE_RE = re.compile(rb"<\?xpacket end[^?]*\?>", re.I)
|
|
1783
|
+
_PDF_STREAM_OPEN_RE = re.compile(rb"stream\r?\n")
|
|
1784
|
+
_PDF_STREAM_CLOSE_RE = re.compile(rb"endstream")
|
|
1785
|
+
|
|
1786
|
+
|
|
1787
|
+
def _pdf_structured_blob(data: bytes) -> bytes:
|
|
1788
|
+
"""Return PDF bytes with stream payloads removed, plus XMP packets.
|
|
1789
|
+
|
|
1790
|
+
Stream payloads are often compressed binary where an AI-marker byte
|
|
1791
|
+
sequence (e.g. "AIGC") can occur by chance. Scanning only dictionaries and
|
|
1792
|
+
XMP packets avoids treating those collisions as metadata findings.
|
|
1793
|
+
"""
|
|
1794
|
+
parts = []
|
|
1795
|
+
last = 0
|
|
1796
|
+
for os_, _oe, _cs, ce in _iter_tag_blocks(data, _PDF_STREAM_OPEN_RE, _PDF_STREAM_CLOSE_RE):
|
|
1797
|
+
parts.append(data[last:os_])
|
|
1798
|
+
parts.append(b"stream endstream")
|
|
1799
|
+
last = ce
|
|
1800
|
+
parts.append(data[last:])
|
|
1801
|
+
no_streams = b"".join(parts)
|
|
1802
|
+
xmp = b"\n".join(
|
|
1803
|
+
data[os_:ce]
|
|
1804
|
+
for os_, _oe, _cs, ce in _iter_tag_blocks(data, _XMP_PACKET_OPEN_RE, _XMP_PACKET_CLOSE_RE)
|
|
1805
|
+
)
|
|
1806
|
+
return no_streams + b"\n" + xmp
|
|
1807
|
+
|
|
1808
|
+
|
|
1809
|
+
def inspect_pdf(path: Path, data: bytes) -> tuple[bool, bool, list[str], dict]:
|
|
1810
|
+
findings: list[str] = []
|
|
1811
|
+
has_c2pa, has_ai, hits = _blob_hits(_pdf_structured_blob(data))
|
|
1812
|
+
findings.extend(f"pdf-structured:{h}" for h in hits)
|
|
1813
|
+
# XMP packet scan
|
|
1814
|
+
xmp_blob = b"\n".join(
|
|
1815
|
+
data[os_:ce]
|
|
1816
|
+
for os_, _oe, _cs, ce in _iter_tag_blocks(data, _XMP_PACKET_OPEN_RE, _XMP_PACKET_CLOSE_RE)
|
|
1817
|
+
)
|
|
1818
|
+
if xmp_blob:
|
|
1819
|
+
findings.append("XMP packet present")
|
|
1820
|
+
has_ai = has_ai or bool(
|
|
1821
|
+
re.search(
|
|
1822
|
+
rb"digitalSourceType|trainedAlgorithmicMedia|SoftwareAgent|c2pa",
|
|
1823
|
+
xmp_blob,
|
|
1824
|
+
re.I,
|
|
1825
|
+
)
|
|
1826
|
+
)
|
|
1827
|
+
tools = run_optional_tools(path)
|
|
1828
|
+
ct = tools.get("c2patool") or {}
|
|
1829
|
+
if ct.get("has_manifest"):
|
|
1830
|
+
has_c2pa = True
|
|
1831
|
+
findings.append("c2patool reports C2PA-related manifest")
|
|
1832
|
+
probe_note = c2patool_probe_note(tools)
|
|
1833
|
+
if probe_note:
|
|
1834
|
+
findings.append(probe_note)
|
|
1835
|
+
return has_c2pa, has_ai or has_c2pa, findings, {"tools": tools}
|
|
1836
|
+
|
|
1837
|
+
|
|
1838
|
+
def _blank_xmp_packets(data: bytes) -> tuple[bytes, int]:
|
|
1839
|
+
"""Overwrite XMP packets with spaces, leaving every byte offset intact.
|
|
1840
|
+
|
|
1841
|
+
Deleting the bytes shifts every xref offset and stream /Length that follows,
|
|
1842
|
+
which is how the stdlib path could hand back a PDF that no longer parses --
|
|
1843
|
+
and with `deep_images="never"` nothing downstream rebuilds it. Blanking
|
|
1844
|
+
leaves the file structurally identical: XMP packets are padded with
|
|
1845
|
+
whitespace by design, so a whitespace-only one is unremarkable.
|
|
1846
|
+
"""
|
|
1847
|
+
out = bytearray(data)
|
|
1848
|
+
count = 0
|
|
1849
|
+
for open_start, _oe, _cs, close_end in _iter_tag_blocks(
|
|
1850
|
+
data, _XMP_PACKET_OPEN_RE, _XMP_PACKET_CLOSE_RE
|
|
1851
|
+
):
|
|
1852
|
+
out[open_start:close_end] = b" " * (close_end - open_start)
|
|
1853
|
+
count += 1
|
|
1854
|
+
return bytes(out), count
|
|
1855
|
+
|
|
1856
|
+
|
|
1857
|
+
def _pdf_structural_rewrite(
|
|
1858
|
+
dest: Path, actions: list[str], deadline: "_Deadline | None" = None
|
|
1859
|
+
) -> bool:
|
|
1860
|
+
"""Rebuild a PDF so unreferenced objects are dropped.
|
|
1861
|
+
|
|
1862
|
+
exiftool's PDF edits are incremental, so freed metadata objects survive in
|
|
1863
|
+
the byte stream. qpdf re-serializes from the object graph, which is what
|
|
1864
|
+
actually removes them. No-op (with a warning) when qpdf is absent.
|
|
1865
|
+
"""
|
|
1866
|
+
qpdf = which("qpdf")
|
|
1867
|
+
if not qpdf:
|
|
1868
|
+
actions.append(
|
|
1869
|
+
"warning: exiftool PDF edits are incremental — the original metadata "
|
|
1870
|
+
"bytes remain recoverable; install qpdf for a structural rewrite"
|
|
1871
|
+
)
|
|
1872
|
+
return False
|
|
1873
|
+
|
|
1874
|
+
if deadline is not None and deadline.spent():
|
|
1875
|
+
actions.append(
|
|
1876
|
+
"qpdf structural rewrite skipped: clean budget exhausted; "
|
|
1877
|
+
"metadata bytes may remain recoverable"
|
|
1878
|
+
)
|
|
1879
|
+
return False
|
|
1880
|
+
|
|
1881
|
+
tmp = dest.with_name(dest.name + ".qpdf-tmp")
|
|
1882
|
+
try:
|
|
1883
|
+
r = subprocess.run(
|
|
1884
|
+
[qpdf, "--linearize", "--", safe_arg(str(dest)), safe_arg(str(tmp))],
|
|
1885
|
+
capture_output=True,
|
|
1886
|
+
text=True,
|
|
1887
|
+
timeout=(_QPDF_TIMEOUT if deadline is None else deadline.timeout(_QPDF_TIMEOUT)),
|
|
1888
|
+
check=False,
|
|
1889
|
+
preexec_fn=subprocess_preexec_fn,
|
|
1890
|
+
creationflags=subprocess_creationflags,
|
|
1891
|
+
)
|
|
1892
|
+
except Exception as e:
|
|
1893
|
+
tmp.unlink(missing_ok=True)
|
|
1894
|
+
actions.append(f"qpdf rewrite failed: {e}; metadata bytes may remain recoverable")
|
|
1895
|
+
return False
|
|
1896
|
+
|
|
1897
|
+
# qpdf exit codes: 0 = clean, 3 = succeeded with warnings (output written).
|
|
1898
|
+
if r.returncode in (0, 3) and tmp.is_file() and tmp.stat().st_size > 0:
|
|
1899
|
+
tmp.replace(dest)
|
|
1900
|
+
actions.append(f"qpdf --linearize structural rewrite (rc={r.returncode})")
|
|
1901
|
+
return True
|
|
1902
|
+
|
|
1903
|
+
tmp.unlink(missing_ok=True)
|
|
1904
|
+
actions.append(
|
|
1905
|
+
f"qpdf rewrite skipped (rc={r.returncode}); metadata bytes may remain recoverable"
|
|
1906
|
+
)
|
|
1907
|
+
return False
|
|
1908
|
+
|
|
1909
|
+
|
|
1910
|
+
# APPn segments a JPEG can carry inside a PDF image XObject. APP0 (JFIF) is
|
|
1911
|
+
# structural and APP2 (ICC) decides how the colours are interpreted, so neither
|
|
1912
|
+
# counts as metadata to strip; APP1 (EXIF/XMP), APP11 (JUMBF/C2PA) and APP13
|
|
1913
|
+
# (Photoshop resources) do.
|
|
1914
|
+
_JPEG_METADATA_MARKERS = frozenset({0xE1, 0xEB, 0xED})
|
|
1915
|
+
_JPEG_SCAN_END = frozenset({0xDA, 0xD9})
|
|
1916
|
+
|
|
1917
|
+
|
|
1918
|
+
def _iter_jpeg_segments(blob: bytes, start: int):
|
|
1919
|
+
"""Yield (marker, payload) for each header segment of one JPEG.
|
|
1920
|
+
|
|
1921
|
+
A marker may be preceded by any number of 0xFF fill bytes, so the run has to
|
|
1922
|
+
be consumed before reading it. Taking the first 0xFF as the marker instead
|
|
1923
|
+
shifts the length read by one byte, and the walk then either stops early or
|
|
1924
|
+
wanders -- either way a provenance segment further along goes unseen.
|
|
1925
|
+
"""
|
|
1926
|
+
i = start + 2
|
|
1927
|
+
limit = len(blob)
|
|
1928
|
+
while i + 3 < limit:
|
|
1929
|
+
if blob[i] != 0xFF:
|
|
1930
|
+
return
|
|
1931
|
+
while i + 1 < limit and blob[i + 1] == 0xFF:
|
|
1932
|
+
i += 1
|
|
1933
|
+
if i + 1 >= limit:
|
|
1934
|
+
return
|
|
1935
|
+
marker = blob[i + 1]
|
|
1936
|
+
if marker in _JPEG_SCAN_END:
|
|
1937
|
+
return
|
|
1938
|
+
if 0xD0 <= marker <= 0xD8:
|
|
1939
|
+
i += 2
|
|
1940
|
+
continue
|
|
1941
|
+
if i + 4 > limit:
|
|
1942
|
+
return
|
|
1943
|
+
length = int.from_bytes(blob[i + 2 : i + 4], "big")
|
|
1944
|
+
if length < 2:
|
|
1945
|
+
return
|
|
1946
|
+
yield marker, blob[i + 4 : i + 2 + length]
|
|
1947
|
+
i += 2 + length
|
|
1948
|
+
|
|
1949
|
+
|
|
1950
|
+
def _jpeg_carries_metadata(blob: bytes, start: int) -> bool:
|
|
1951
|
+
"""True when this JPEG holds a metadata APPn segment of any kind."""
|
|
1952
|
+
return any(marker in _JPEG_METADATA_MARKERS for marker, _ in _iter_jpeg_segments(blob, start))
|
|
1953
|
+
|
|
1954
|
+
|
|
1955
|
+
def _jpeg_carries_provenance(blob: bytes, start: int) -> bool:
|
|
1956
|
+
"""True when this JPEG holds JUMBF or a provenance XMP packet."""
|
|
1957
|
+
for marker, payload in _iter_jpeg_segments(blob, start):
|
|
1958
|
+
if marker in _JPEG_PROVENANCE_MARKERS:
|
|
1959
|
+
lowered = payload.lower()
|
|
1960
|
+
if any(sig in lowered for sig in _PROVENANCE_SIGNATURES):
|
|
1961
|
+
return True
|
|
1962
|
+
return False
|
|
1963
|
+
|
|
1964
|
+
|
|
1965
|
+
def _any_jpeg(data: bytes, predicate) -> bool:
|
|
1966
|
+
i = 0
|
|
1967
|
+
while True:
|
|
1968
|
+
start = data.find(b"\xff\xd8\xff", i)
|
|
1969
|
+
if start < 0:
|
|
1970
|
+
return False
|
|
1971
|
+
if predicate(data, start):
|
|
1972
|
+
return True
|
|
1973
|
+
i = start + 2
|
|
1974
|
+
|
|
1975
|
+
|
|
1976
|
+
# APP11 carries JUMBF, which is how C2PA travels; an APP1 XMP packet can name
|
|
1977
|
+
# the provenance chain instead. Ordinary EXIF is deliberately not in this set.
|
|
1978
|
+
_JPEG_PROVENANCE_MARKERS = frozenset({0xE1, 0xEB})
|
|
1979
|
+
_PROVENANCE_SIGNATURES = (b"jumbf", b"c2pa", b"contentauth", b"dcterms:provenance")
|
|
1980
|
+
|
|
1981
|
+
|
|
1982
|
+
def embedded_provenance_present(data: bytes) -> bool:
|
|
1983
|
+
"""True when a JPEG inside *data* carries C2PA/JUMBF or provenance XMP.
|
|
1984
|
+
|
|
1985
|
+
inspect_pdf's byte scan skips stream payloads, so a manifest that lives only
|
|
1986
|
+
inside an image XObject is invisible to it unless c2patool happens to be
|
|
1987
|
+
installed. `auto` decides whether to re-distill from exactly that scan, so
|
|
1988
|
+
without this a machine without c2patool would quietly leave embedded C2PA in
|
|
1989
|
+
place -- the case this whole pass exists for.
|
|
1990
|
+
|
|
1991
|
+
Ordinary EXIF is not provenance and must not appear here: `auto` promises
|
|
1992
|
+
not to spend a re-distill on camera and editor traces.
|
|
1993
|
+
"""
|
|
1994
|
+
return _any_jpeg(data, _jpeg_carries_provenance)
|
|
1995
|
+
|
|
1996
|
+
|
|
1997
|
+
def embedded_image_metadata_present(data: bytes) -> bool:
|
|
1998
|
+
"""True when a JPEG inside *data* still carries a metadata APPn segment.
|
|
1999
|
+
|
|
2000
|
+
Ghostscript's JPEG pass-through copies the image bytes verbatim, so EXIF
|
|
2001
|
+
that lives inside the stream survives a lossless deep pass. Detecting it
|
|
2002
|
+
needs no external tool: the segment headers are readable straight from the
|
|
2003
|
+
PDF's bytes.
|
|
2004
|
+
"""
|
|
2005
|
+
return _any_jpeg(data, _jpeg_carries_metadata)
|
|
2006
|
+
|
|
2007
|
+
|
|
2008
|
+
DEEP_IMAGE_MODES = frozenset({"auto", "always", "lossless", "never"})
|
|
2009
|
+
|
|
2010
|
+
# One PDF can run two Ghostscript passes and three exiftool/qpdf pairs. Left to
|
|
2011
|
+
# their own timeouts that is about nineteen minutes of wall clock, on a request
|
|
2012
|
+
# thread, and /clean/batch works through its files one after another. Budget the
|
|
2013
|
+
# whole clean instead and hand each child whatever is left of it.
|
|
2014
|
+
PDF_CLEAN_BUDGET_SECONDS = float(os.environ.get("WATERMARKS_PDF_CLEAN_BUDGET", "420"))
|
|
2015
|
+
_EXIFTOOL_TIMEOUT = 60.0
|
|
2016
|
+
_QPDF_TIMEOUT = 120.0
|
|
2017
|
+
_GHOSTSCRIPT_TIMEOUT = 300.0
|
|
2018
|
+
|
|
2019
|
+
|
|
2020
|
+
class _Deadline:
|
|
2021
|
+
"""What is left of a shared budget, clamped to each tool's own ceiling."""
|
|
2022
|
+
|
|
2023
|
+
def __init__(self, budget: float) -> None:
|
|
2024
|
+
self._end = time.monotonic() + budget
|
|
2025
|
+
|
|
2026
|
+
def remaining(self) -> float:
|
|
2027
|
+
return max(0.0, self._end - time.monotonic())
|
|
2028
|
+
|
|
2029
|
+
def spent(self) -> bool:
|
|
2030
|
+
return self.remaining() <= 0.0
|
|
2031
|
+
|
|
2032
|
+
def timeout(self, cap: float) -> float:
|
|
2033
|
+
# Never hand subprocess 0: it would raise instead of running.
|
|
2034
|
+
return max(1.0, min(cap, self.remaining()))
|
|
2035
|
+
|
|
2036
|
+
|
|
2037
|
+
_GHOSTSCRIPT_NAMES = ("gs", "gswin64c", "gswin32c")
|
|
2038
|
+
|
|
2039
|
+
|
|
2040
|
+
def which_ghostscript() -> str | None:
|
|
2041
|
+
"""Locate a Ghostscript binary (POSIX ``gs``, Windows ``gswin64c``)."""
|
|
2042
|
+
for name in _GHOSTSCRIPT_NAMES:
|
|
2043
|
+
found = which(name)
|
|
2044
|
+
if found:
|
|
2045
|
+
return found
|
|
2046
|
+
return None
|
|
2047
|
+
|
|
2048
|
+
|
|
2049
|
+
def _exiftool_strip(
|
|
2050
|
+
exiftool: str, dest: Path, actions: list[str], deadline: "_Deadline | None" = None
|
|
2051
|
+
) -> None:
|
|
2052
|
+
"""Run ``exiftool -all=`` over *dest* in place, recording the outcome."""
|
|
2053
|
+
if deadline is not None and deadline.spent():
|
|
2054
|
+
actions.append("exiftool skipped: clean budget exhausted")
|
|
2055
|
+
return
|
|
2056
|
+
try:
|
|
2057
|
+
r = subprocess.run(
|
|
2058
|
+
[exiftool, "-all=", "-overwrite_original", safe_arg(str(dest))],
|
|
2059
|
+
capture_output=True,
|
|
2060
|
+
text=True,
|
|
2061
|
+
timeout=(
|
|
2062
|
+
_EXIFTOOL_TIMEOUT if deadline is None else deadline.timeout(_EXIFTOOL_TIMEOUT)
|
|
2063
|
+
),
|
|
2064
|
+
check=False,
|
|
2065
|
+
preexec_fn=subprocess_preexec_fn,
|
|
2066
|
+
creationflags=subprocess_creationflags,
|
|
2067
|
+
)
|
|
2068
|
+
actions.append(f"exiftool -all= (rc={r.returncode})")
|
|
2069
|
+
except Exception as e:
|
|
2070
|
+
actions.append(f"exiftool failed: {e}")
|
|
2071
|
+
|
|
2072
|
+
|
|
2073
|
+
def _pdf_deep_image_clean(
|
|
2074
|
+
dest: Path,
|
|
2075
|
+
actions: list[str],
|
|
2076
|
+
*,
|
|
2077
|
+
reencode: bool = False,
|
|
2078
|
+
deadline: "_Deadline | None" = None,
|
|
2079
|
+
) -> bool:
|
|
2080
|
+
"""Drop metadata carried *inside* embedded images by re-distilling.
|
|
2081
|
+
|
|
2082
|
+
``exiftool -all=`` and the qpdf rewrite both stop at document level. EXIF,
|
|
2083
|
+
Photoshop resource blocks and C2PA manifests that live in an image XObject
|
|
2084
|
+
survive them untouched. Ghostscript's pdfwrite rebuilds the page from the
|
|
2085
|
+
object graph, and with JPEG pass-through the compressed image data is
|
|
2086
|
+
copied byte-for-byte -- pixels are preserved while the metadata wrapped
|
|
2087
|
+
around them is dropped. No-op (with a warning) when Ghostscript is absent.
|
|
2088
|
+
|
|
2089
|
+
Pass-through has one blind spot: a marker living *inside* the JPEG itself
|
|
2090
|
+
(EXIF in APP1, a C2PA/JUMBF manifest in APP11) travels with the bytes it is
|
|
2091
|
+
attached to. ``reencode=True`` turns pass-through off so the image is
|
|
2092
|
+
recompressed and those segments are dropped -- effective, but lossy, so
|
|
2093
|
+
callers escalate to it only when a marker provably survived the cheap pass.
|
|
2094
|
+
"""
|
|
2095
|
+
if deadline is not None and deadline.spent():
|
|
2096
|
+
actions.append("deep image pass skipped: clean budget exhausted")
|
|
2097
|
+
return False
|
|
2098
|
+
|
|
2099
|
+
gs = which_ghostscript()
|
|
2100
|
+
if not gs:
|
|
2101
|
+
actions.append(
|
|
2102
|
+
"warning: metadata inside embedded images left in place; "
|
|
2103
|
+
"install ghostscript for the deep image pass"
|
|
2104
|
+
)
|
|
2105
|
+
return False
|
|
2106
|
+
|
|
2107
|
+
# A predictable path beside *dest* is a symlink-swap target: Ghostscript
|
|
2108
|
+
# opens it directly, which would sidestep the no-symlink contract that
|
|
2109
|
+
# safe_write_bytes upholds. Write into a private directory instead and
|
|
2110
|
+
# hand the validated bytes over through the safe writer.
|
|
2111
|
+
with tempfile.TemporaryDirectory(prefix="wr-gs-") as staging:
|
|
2112
|
+
tmp = Path(staging) / "deep-image.pdf"
|
|
2113
|
+
return _run_ghostscript(gs, dest, tmp, actions, reencode=reencode, deadline=deadline)
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
def _run_ghostscript(
|
|
2117
|
+
gs: str,
|
|
2118
|
+
dest: Path,
|
|
2119
|
+
tmp: Path,
|
|
2120
|
+
actions: list[str],
|
|
2121
|
+
*,
|
|
2122
|
+
reencode: bool,
|
|
2123
|
+
deadline: "_Deadline | None" = None,
|
|
2124
|
+
) -> bool:
|
|
2125
|
+
try:
|
|
2126
|
+
r = subprocess.run(
|
|
2127
|
+
[
|
|
2128
|
+
gs,
|
|
2129
|
+
"-dBATCH",
|
|
2130
|
+
"-dNOPAUSE",
|
|
2131
|
+
"-dQUIET",
|
|
2132
|
+
"-dSAFER",
|
|
2133
|
+
"-sDEVICE=pdfwrite",
|
|
2134
|
+
"-dPDFSETTINGS=/prepress",
|
|
2135
|
+
"-dAutoRotatePages=/None",
|
|
2136
|
+
f"-dPassThroughJPEGImages={'false' if reencode else 'true'}",
|
|
2137
|
+
# JPEG2000 streams get the same treatment; other codecs (Flate,
|
|
2138
|
+
# CCITT) are re-encoded by pdfwrite and pass-through does not
|
|
2139
|
+
# apply to them, which the docs say plainly.
|
|
2140
|
+
f"-dPassThroughJPXImages={'false' if reencode else 'true'}",
|
|
2141
|
+
"-dDownsampleColorImages=false",
|
|
2142
|
+
"-dDownsampleGrayImages=false",
|
|
2143
|
+
"-dDownsampleMonoImages=false",
|
|
2144
|
+
f"-sOutputFile={tmp}",
|
|
2145
|
+
safe_arg(str(dest)),
|
|
2146
|
+
],
|
|
2147
|
+
capture_output=True,
|
|
2148
|
+
text=True,
|
|
2149
|
+
timeout=(
|
|
2150
|
+
_GHOSTSCRIPT_TIMEOUT if deadline is None else deadline.timeout(_GHOSTSCRIPT_TIMEOUT)
|
|
2151
|
+
),
|
|
2152
|
+
check=False,
|
|
2153
|
+
preexec_fn=subprocess_preexec_fn,
|
|
2154
|
+
creationflags=subprocess_creationflags,
|
|
2155
|
+
)
|
|
2156
|
+
except Exception as e:
|
|
2157
|
+
actions.append(f"ghostscript deep image pass failed: {e}")
|
|
2158
|
+
return False
|
|
2159
|
+
|
|
2160
|
+
if r.returncode != 0 or not tmp.is_file() or tmp.stat().st_size == 0:
|
|
2161
|
+
actions.append(
|
|
2162
|
+
f"ghostscript deep image pass skipped (rc={r.returncode}); "
|
|
2163
|
+
"metadata inside embedded images may remain"
|
|
2164
|
+
)
|
|
2165
|
+
return False
|
|
2166
|
+
|
|
2167
|
+
safe_write_bytes(dest, tmp.read_bytes())
|
|
2168
|
+
actions.append(
|
|
2169
|
+
"ghostscript pdfwrite deep image pass, images re-encoded (rc=0)"
|
|
2170
|
+
if reencode
|
|
2171
|
+
else "ghostscript pdfwrite deep image pass, images passed through (rc=0)"
|
|
2172
|
+
)
|
|
2173
|
+
return True
|
|
2174
|
+
|
|
2175
|
+
|
|
2176
|
+
def clean_pdf(path: Path, dest: Path, *, deep_images: str = "auto") -> tuple[list[str], dict]:
|
|
2177
|
+
"""Best-effort PDF clean. Prefers exiftool; falls back to XMP strip warning.
|
|
2178
|
+
|
|
2179
|
+
``deep_images`` controls the Ghostscript re-distill that reaches metadata
|
|
2180
|
+
inside embedded images:
|
|
2181
|
+
|
|
2182
|
+
* ``"auto"`` (default) -- run it only when the document-level strip left
|
|
2183
|
+
AI/C2PA markers behind, and recompress the images if a marker survives
|
|
2184
|
+
even that.
|
|
2185
|
+
* ``"always"`` -- run it for every PDF, which also clears non-AI traces
|
|
2186
|
+
such as camera and editor EXIF, with the same escalation.
|
|
2187
|
+
* ``"lossless"`` -- deep pass without the recompressing escalation, so
|
|
2188
|
+
image data is never touched.
|
|
2189
|
+
* ``"never"`` -- skip it.
|
|
2190
|
+
"""
|
|
2191
|
+
actions: list[str] = []
|
|
2192
|
+
data = path.read_bytes()
|
|
2193
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
2194
|
+
|
|
2195
|
+
# Silently treating a typo as "auto" would turn a request for lossless
|
|
2196
|
+
# cleaning into one that may recompress, so refuse it instead.
|
|
2197
|
+
if deep_images not in DEEP_IMAGE_MODES:
|
|
2198
|
+
raise ValueError(
|
|
2199
|
+
f"deep_images must be one of {sorted(DEEP_IMAGE_MODES)}, got {deep_images!r}"
|
|
2200
|
+
)
|
|
2201
|
+
|
|
2202
|
+
deadline = _Deadline(PDF_CLEAN_BUDGET_SECONDS)
|
|
2203
|
+
|
|
2204
|
+
exiftool = which("exiftool")
|
|
2205
|
+
rewritten = False
|
|
2206
|
+
|
|
2207
|
+
if exiftool:
|
|
2208
|
+
safe_write_bytes(dest, data)
|
|
2209
|
+
_exiftool_strip(exiftool, dest, actions, deadline)
|
|
2210
|
+
# exiftool writes PDFs *incrementally*: it appends a
|
|
2211
|
+
# %BeginExifToolUpdate block that frees the Info object and drops
|
|
2212
|
+
# /Info from the trailer, but the original metadata bytes stay in the
|
|
2213
|
+
# file verbatim and are trivially recoverable (exiftool itself can
|
|
2214
|
+
# revert them with -PDF-update:all=). A structural rewrite is what
|
|
2215
|
+
# actually drops the now-unreferenced objects.
|
|
2216
|
+
rewritten = _pdf_structural_rewrite(dest, actions, deadline)
|
|
2217
|
+
document_mode = "exiftool"
|
|
2218
|
+
else:
|
|
2219
|
+
# Degraded document-level strip: obvious XMP packets and nothing else.
|
|
2220
|
+
# The deep-image ladder below still runs -- Ghostscript reaches metadata
|
|
2221
|
+
# inside image XObjects on its own, and gating that on exiftool left the
|
|
2222
|
+
# only tool that can do that job unused.
|
|
2223
|
+
blanked, n = _blank_xmp_packets(data)
|
|
2224
|
+
safe_write_bytes(dest, blanked if n else data)
|
|
2225
|
+
if n:
|
|
2226
|
+
actions.append(f"blanked XMP xpacket x{n} (degraded; byte offsets preserved)")
|
|
2227
|
+
actions.append("warning: pure-stdlib PDF strip is best-effort; prefer exiftool")
|
|
2228
|
+
document_mode = "stdlib-xmp"
|
|
2229
|
+
else:
|
|
2230
|
+
actions.append(
|
|
2231
|
+
"no PDF cleaner available (install exiftool for reliable metadata "
|
|
2232
|
+
"strip); document-level metadata left as-is"
|
|
2233
|
+
)
|
|
2234
|
+
document_mode = "copy"
|
|
2235
|
+
|
|
2236
|
+
# Document-level strip is done. Metadata inside embedded images is out
|
|
2237
|
+
# of reach from here, so decide whether to re-distill.
|
|
2238
|
+
mode = deep_images
|
|
2239
|
+
|
|
2240
|
+
def _markers_left() -> bool:
|
|
2241
|
+
current = dest.read_bytes()
|
|
2242
|
+
res_c2pa, res_ai, _f, _d = inspect_pdf(dest, current)
|
|
2243
|
+
# inspect_pdf excludes stream payloads, so ask the streams directly too:
|
|
2244
|
+
# without c2patool nothing else would notice a manifest that only exists
|
|
2245
|
+
# inside an image.
|
|
2246
|
+
return bool(res_c2pa or res_ai) or embedded_provenance_present(current)
|
|
2247
|
+
|
|
2248
|
+
def _settle(ran: bool) -> None:
|
|
2249
|
+
# pdfwrite stamps its own /Producer, and exiftool edits PDFs
|
|
2250
|
+
# incrementally, so re-serialize once more after removing it.
|
|
2251
|
+
nonlocal rewritten
|
|
2252
|
+
if not ran:
|
|
2253
|
+
return
|
|
2254
|
+
if exiftool:
|
|
2255
|
+
_exiftool_strip(exiftool, dest, actions, deadline)
|
|
2256
|
+
rewritten = _pdf_structural_rewrite(dest, actions, deadline) or rewritten
|
|
2257
|
+
else:
|
|
2258
|
+
# Trading a vendor mark for a Ghostscript one is still worth it,
|
|
2259
|
+
# but the caller should not have to discover the swap.
|
|
2260
|
+
actions.append(
|
|
2261
|
+
"warning: the re-distill stamped its own /Producer and there "
|
|
2262
|
+
"is no exiftool to remove it"
|
|
2263
|
+
)
|
|
2264
|
+
|
|
2265
|
+
deep_ran = False
|
|
2266
|
+
reencoded = False
|
|
2267
|
+
if mode == "never":
|
|
2268
|
+
pass
|
|
2269
|
+
elif mode in ("always", "lossless") or _markers_left():
|
|
2270
|
+
if mode == "auto":
|
|
2271
|
+
actions.append(
|
|
2272
|
+
"residual AI/C2PA markers after document-level strip "
|
|
2273
|
+
"(embedded image suspected); running deep image pass"
|
|
2274
|
+
)
|
|
2275
|
+
deep_ran = _pdf_deep_image_clean(dest, actions, deadline=deadline)
|
|
2276
|
+
_settle(deep_ran)
|
|
2277
|
+
# Anything inside the JPEG itself rides along with a passed-through
|
|
2278
|
+
# stream. Recompressing is the only way left to shift it, so spend
|
|
2279
|
+
# the quality only when something demonstrably survived: AI/C2PA
|
|
2280
|
+
# markers always count, and under `always` -- which promises to
|
|
2281
|
+
# clear camera and editor traces too -- so does ordinary EXIF.
|
|
2282
|
+
#
|
|
2283
|
+
# Two things make asking pointless. In `lossless` the answer is
|
|
2284
|
+
# discarded, and _markers_left() runs a full inspect_pdf that spawns
|
|
2285
|
+
# tools of its own. And if rung 1 never ran -- no Ghostscript, or the
|
|
2286
|
+
# budget is gone -- rung 2 cannot run either, and asking would only
|
|
2287
|
+
# append the same warning twice.
|
|
2288
|
+
survived = (
|
|
2289
|
+
mode != "lossless"
|
|
2290
|
+
and deep_ran
|
|
2291
|
+
and (
|
|
2292
|
+
_markers_left()
|
|
2293
|
+
or (mode == "always" and embedded_image_metadata_present(dest.read_bytes()))
|
|
2294
|
+
)
|
|
2295
|
+
)
|
|
2296
|
+
if survived:
|
|
2297
|
+
actions.append(
|
|
2298
|
+
"metadata survived the lossless pass (it is inside the image "
|
|
2299
|
+
"stream); escalating to a re-encoding pass"
|
|
2300
|
+
)
|
|
2301
|
+
reencoded = _pdf_deep_image_clean(dest, actions, reencode=True, deadline=deadline)
|
|
2302
|
+
deep_ran = deep_ran or reencoded
|
|
2303
|
+
_settle(reencoded)
|
|
2304
|
+
else:
|
|
2305
|
+
actions.append(
|
|
2306
|
+
"deep image pass not needed for AI/C2PA markers; pass "
|
|
2307
|
+
'deep_images="always" to also clear non-AI EXIF inside images'
|
|
2308
|
+
)
|
|
2309
|
+
|
|
2310
|
+
c2patool = which("c2patool")
|
|
2311
|
+
# c2patool does not always strip; leave note
|
|
2312
|
+
if c2patool:
|
|
2313
|
+
actions.append("c2patool available for inspect; strip via exiftool/re-export")
|
|
2314
|
+
|
|
2315
|
+
meta: dict[str, Any] = {
|
|
2316
|
+
"mode": document_mode,
|
|
2317
|
+
"structural_rewrite": rewritten,
|
|
2318
|
+
"deep_images": mode,
|
|
2319
|
+
"deep_image_pass": deep_ran,
|
|
2320
|
+
"images_reencoded": reencoded,
|
|
2321
|
+
}
|
|
2322
|
+
if not exiftool:
|
|
2323
|
+
# The deep pass may well have run, but the document-level strip was
|
|
2324
|
+
# the stdlib one, so the result is still best-effort.
|
|
2325
|
+
meta["degraded"] = True
|
|
2326
|
+
return actions, meta
|
|
2327
|
+
|
|
2328
|
+
|
|
2329
|
+
# ---------------------------------------------------------------------------
|
|
2330
|
+
# Unified API
|
|
2331
|
+
# ---------------------------------------------------------------------------
|
|
2332
|
+
|
|
2333
|
+
|
|
2334
|
+
def inspect_container(path: Path) -> ContainerInspectReport:
|
|
2335
|
+
data = path.read_bytes()
|
|
2336
|
+
fmt = detect_container_format(path, data)
|
|
2337
|
+
tools: dict[str, Any] = {}
|
|
2338
|
+
details: dict[str, Any] = {}
|
|
2339
|
+
|
|
2340
|
+
if fmt == "svg":
|
|
2341
|
+
has_c2pa, has_ai, findings, details = inspect_svg(data)
|
|
2342
|
+
elif fmt == "pdf":
|
|
2343
|
+
has_c2pa, has_ai, findings, details = inspect_pdf(path, data)
|
|
2344
|
+
tools = details.pop("tools", {})
|
|
2345
|
+
elif fmt == "docx":
|
|
2346
|
+
has_c2pa, has_ai, findings, details = inspect_docx(data)
|
|
2347
|
+
elif fmt == "xlsx":
|
|
2348
|
+
has_c2pa, has_ai, findings, details = inspect_xlsx(data)
|
|
2349
|
+
elif fmt == "pptx":
|
|
2350
|
+
has_c2pa, has_ai, findings, details = inspect_pptx(data)
|
|
2351
|
+
elif fmt == "odt":
|
|
2352
|
+
has_c2pa, has_ai, findings, details = inspect_odt(data)
|
|
2353
|
+
elif fmt == "epub":
|
|
2354
|
+
has_c2pa, has_ai, findings, details = inspect_epub(data)
|
|
2355
|
+
elif fmt == "html":
|
|
2356
|
+
# surrogateescape, not replace: clean_container() decodes the same way,
|
|
2357
|
+
# and U+FFFD substitutions would make the two disagree on the counts.
|
|
2358
|
+
body = data.decode("utf-8", errors="surrogateescape")
|
|
2359
|
+
has_c2pa, has_ai, findings, details = inspect_html(body)
|
|
2360
|
+
elif fmt == "markdown":
|
|
2361
|
+
body = data.decode("utf-8", errors="surrogateescape")
|
|
2362
|
+
has_c2pa, has_ai, findings, details = inspect_markdown(body)
|
|
2363
|
+
else:
|
|
2364
|
+
has_c2pa, has_ai, findings = False, False, [f"unsupported container: {fmt}"]
|
|
2365
|
+
details = {"unsupported": True}
|
|
2366
|
+
|
|
2367
|
+
# Layer A body scan for exactly the formats clean_container() scrubs, so
|
|
2368
|
+
# inspect predicts clean rather than contradicting it.
|
|
2369
|
+
layer_a_total = 0
|
|
2370
|
+
layer_a_hits: list[dict] = []
|
|
2371
|
+
if fmt in ("markdown", "html"):
|
|
2372
|
+
from text_unicode import inspect_text # local import to avoid cycles
|
|
2373
|
+
|
|
2374
|
+
ta = inspect_text(body).to_dict()
|
|
2375
|
+
layer_a_total = ta["suspicious_total"]
|
|
2376
|
+
layer_a_hits = ta["hits"]
|
|
2377
|
+
for h in layer_a_hits:
|
|
2378
|
+
findings.append(f"layer-a: {h['codepoint']} {h['label']} x{h['count']} ({h['kind']})")
|
|
2379
|
+
elif fmt == "epub":
|
|
2380
|
+
from text_unicode import inspect_text # local import to avoid cycles
|
|
2381
|
+
|
|
2382
|
+
encrypted = _epub_encrypted_parts(data)
|
|
2383
|
+
budget: list[int] = [0]
|
|
2384
|
+
try:
|
|
2385
|
+
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
|
2386
|
+
for info in zf.infolist():
|
|
2387
|
+
if info.filename in encrypted:
|
|
2388
|
+
continue
|
|
2389
|
+
if info.filename.lower().endswith((".xhtml", ".html", ".htm")):
|
|
2390
|
+
ta = inspect_text(
|
|
2391
|
+
_read_zip_member(zf, info, budget).decode(
|
|
2392
|
+
"utf-8", errors="surrogateescape"
|
|
2393
|
+
)
|
|
2394
|
+
).to_dict()
|
|
2395
|
+
layer_a_total += ta["suspicious_total"]
|
|
2396
|
+
for h in ta["hits"]:
|
|
2397
|
+
layer_a_hits.append(h)
|
|
2398
|
+
findings.append(
|
|
2399
|
+
f"layer-a ({info.filename}): {h['codepoint']} {h['label']} "
|
|
2400
|
+
f"x{h['count']} ({h['kind']})"
|
|
2401
|
+
)
|
|
2402
|
+
except zipfile.BadZipFile:
|
|
2403
|
+
pass
|
|
2404
|
+
|
|
2405
|
+
notes: list[str] = []
|
|
2406
|
+
if fmt == "pdf":
|
|
2407
|
+
notes.append(
|
|
2408
|
+
"PDF inspection is best-effort; exiftool/c2patool give more reliable metadata detection"
|
|
2409
|
+
)
|
|
2410
|
+
elif fmt in ("docx", "xlsx", "pptx"):
|
|
2411
|
+
notes.append(f"{fmt.upper()}: metadata/provenance and embedded media are scanned")
|
|
2412
|
+
elif fmt == "epub":
|
|
2413
|
+
notes.append(
|
|
2414
|
+
"EPUB: package-document metadata, XHTML meta/JSON-LD, and embedded media are scanned"
|
|
2415
|
+
)
|
|
2416
|
+
if "unsupported" in details:
|
|
2417
|
+
notes.append(f"format not fully inspected: {fmt}")
|
|
2418
|
+
if layer_a_total:
|
|
2419
|
+
notes.append(
|
|
2420
|
+
f"layer A: {layer_a_total} invisible/format codepoint(s) in body text; "
|
|
2421
|
+
"clean removes these"
|
|
2422
|
+
)
|
|
2423
|
+
|
|
2424
|
+
if fmt in ("svg", "pdf", "docx", "xlsx", "pptx") and not tools:
|
|
2425
|
+
tools = run_optional_tools(path)
|
|
2426
|
+
|
|
2427
|
+
return ContainerInspectReport(
|
|
2428
|
+
path=str(path),
|
|
2429
|
+
format=fmt,
|
|
2430
|
+
has_c2pa=has_c2pa,
|
|
2431
|
+
has_ai_metadata=has_ai,
|
|
2432
|
+
findings=findings,
|
|
2433
|
+
tools=tools,
|
|
2434
|
+
details=details,
|
|
2435
|
+
notes=notes,
|
|
2436
|
+
layer_a_total=layer_a_total,
|
|
2437
|
+
layer_a_hits=layer_a_hits,
|
|
2438
|
+
)
|
|
2439
|
+
|
|
2440
|
+
|
|
2441
|
+
def clean_container(
|
|
2442
|
+
path: Path,
|
|
2443
|
+
dest: Path,
|
|
2444
|
+
fmt: str | None = None,
|
|
2445
|
+
*,
|
|
2446
|
+
also_layer_a_text: bool = True,
|
|
2447
|
+
deep_images: str = "auto",
|
|
2448
|
+
) -> dict[str, Any]:
|
|
2449
|
+
"""Clean container metadata; optionally Layer-A scrub text bodies for md/html.
|
|
2450
|
+
|
|
2451
|
+
``fmt`` pins the container format when the caller already knows it. This
|
|
2452
|
+
matters for ``--in-place`` flows where *path* is a ``.bak`` copy whose
|
|
2453
|
+
suffix would otherwise make markdown/HTML (which have no magic bytes)
|
|
2454
|
+
classify as ``unknown``.
|
|
2455
|
+
"""
|
|
2456
|
+
from text_unicode import clean_text # local import to avoid cycles
|
|
2457
|
+
|
|
2458
|
+
data = path.read_bytes()
|
|
2459
|
+
fmt = fmt or detect_container_format(path, data)
|
|
2460
|
+
actions: list[str] = []
|
|
2461
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
2462
|
+
meta: dict[str, Any] = {"format": fmt}
|
|
2463
|
+
|
|
2464
|
+
if fmt == "svg":
|
|
2465
|
+
cleaned, actions = clean_svg(data)
|
|
2466
|
+
safe_write_bytes(dest, cleaned)
|
|
2467
|
+
elif fmt == "pdf":
|
|
2468
|
+
actions, meta_extra = clean_pdf(path, dest, deep_images=deep_images)
|
|
2469
|
+
meta.update(meta_extra)
|
|
2470
|
+
elif fmt == "docx":
|
|
2471
|
+
cleaned, actions = clean_docx(data, also_layer_a_text=also_layer_a_text)
|
|
2472
|
+
safe_write_bytes(dest, cleaned)
|
|
2473
|
+
elif fmt == "xlsx":
|
|
2474
|
+
cleaned, actions = clean_xlsx(data, also_layer_a_text=also_layer_a_text)
|
|
2475
|
+
safe_write_bytes(dest, cleaned)
|
|
2476
|
+
elif fmt == "pptx":
|
|
2477
|
+
cleaned, actions = clean_pptx(data, also_layer_a_text=also_layer_a_text)
|
|
2478
|
+
safe_write_bytes(dest, cleaned)
|
|
2479
|
+
elif fmt == "odt":
|
|
2480
|
+
cleaned, actions = clean_odt(data, also_layer_a_text=also_layer_a_text)
|
|
2481
|
+
safe_write_bytes(dest, cleaned)
|
|
2482
|
+
elif fmt == "epub":
|
|
2483
|
+
cleaned, actions = clean_epub(data, also_layer_a_text=also_layer_a_text)
|
|
2484
|
+
safe_write_bytes(dest, cleaned)
|
|
2485
|
+
elif fmt == "html":
|
|
2486
|
+
text = data.decode("utf-8", errors="surrogateescape")
|
|
2487
|
+
text, actions = clean_html(text)
|
|
2488
|
+
if also_layer_a_text:
|
|
2489
|
+
text2, stats = clean_text(text)
|
|
2490
|
+
if stats["removed_count"] or stats["replaced_count"]:
|
|
2491
|
+
actions.append(
|
|
2492
|
+
f"layer A text: removed={stats['removed_count']} replaced={stats['replaced_count']}"
|
|
2493
|
+
)
|
|
2494
|
+
text = text2
|
|
2495
|
+
safe_write_text(dest, text)
|
|
2496
|
+
elif fmt == "markdown":
|
|
2497
|
+
text = data.decode("utf-8", errors="surrogateescape")
|
|
2498
|
+
text, actions = clean_markdown(text)
|
|
2499
|
+
if also_layer_a_text:
|
|
2500
|
+
text2, stats = clean_text(text)
|
|
2501
|
+
if stats["removed_count"] or stats["replaced_count"]:
|
|
2502
|
+
actions.append(
|
|
2503
|
+
f"layer A text: removed={stats['removed_count']} replaced={stats['replaced_count']}"
|
|
2504
|
+
)
|
|
2505
|
+
text = text2
|
|
2506
|
+
safe_write_text(dest, text)
|
|
2507
|
+
else:
|
|
2508
|
+
raise ValueError(f"unsupported container format: {fmt}")
|
|
2509
|
+
|
|
2510
|
+
after = inspect_container(dest)
|
|
2511
|
+
return {
|
|
2512
|
+
"input": str(path),
|
|
2513
|
+
"output": str(dest),
|
|
2514
|
+
"format": fmt,
|
|
2515
|
+
"actions": actions,
|
|
2516
|
+
"bytes_in": len(data),
|
|
2517
|
+
"bytes_out": dest.stat().st_size,
|
|
2518
|
+
"still_has_c2pa": after.has_c2pa,
|
|
2519
|
+
"still_has_ai_metadata": after.has_ai_metadata,
|
|
2520
|
+
"post_findings": after.findings,
|
|
2521
|
+
"meta": meta,
|
|
2522
|
+
}
|