clm-kernel 0.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (333) hide show
  1. clm/__init__.py +230 -0
  2. clm/__main__.py +25 -0
  3. clm/cross_cutting/__init__.py +50 -0
  4. clm/cross_cutting/_test_runner.py +36 -0
  5. clm/cross_cutting/async_utils.py +25 -0
  6. clm/cross_cutting/cel_builder.py +123 -0
  7. clm/cross_cutting/cel_converter.py +697 -0
  8. clm/cross_cutting/cel_engine.py +252 -0
  9. clm/cross_cutting/cel_ops.py +72 -0
  10. clm/cross_cutting/cel_ops_db.py +157 -0
  11. clm/cross_cutting/cel_ops_files.py +99 -0
  12. clm/cross_cutting/cel_ops_state.py +71 -0
  13. clm/cross_cutting/cel_ops_strings.py +64 -0
  14. clm/cross_cutting/cel_resolver.py +63 -0
  15. clm/cross_cutting/cel_slices.py +56 -0
  16. clm/cross_cutting/cel_store.py +330 -0
  17. clm/cross_cutting/clm_logger.py +140 -0
  18. clm/cross_cutting/config/__init__.py +155 -0
  19. clm/cross_cutting/config/config_constants.py +392 -0
  20. clm/cross_cutting/config/env_parameters.py +90 -0
  21. clm/cross_cutting/config/logging.py +378 -0
  22. clm/cross_cutting/config/settings.py +504 -0
  23. clm/cross_cutting/context.py +5 -0
  24. clm/cross_cutting/domain_models.py +55 -0
  25. clm/cross_cutting/domain_types.py +61 -0
  26. clm/cross_cutting/effects.py +67 -0
  27. clm/cross_cutting/errors.py +153 -0
  28. clm/cross_cutting/governance.py +331 -0
  29. clm/cross_cutting/guards.py +41 -0
  30. clm/cross_cutting/mcard.py +321 -0
  31. clm/cross_cutting/native.py +185 -0
  32. clm/cross_cutting/observability.py +289 -0
  33. clm/cross_cutting/pcard.py +434 -0
  34. clm/cross_cutting/registry.py +71 -0
  35. clm/cross_cutting/resources.py +84 -0
  36. clm/cross_cutting/secrets.py +44 -0
  37. clm/cross_cutting/subprocess_utils.py +118 -0
  38. clm/cross_cutting/telemetry.py +164 -0
  39. clm/cross_cutting/testing/__init__.py +8 -0
  40. clm/cross_cutting/testing/comparator.py +187 -0
  41. clm/cross_cutting/testing/datasets.py +87 -0
  42. clm/cross_cutting/timeutil.py +20 -0
  43. clm/cross_cutting/types/new_type_check.py +0 -0
  44. clm/cross_cutting/utils/__init__.py +1 -0
  45. clm/cross_cutting/utils/url_safety.py +50 -0
  46. clm/cross_cutting/vcard.py +560 -0
  47. clm/cross_cutting/writer.py +118 -0
  48. clm/layer0/README.md +50 -0
  49. clm/layer0/__init__.py +223 -0
  50. clm/layer0/algebra.py +60 -0
  51. clm/layer0/algebra_baldwin.py +133 -0
  52. clm/layer0/algebra_baldwin_generators.py +36 -0
  53. clm/layer0/algebra_equivalence.py +262 -0
  54. clm/layer0/algebra_ingest.py +306 -0
  55. clm/layer0/algebra_manifest_io.py +156 -0
  56. clm/layer0/assembler.py +481 -0
  57. clm/layer0/cel/__init__.py +23 -0
  58. clm/layer0/cel_eval.py +491 -0
  59. clm/layer0/codec.py +106 -0
  60. clm/layer0/context.py +122 -0
  61. clm/layer0/db.py +160 -0
  62. clm/layer0/exceptions.py +73 -0
  63. clm/layer0/fibration.py +106 -0
  64. clm/layer0/gates.py +50 -0
  65. clm/layer0/hash.py +80 -0
  66. clm/layer0/loader.py +826 -0
  67. clm/layer0/math_utils.py +163 -0
  68. clm/layer0/mcard.py +96 -0
  69. clm/layer0/model/__init__.py +0 -0
  70. clm/layer0/model/action.py +223 -0
  71. clm/layer0/model/card.py +280 -0
  72. clm/layer0/model/card_triad.py +427 -0
  73. clm/layer0/model/compatibility.py +60 -0
  74. clm/layer0/model/dictionary.py +46 -0
  75. clm/layer0/model/dots.py +331 -0
  76. clm/layer0/model/event_producer.py +88 -0
  77. clm/layer0/model/g_time.py +89 -0
  78. clm/layer0/model/handle.py +163 -0
  79. clm/layer0/model/hash/__init__.py +6 -0
  80. clm/layer0/model/hash/algorithms/__init__.py +3 -0
  81. clm/layer0/model/hash/algorithms/custom_hash.py +7 -0
  82. clm/layer0/model/hash/algorithms/local_sha256.py +178 -0
  83. clm/layer0/model/hash/constants.py +53 -0
  84. clm/layer0/model/hash/enums.py +60 -0
  85. clm/layer0/model/hash/validator.py +307 -0
  86. clm/layer0/model/interpreter.py +275 -0
  87. clm/layer0/model/pagination.py +45 -0
  88. clm/layer0/model/pcard.py +716 -0
  89. clm/layer0/model/schema.py +44 -0
  90. clm/layer0/model/utils/__init__.py +1 -0
  91. clm/layer0/model/utils/content_analyzer.py +152 -0
  92. clm/layer0/model/validators/__init__.py +1 -0
  93. clm/layer0/model/validators/base_validator.py +44 -0
  94. clm/layer0/model/validators/binary_validator.py +66 -0
  95. clm/layer0/model/validators/text_validator.py +82 -0
  96. clm/layer0/model/validators/validation_registry.py +49 -0
  97. clm/layer0/model/vcard.py +823 -0
  98. clm/layer0/model/vcard_ext/__init__.py +39 -0
  99. clm/layer0/model/vcard_ext/core.py +60 -0
  100. clm/layer0/model/vcard_ext/network.py +23 -0
  101. clm/layer0/model/vcard_ext/observability.py +63 -0
  102. clm/layer0/model/vcard_ext/storage.py +60 -0
  103. clm/layer0/model/vcard_ext/vendors.py +211 -0
  104. clm/layer0/model/vcard_sandwich.py +119 -0
  105. clm/layer0/model/vcard_vocabulary.py +463 -0
  106. clm/layer0/model/workflow.py +274 -0
  107. clm/layer0/narrative.py +431 -0
  108. clm/layer0/ontology/collection.py +125 -0
  109. clm/layer0/parser.py +429 -0
  110. clm/layer0/relations.py +73 -0
  111. clm/layer0/rule_evaluator.py +251 -0
  112. clm/layer0/schema/__init__.py +38 -0
  113. clm/layer0/schema/ddl.py +97 -0
  114. clm/layer0/templates/abstract.yaml +99 -0
  115. clm/layer0/templates/balanced.yaml +253 -0
  116. clm/layer0/templates/concrete.yaml +172 -0
  117. clm/layer0/type_registry.py +223 -0
  118. clm/layer0/types.py +204 -0
  119. clm/layer0/utils.py +886 -0
  120. clm/layer0/verifier_core.py +262 -0
  121. clm/layer1/__init__.py +31 -0
  122. clm/layer1/action_dispatcher.py +275 -0
  123. clm/layer1/adapter_registry.py +233 -0
  124. clm/layer1/algebra_baldwin_generators.py +192 -0
  125. clm/layer1/arrow_compiler.py +525 -0
  126. clm/layer1/batch_processor.py +152 -0
  127. clm/layer1/cel/__init__.py +27 -0
  128. clm/layer1/cel/builder.py +173 -0
  129. clm/layer1/engine.py +759 -0
  130. clm/layer1/engine_async.py +89 -0
  131. clm/layer1/eval_callback.py +49 -0
  132. clm/layer1/gate_composer.py +229 -0
  133. clm/layer1/hoare_sandwich.py +298 -0
  134. clm/layer1/identity_guard.py +348 -0
  135. clm/layer1/matrix.py +138 -0
  136. clm/layer1/neural_runtime.py +221 -0
  137. clm/layer1/operations/__init__.py +123 -0
  138. clm/layer1/operations/augment.py +156 -0
  139. clm/layer1/operations/base.py +109 -0
  140. clm/layer1/operations/builtins.py +338 -0
  141. clm/layer1/operations/dispatch.py +144 -0
  142. clm/layer1/operations/exclude.py +107 -0
  143. clm/layer1/operations/handle.py +197 -0
  144. clm/layer1/operations/identity.py +139 -0
  145. clm/layer1/operations/invert.py +66 -0
  146. clm/layer1/operations/loader.py +116 -0
  147. clm/layer1/operations/manipulators/__init__.py +23 -0
  148. clm/layer1/operations/manipulators/document.py +211 -0
  149. clm/layer1/operations/manipulators/media.py +134 -0
  150. clm/layer1/operations/manipulators/structured.py +159 -0
  151. clm/layer1/operations/manipulators/tabular.py +334 -0
  152. clm/layer1/operations/port.py +65 -0
  153. clm/layer1/operations/services.py +529 -0
  154. clm/layer1/operations/split.py +117 -0
  155. clm/layer1/operations/substitute.py +75 -0
  156. clm/layer1/petri.py +1052 -0
  157. clm/layer1/pocketflow_orchestrator.py +408 -0
  158. clm/layer1/pocketflow_scheduler.py +519 -0
  159. clm/layer1/protocols.py +197 -0
  160. clm/layer1/runtime_adapter.py +718 -0
  161. clm/layer1/runtimes/__init__.py +71 -0
  162. clm/layer1/runtimes/_generated_assertions.py +125 -0
  163. clm/layer1/runtimes/_generated_types.py +42 -0
  164. clm/layer1/runtimes/base.py +265 -0
  165. clm/layer1/runtimes/binary.py +101 -0
  166. clm/layer1/runtimes/factory.py +203 -0
  167. clm/layer1/runtimes/javascript.py +321 -0
  168. clm/layer1/runtimes/javascript_runtime.js +33 -0
  169. clm/layer1/runtimes/js_scripts/isomorphism.mjs +22 -0
  170. clm/layer1/runtimes/js_scripts/loader.mjs +52 -0
  171. clm/layer1/runtimes/lambda_calc.py +126 -0
  172. clm/layer1/runtimes/python.py +288 -0
  173. clm/layer1/runtimes/python_runtime.py +41 -0
  174. clm/layer1/runtimes/script.py +114 -0
  175. clm/layer1/sandbox.py +334 -0
  176. clm/layer1/savepoint_guard.py +136 -0
  177. clm/layer1/sparse_net.py +122 -0
  178. clm/layer1/spec_tester.py +165 -0
  179. clm/layer1/store.py +295 -0
  180. clm/layer1/test_runner.py +498 -0
  181. clm/layer1/type_inspector.py +166 -0
  182. clm/layer1/verifier.py +772 -0
  183. clm/layer1/vm.py +91 -0
  184. clm/layer2/__init__.py +78 -0
  185. clm/layer2/card_collection.py +678 -0
  186. clm/layer2/classifier.py +479 -0
  187. clm/layer2/engine/__init__.py +1 -0
  188. clm/layer2/engine/abstract_sql_engine.py +320 -0
  189. clm/layer2/engine/base.py +193 -0
  190. clm/layer2/engine/duckdb_engine.py +576 -0
  191. clm/layer2/engine/sqlite_engine.py +518 -0
  192. clm/layer2/fiber/__init__.py +31 -0
  193. clm/layer2/fiber/context.py +89 -0
  194. clm/layer2/fiber/disposable.py +163 -0
  195. clm/layer2/fiber/fiber.py +125 -0
  196. clm/layer2/fiber/hoare.py +191 -0
  197. clm/layer2/file_io.py +376 -0
  198. clm/layer2/file_registrar.py +77 -0
  199. clm/layer2/ingest.py +202 -0
  200. clm/layer2/lineage.py +322 -0
  201. clm/layer2/mcard_fs.py +262 -0
  202. clm/layer2/merkle.py +11 -0
  203. clm/layer2/mime.py +244 -0
  204. clm/layer2/mime_detector.py +245 -0
  205. clm/layer2/private_collection.py +118 -0
  206. clm/layer2/shared_store.py +11 -0
  207. clm/layer2/storage/__init__.py +190 -0
  208. clm/layer2/storage/db_bitemporal.py +141 -0
  209. clm/layer2/storage/db_driver.py +331 -0
  210. clm/layer2/storage/db_hyperlinks.py +214 -0
  211. clm/layer2/storage/db_pool.py +139 -0
  212. clm/layer2/storage/db_repository.py +229 -0
  213. clm/layer2/storage/savepoint.py +78 -0
  214. clm/layer2/storage/sqlite_wal.py +74 -0
  215. clm/layer2/tridb.py +245 -0
  216. clm/layer2/vcard_receipt.py +71 -0
  217. clm/layer2/vfs.py +17 -0
  218. clm/layer3/__init__.py +141 -0
  219. clm/layer3/agency/__init__.py +19 -0
  220. clm/layer3/agency/config.py +220 -0
  221. clm/layer3/agency/providers/__init__.py +15 -0
  222. clm/layer3/agency/providers/base.py +91 -0
  223. clm/layer3/agency/providers/mlc_llm.py +169 -0
  224. clm/layer3/agency/providers/ollama.py +203 -0
  225. clm/layer3/agency/router.py +71 -0
  226. clm/layer3/agency/runtime.py +524 -0
  227. clm/layer3/gateway.py +353 -0
  228. clm/layer3/open_interpreter_adapter.py +159 -0
  229. clm/layer3/rag/__init__.py +70 -0
  230. clm/layer3/rag/cli.py +228 -0
  231. clm/layer3/rag/config.py +153 -0
  232. clm/layer3/rag/embeddings/__init__.py +16 -0
  233. clm/layer3/rag/embeddings/base.py +79 -0
  234. clm/layer3/rag/embeddings/ollama.py +211 -0
  235. clm/layer3/rag/embeddings/vision.py +290 -0
  236. clm/layer3/rag/engine.py +276 -0
  237. clm/layer3/rag/graph/__init__.py +22 -0
  238. clm/layer3/rag/graph/community.py +198 -0
  239. clm/layer3/rag/graph/engine.py +433 -0
  240. clm/layer3/rag/graph/extractor.py +255 -0
  241. clm/layer3/rag/graph/schema.py +115 -0
  242. clm/layer3/rag/graph/store.py +641 -0
  243. clm/layer3/rag/indexer.py +333 -0
  244. clm/layer3/rag/llm_providers.json +52 -0
  245. clm/layer3/rag/semantic_versioning.py +313 -0
  246. clm/layer3/rag/vector/__init__.py +28 -0
  247. clm/layer3/rag/vector/handle_vector_store.py +749 -0
  248. clm/layer3/rag/vector/schema.py +151 -0
  249. clm/layer3/rag/vector/store.py +632 -0
  250. clm/layer3/reticulum/__init__.py +55 -0
  251. clm/layer3/reticulum/bridge.py +74 -0
  252. clm/layer3/reticulum/daemon.py +88 -0
  253. clm/layer3/reticulum/discovery.py +164 -0
  254. clm/layer3/reticulum/identity.py +107 -0
  255. clm/layer3/reticulum/media.py +148 -0
  256. clm/layer3/reticulum/transport.py +133 -0
  257. clm/layer3/satori/__init__.py +209 -0
  258. clm/layer3/satori/alpha_conversion.py +168 -0
  259. clm/layer3/satori/beta_reduction.py +359 -0
  260. clm/layer3/satori/continuation.py +38 -0
  261. clm/layer3/satori/eta_conversion.py +205 -0
  262. clm/layer3/satori/free_variables.py +163 -0
  263. clm/layer3/satori/io_effects.py +370 -0
  264. clm/layer3/satori/lambda_runtime.py +726 -0
  265. clm/layer3/satori/lambda_term.py +230 -0
  266. clm/layer3/satori/sheaf_audit.py +261 -0
  267. clm/layer3/satori/speech_act.py +279 -0
  268. clm/layer4/README.md +9 -0
  269. clm/layer4/__init__.py +67 -0
  270. clm/layer4/_sovereign_executor.py +580 -0
  271. clm/layer4/action_commands.py +719 -0
  272. clm/layer4/api.py +236 -0
  273. clm/layer4/bootstrap/__init__.py +13 -0
  274. clm/layer4/bootstrap/bootstrapper.py +118 -0
  275. clm/layer4/bootstrap/cleanup.py +44 -0
  276. clm/layer4/bootstrap/clm_hooks.py +49 -0
  277. clm/layer4/bootstrap/genesis.py +77 -0
  278. clm/layer4/bootstrap/injection.py +125 -0
  279. clm/layer4/cli.py +549 -0
  280. clm/layer4/collection_manager.py +553 -0
  281. clm/layer4/commands/__init__.py +81 -0
  282. clm/layer4/commands/action_cmd.py +42 -0
  283. clm/layer4/commands/check_cmd.py +82 -0
  284. clm/layer4/commands/envelope.py +73 -0
  285. clm/layer4/commands/evaluate_cmd.py +224 -0
  286. clm/layer4/commands/mcard_cmd.py +24 -0
  287. clm/layer4/commands/petri_cmd.py +92 -0
  288. clm/layer4/commands/tridb_cmd.py +319 -0
  289. clm/layer4/commands/version_cmd.py +27 -0
  290. clm/layer4/diff_commands.py +167 -0
  291. clm/layer4/eoa_sponsor.py +232 -0
  292. clm/layer4/events.py +69 -0
  293. clm/layer4/flux_gateway.py +133 -0
  294. clm/layer4/gateway.py +91 -0
  295. clm/layer4/loader.py +307 -0
  296. clm/layer4/mcard_commands.py +166 -0
  297. clm/layer4/membrane.py +273 -0
  298. clm/layer4/navigation.py +50 -0
  299. clm/layer4/protocol/protocol_entry_python.py +108 -0
  300. clm/layer4/protocol/python_sync.py +126 -0
  301. clm/layer4/runner.py +150 -0
  302. clm/layer4/satori.py +262 -0
  303. clm/layer4/server/python_sync_client.py +150 -0
  304. clm/layer4/server/python_sync_server.py +139 -0
  305. clm/layer4/server/service_api.py +315 -0
  306. clm/layer4/server/start_servers.py +107 -0
  307. clm/layer4/server/stop_servers.py +24 -0
  308. clm/layer4/simulated_optimization_fix.py +0 -0
  309. clm/layer4/storage.py +463 -0
  310. clm/layer4/tridb_admin.py +268 -0
  311. clm/layer4/websocket/test_ws_client.py +104 -0
  312. clm/layer4/websocket/test_ws_server.py +138 -0
  313. clm/layer4/websocket/ws_server_local.py +160 -0
  314. clm/layer5/__init__.py +51 -0
  315. clm/layer5/async_batch_writer.py +133 -0
  316. clm/layer5/learning.py +276 -0
  317. clm/layer5/telemetry_hook.py +107 -0
  318. clm/layer5/type_lattice.py +238 -0
  319. clm/py.typed +0 -0
  320. clm/schemas/__init__.py +652 -0
  321. clm/schemas/abstract_face_schema.yaml +77 -0
  322. clm/schemas/cel_schema.yaml +416 -0
  323. clm/schemas/lifecycle/__init__.py +45 -0
  324. clm/schemas/lifecycle/models.py +123 -0
  325. clm/schemas/lifecycle/service.py +364 -0
  326. clm/schemas/mcard_schema.sql +192 -0
  327. clm/schemas/mcard_vector_schema.sql +283 -0
  328. clm/schemas/mlp-context-v1.json +91 -0
  329. clm_kernel-0.0.0.dist-info/METADATA +121 -0
  330. clm_kernel-0.0.0.dist-info/RECORD +333 -0
  331. clm_kernel-0.0.0.dist-info/WHEEL +4 -0
  332. clm_kernel-0.0.0.dist-info/entry_points.txt +3 -0
  333. clm_kernel-0.0.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,44 @@
1
+ # mcard/model/schema.py
2
+ """
3
+ DEPRECATED: This module is a backward-compatibility shim.
4
+
5
+ All schemas are now loaded from the singleton:
6
+ from clm.schemas import MCardSchema
7
+ schema = MCardSchema.get_instance()
8
+
9
+ The single source of truth is: schema/mcard_schema.sql
10
+
11
+ WHY THIS FILE EXISTS:
12
+ ---------------------
13
+ This file exists only for backward compatibility with legacy imports like:
14
+ from clm.layer0.model.schema import CARD_TABLE_SCHEMA
15
+
16
+ New code should import directly from clm.schemas:
17
+ from clm.schemas import MCardSchema, get_schema
18
+
19
+ This file will be removed in a future version.
20
+ """
21
+
22
+ from clm.schemas import (
23
+ CARD_TABLE_SCHEMA,
24
+ HANDLE_HISTORY_SCHEMA,
25
+ HANDLE_INDEX_SCHEMA,
26
+ HANDLE_REGISTRY_SCHEMA,
27
+ MCardSchema,
28
+ get_schema,
29
+ init_all_tables,
30
+ init_core_tables,
31
+ init_handle_tables,
32
+ )
33
+
34
+ __all__ = [
35
+ "CARD_TABLE_SCHEMA",
36
+ "HANDLE_REGISTRY_SCHEMA",
37
+ "HANDLE_HISTORY_SCHEMA",
38
+ "HANDLE_INDEX_SCHEMA",
39
+ "MCardSchema",
40
+ "get_schema",
41
+ "init_all_tables",
42
+ "init_core_tables",
43
+ "init_handle_tables",
44
+ ]
@@ -0,0 +1 @@
1
+ # Content analysis utilities
@@ -0,0 +1,152 @@
1
+ """
2
+ Content analysis utilities for detecting problematic content patterns.
3
+ """
4
+
5
+ # Constants for sampling and limits
6
+ MAX_SAMPLE_BYTES = 64 * 1024 # 64KB
7
+ MAX_SAMPLE_LINES = 100
8
+
9
+ # Heuristic thresholds for problematic content detection
10
+ _EXT_LONG_LINE_KNOWN = {
11
+ ".js",
12
+ ".mjs",
13
+ ".min.js",
14
+ ".bundle.js",
15
+ ".css",
16
+ ".min.css",
17
+ ".json",
18
+ ".jsonl",
19
+ ".xml",
20
+ ".svg",
21
+ ".html",
22
+ ".htm",
23
+ ".map", # source maps
24
+ ".wasm",
25
+ }
26
+
27
+ _PROBLEMATIC_MAX_KNOWN_TYPE = 1024 * 1024 # 1MB
28
+ _PROBLEMATIC_SAMPLE_SIZE = 32 * 1024 # 32KB
29
+ _PROBLEMATIC_MIN_MULTI_POSITION = 100 * 1024 # 100KB
30
+ _PATHOLOGICAL_NO_BREAK_MIN = 8192
31
+ _PATHOLOGICAL_NO_BREAK_STRICT_MIN = 32 * 1024
32
+ _NULL_RATIO_THRESHOLD = 0.1
33
+ _CTRL_RATIO_THRESHOLD = 0.2
34
+ _MAX_LINE_LENGTH_KNOWN = 100000
35
+ _MAX_LINE_LENGTH_DEFAULT = 50000
36
+ _AVG_LINE_LENGTH_KNOWN = 20000
37
+ _AVG_LINE_LENGTH_DEFAULT = 5000
38
+
39
+
40
+ class ContentAnalyzer:
41
+ """Utility class for analyzing content patterns and detecting problematic content."""
42
+
43
+ @staticmethod
44
+ def is_known_long_line_extension(file_extension: str | None) -> bool:
45
+ """Return True if extension is typically legitimate with long lines."""
46
+ if not file_extension:
47
+ return False
48
+ return file_extension.lower() in _EXT_LONG_LINE_KNOWN
49
+
50
+ @staticmethod
51
+ def is_unstructured_binary(sample: bytes) -> bool:
52
+ """Heuristic check for unstructured binary content."""
53
+ if len(sample) < 512:
54
+ return False
55
+ null_count = sample.count(b"\x00")
56
+ control_chars = sum(1 for b in sample if b < 32 and b not in (9, 10, 13))
57
+ null_ratio = null_count / len(sample)
58
+ control_ratio = control_chars / len(sample)
59
+ return (
60
+ null_ratio > _NULL_RATIO_THRESHOLD or control_ratio > _CTRL_RATIO_THRESHOLD
61
+ )
62
+
63
+ @staticmethod
64
+ def has_pathological_lines(sample: bytes, is_known_type: bool) -> bool:
65
+ """Detect pathological line structures (extremely long or no breaks)."""
66
+ max_line_len = (
67
+ _MAX_LINE_LENGTH_KNOWN if is_known_type else _MAX_LINE_LENGTH_DEFAULT
68
+ )
69
+ max_avg_len = (
70
+ _AVG_LINE_LENGTH_KNOWN if is_known_type else _AVG_LINE_LENGTH_DEFAULT
71
+ )
72
+
73
+ if (
74
+ b"\n" not in sample
75
+ and b"\r" not in sample
76
+ and len(sample) >= _PATHOLOGICAL_NO_BREAK_MIN
77
+ ):
78
+ if (not is_known_type) or len(sample) >= _PATHOLOGICAL_NO_BREAK_STRICT_MIN:
79
+ return True
80
+
81
+ lines = sample.replace(b"\r\n", b"\n").replace(b"\r", b"\n").split(b"\n")
82
+ if not lines:
83
+ return False
84
+
85
+ if any(len(line) > max_line_len for line in lines):
86
+ return True
87
+
88
+ if len(lines) > 1:
89
+ avg_len = sum(len(line) for line in lines) / len(lines)
90
+ if avg_len > max_avg_len:
91
+ return True
92
+
93
+ return False
94
+
95
+ @staticmethod
96
+ def is_problematic_bytes(content: bytes, file_extension: str | None = None) -> bool:
97
+ """
98
+ Byte-content heuristic for detecting problematic content patterns.
99
+
100
+ Args:
101
+ content: The byte content to analyze
102
+ file_extension: Optional file extension for type-aware thresholds
103
+
104
+ Returns:
105
+ True if content appears problematic
106
+ """
107
+ if not content:
108
+ return False
109
+
110
+ is_known_type = ContentAnalyzer.is_known_long_line_extension(file_extension)
111
+ sample = content[:_PROBLEMATIC_SAMPLE_SIZE]
112
+
113
+ if ContentAnalyzer.is_unstructured_binary(sample):
114
+ return True
115
+
116
+ if ContentAnalyzer.has_pathological_lines(sample, is_known_type):
117
+ return True
118
+
119
+ return False
120
+
121
+ @staticmethod
122
+ def prepare_content_sample(
123
+ content: bytes,
124
+ max_bytes: int = MAX_SAMPLE_BYTES,
125
+ max_lines: int = MAX_SAMPLE_LINES,
126
+ ) -> tuple[str, list[str], str]:
127
+ """
128
+ Prepare content sample for analysis.
129
+
130
+ Truncates raw bytes *before* decoding to avoid O(n) decode on
131
+ multi-MB inputs. This matches the TypeScript DetectionStrategy
132
+ which caps the analytical sample to 8 192 bytes.
133
+
134
+ Args:
135
+ content: Raw byte content
136
+ max_bytes: Maximum bytes to sample
137
+ max_lines: Maximum lines to process
138
+
139
+ Returns:
140
+ Tuple of (content_sample, lines, first_line)
141
+ """
142
+ # Cap bytes BEFORE decoding — avoids decoding multi-MB blobs.
143
+ sample_bytes = content[:max_bytes]
144
+ try:
145
+ content_sample = sample_bytes.decode("utf-8", errors="replace")
146
+ except Exception:
147
+ content_sample = str(sample_bytes, errors="replace")
148
+
149
+ lines = content_sample.split("\n")[:max_lines]
150
+ first_line = lines[0] if lines else ""
151
+
152
+ return content_sample, lines, first_line
@@ -0,0 +1 @@
1
+ # Content validation modules
@@ -0,0 +1,44 @@
1
+ """
2
+ Base validator interface for content validation.
3
+ """
4
+
5
+ from abc import ABC, abstractmethod
6
+
7
+
8
+ class ValidationError(Exception):
9
+ """Exception raised for validation errors."""
10
+
11
+ def __init__(self, message):
12
+ self.message = message
13
+ super().__init__(self.message)
14
+
15
+
16
+ class BaseValidator(ABC):
17
+ """Abstract base class for content validators."""
18
+
19
+ @abstractmethod
20
+ def validate(self, content: str | bytes, mime_type: str) -> None:
21
+ """
22
+ Validate content for a specific MIME type.
23
+
24
+ Args:
25
+ content: The content to validate
26
+ mime_type: The detected MIME type
27
+
28
+ Raises:
29
+ ValidationError: If content is invalid
30
+ """
31
+ pass
32
+
33
+ @abstractmethod
34
+ def can_validate(self, mime_type: str) -> bool:
35
+ """
36
+ Check if this validator can handle the given MIME type.
37
+
38
+ Args:
39
+ mime_type: The MIME type to check
40
+
41
+ Returns:
42
+ True if this validator can handle the MIME type
43
+ """
44
+ pass
@@ -0,0 +1,66 @@
1
+ """
2
+ Binary content validator.
3
+ """
4
+
5
+ from .base_validator import BaseValidator, ValidationError
6
+
7
+
8
+ class BinaryValidator(BaseValidator):
9
+ """Validator for binary content types."""
10
+
11
+ BINARY_MIME_TYPES = {
12
+ "image/png",
13
+ "image/jpeg",
14
+ "image/gif",
15
+ "image/bmp",
16
+ "application/pdf",
17
+ "application/zip",
18
+ "video/mp4",
19
+ "audio/wav",
20
+ "application/octet-stream",
21
+ }
22
+
23
+ def can_validate(self, mime_type: str) -> bool:
24
+ return mime_type in self.BINARY_MIME_TYPES or mime_type.startswith(
25
+ ("image/", "audio/", "video/")
26
+ )
27
+
28
+ def validate(self, content: str | bytes, mime_type: str) -> None:
29
+ """Validate binary content based on MIME type."""
30
+ if isinstance(content, str):
31
+ content = content.encode("utf-8")
32
+
33
+ if mime_type.startswith("image/"):
34
+ self._validate_image(content, mime_type)
35
+ elif mime_type == "application/pdf":
36
+ self._validate_pdf(content)
37
+ elif mime_type == "application/zip":
38
+ self._validate_zip(content)
39
+
40
+ def _validate_image(self, content: bytes, mime_type: str) -> None:
41
+ """Validate image content."""
42
+ if mime_type == "image/png" and len(content) <= 8:
43
+ raise ValidationError("Invalid PNG content: truncated file")
44
+ elif mime_type == "image/jpeg" and len(content) <= 3:
45
+ raise ValidationError("Invalid JPEG content: truncated file")
46
+ elif mime_type == "image/gif" and len(content) <= 6:
47
+ raise ValidationError("Invalid GIF content: truncated file")
48
+
49
+ # Check for proper signature
50
+ signatures = {
51
+ "image/png": b"\x89PNG\r\n\x1a\n",
52
+ "image/jpeg": b"\xff\xd8\xff",
53
+ "image/gif": b"GIF8",
54
+ }
55
+ if mime_type in signatures and not content.startswith(signatures[mime_type]):
56
+ raise ValidationError(f"Invalid {mime_type} content: missing proper header")
57
+
58
+ def _validate_pdf(self, content: bytes) -> None:
59
+ """Validate PDF content."""
60
+ if not content.startswith(b"%PDF-"):
61
+ raise ValidationError("Invalid PDF content")
62
+
63
+ def _validate_zip(self, content: bytes) -> None:
64
+ """Validate ZIP content."""
65
+ if len(content) <= 4:
66
+ raise ValidationError("Invalid ZIP content")
@@ -0,0 +1,82 @@
1
+ """
2
+ Text content validator.
3
+ """
4
+
5
+ import json
6
+
7
+ try:
8
+ from defusedxml.ElementTree import ParseError
9
+ from defusedxml.ElementTree import fromstring as safe_fromstring
10
+ except ImportError:
11
+ from xml.etree.ElementTree import ParseError
12
+ from xml.etree.ElementTree import fromstring as safe_fromstring # nosec B314
13
+
14
+ from .base_validator import BaseValidator, ValidationError
15
+
16
+
17
+ class TextValidator(BaseValidator):
18
+ """Validator for text-based content types."""
19
+
20
+ TEXT_MIME_TYPES = {
21
+ "text/plain",
22
+ "application/json",
23
+ "application/xml",
24
+ "text/xml",
25
+ "image/svg+xml",
26
+ "text/html",
27
+ "text/markdown",
28
+ }
29
+
30
+ def can_validate(self, mime_type: str) -> bool:
31
+ return mime_type in self.TEXT_MIME_TYPES
32
+
33
+ def validate(self, content: str | bytes, mime_type: str) -> None:
34
+ """Validate text content based on MIME type."""
35
+ if isinstance(content, bytes):
36
+ try:
37
+ text_content = content.decode("utf-8")
38
+ except UnicodeDecodeError as e:
39
+ raise ValidationError("Invalid content: not valid UTF-8") from e
40
+ else:
41
+ text_content = content
42
+
43
+ if mime_type == "text/plain":
44
+ self._validate_plain_text(text_content)
45
+ elif mime_type == "application/json":
46
+ self._validate_json(text_content)
47
+ elif mime_type in ("application/xml", "text/xml", "image/svg+xml"):
48
+ self._validate_xml(text_content)
49
+
50
+ def _validate_plain_text(self, content: str) -> None:
51
+ """Validate plain text content."""
52
+ if not content.strip():
53
+ raise ValidationError("Invalid content: empty text")
54
+ if len(content.strip()) < 3:
55
+ raise ValidationError("Invalid content: too short")
56
+ if all(ord(c) < 32 for c in content.strip()):
57
+ raise ValidationError("Invalid content: contains only control characters")
58
+ # Heuristic: valid plain text should contain some whitespace
59
+ if not any(c.isspace() for c in content):
60
+ # Check for multi-word content without spaces
61
+ if (
62
+ len(content.split()) == 1 and len(content) > 20
63
+ ): # Arbitrary length check
64
+ raise ValidationError("Invalid content: likely not plain text")
65
+
66
+ def _validate_json(self, content: str) -> None:
67
+ """Validate JSON content."""
68
+ try:
69
+ # Check for comments before attempting to parse
70
+ lines = content.split("\n")
71
+ if any(line.strip().startswith("//") for line in lines):
72
+ raise ValidationError("Invalid JSON content: contains comments")
73
+ json.loads(content)
74
+ except json.JSONDecodeError as e:
75
+ raise ValidationError("Invalid JSON content") from e
76
+
77
+ def _validate_xml(self, content: str) -> None:
78
+ """Validate XML content."""
79
+ try:
80
+ safe_fromstring(content) # nosec B314
81
+ except ParseError as e:
82
+ raise ValidationError("Invalid XML content") from e
@@ -0,0 +1,49 @@
1
+ """
2
+ Central registry for content validators.
3
+ """
4
+
5
+ from .base_validator import BaseValidator, ValidationError
6
+ from .binary_validator import BinaryValidator
7
+ from .text_validator import TextValidator
8
+
9
+
10
+ class ValidationRegistry:
11
+ """Central registry for managing content validators."""
12
+
13
+ def __init__(self):
14
+ self.validators: list[BaseValidator] = [
15
+ TextValidator(),
16
+ BinaryValidator(),
17
+ ]
18
+
19
+ def validate(self, content: str | bytes, mime_type: str) -> None:
20
+ """
21
+ Validate content using appropriate validator.
22
+
23
+ Args:
24
+ content: The content to validate
25
+ mime_type: The detected MIME type
26
+
27
+ Raises:
28
+ ValidationError: If content is invalid
29
+ """
30
+ if not content:
31
+ raise ValidationError("Empty content")
32
+
33
+ # Find appropriate validator
34
+ for validator in self.validators:
35
+ if validator.can_validate(mime_type):
36
+ validator.validate(content, mime_type)
37
+ return
38
+
39
+ # If no specific validator found, do basic validation
40
+ self._basic_validation(content)
41
+
42
+ def _basic_validation(self, content: str | bytes) -> None:
43
+ """Basic validation for unknown content types."""
44
+ if isinstance(content, bytes) and not content.strip():
45
+ raise ValidationError("Invalid content: empty byte array")
46
+
47
+
48
+ # Global registry instance
49
+ validation_registry = ValidationRegistry()