clm-kernel 0.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clm/__init__.py +230 -0
- clm/__main__.py +25 -0
- clm/cross_cutting/__init__.py +50 -0
- clm/cross_cutting/_test_runner.py +36 -0
- clm/cross_cutting/async_utils.py +25 -0
- clm/cross_cutting/cel_builder.py +123 -0
- clm/cross_cutting/cel_converter.py +697 -0
- clm/cross_cutting/cel_engine.py +252 -0
- clm/cross_cutting/cel_ops.py +72 -0
- clm/cross_cutting/cel_ops_db.py +157 -0
- clm/cross_cutting/cel_ops_files.py +99 -0
- clm/cross_cutting/cel_ops_state.py +71 -0
- clm/cross_cutting/cel_ops_strings.py +64 -0
- clm/cross_cutting/cel_resolver.py +63 -0
- clm/cross_cutting/cel_slices.py +56 -0
- clm/cross_cutting/cel_store.py +330 -0
- clm/cross_cutting/clm_logger.py +140 -0
- clm/cross_cutting/config/__init__.py +155 -0
- clm/cross_cutting/config/config_constants.py +392 -0
- clm/cross_cutting/config/env_parameters.py +90 -0
- clm/cross_cutting/config/logging.py +378 -0
- clm/cross_cutting/config/settings.py +504 -0
- clm/cross_cutting/context.py +5 -0
- clm/cross_cutting/domain_models.py +55 -0
- clm/cross_cutting/domain_types.py +61 -0
- clm/cross_cutting/effects.py +67 -0
- clm/cross_cutting/errors.py +153 -0
- clm/cross_cutting/governance.py +331 -0
- clm/cross_cutting/guards.py +41 -0
- clm/cross_cutting/mcard.py +321 -0
- clm/cross_cutting/native.py +185 -0
- clm/cross_cutting/observability.py +289 -0
- clm/cross_cutting/pcard.py +434 -0
- clm/cross_cutting/registry.py +71 -0
- clm/cross_cutting/resources.py +84 -0
- clm/cross_cutting/secrets.py +44 -0
- clm/cross_cutting/subprocess_utils.py +118 -0
- clm/cross_cutting/telemetry.py +164 -0
- clm/cross_cutting/testing/__init__.py +8 -0
- clm/cross_cutting/testing/comparator.py +187 -0
- clm/cross_cutting/testing/datasets.py +87 -0
- clm/cross_cutting/timeutil.py +20 -0
- clm/cross_cutting/types/new_type_check.py +0 -0
- clm/cross_cutting/utils/__init__.py +1 -0
- clm/cross_cutting/utils/url_safety.py +50 -0
- clm/cross_cutting/vcard.py +560 -0
- clm/cross_cutting/writer.py +118 -0
- clm/layer0/README.md +50 -0
- clm/layer0/__init__.py +223 -0
- clm/layer0/algebra.py +60 -0
- clm/layer0/algebra_baldwin.py +133 -0
- clm/layer0/algebra_baldwin_generators.py +36 -0
- clm/layer0/algebra_equivalence.py +262 -0
- clm/layer0/algebra_ingest.py +306 -0
- clm/layer0/algebra_manifest_io.py +156 -0
- clm/layer0/assembler.py +481 -0
- clm/layer0/cel/__init__.py +23 -0
- clm/layer0/cel_eval.py +491 -0
- clm/layer0/codec.py +106 -0
- clm/layer0/context.py +122 -0
- clm/layer0/db.py +160 -0
- clm/layer0/exceptions.py +73 -0
- clm/layer0/fibration.py +106 -0
- clm/layer0/gates.py +50 -0
- clm/layer0/hash.py +80 -0
- clm/layer0/loader.py +826 -0
- clm/layer0/math_utils.py +163 -0
- clm/layer0/mcard.py +96 -0
- clm/layer0/model/__init__.py +0 -0
- clm/layer0/model/action.py +223 -0
- clm/layer0/model/card.py +280 -0
- clm/layer0/model/card_triad.py +427 -0
- clm/layer0/model/compatibility.py +60 -0
- clm/layer0/model/dictionary.py +46 -0
- clm/layer0/model/dots.py +331 -0
- clm/layer0/model/event_producer.py +88 -0
- clm/layer0/model/g_time.py +89 -0
- clm/layer0/model/handle.py +163 -0
- clm/layer0/model/hash/__init__.py +6 -0
- clm/layer0/model/hash/algorithms/__init__.py +3 -0
- clm/layer0/model/hash/algorithms/custom_hash.py +7 -0
- clm/layer0/model/hash/algorithms/local_sha256.py +178 -0
- clm/layer0/model/hash/constants.py +53 -0
- clm/layer0/model/hash/enums.py +60 -0
- clm/layer0/model/hash/validator.py +307 -0
- clm/layer0/model/interpreter.py +275 -0
- clm/layer0/model/pagination.py +45 -0
- clm/layer0/model/pcard.py +716 -0
- clm/layer0/model/schema.py +44 -0
- clm/layer0/model/utils/__init__.py +1 -0
- clm/layer0/model/utils/content_analyzer.py +152 -0
- clm/layer0/model/validators/__init__.py +1 -0
- clm/layer0/model/validators/base_validator.py +44 -0
- clm/layer0/model/validators/binary_validator.py +66 -0
- clm/layer0/model/validators/text_validator.py +82 -0
- clm/layer0/model/validators/validation_registry.py +49 -0
- clm/layer0/model/vcard.py +823 -0
- clm/layer0/model/vcard_ext/__init__.py +39 -0
- clm/layer0/model/vcard_ext/core.py +60 -0
- clm/layer0/model/vcard_ext/network.py +23 -0
- clm/layer0/model/vcard_ext/observability.py +63 -0
- clm/layer0/model/vcard_ext/storage.py +60 -0
- clm/layer0/model/vcard_ext/vendors.py +211 -0
- clm/layer0/model/vcard_sandwich.py +119 -0
- clm/layer0/model/vcard_vocabulary.py +463 -0
- clm/layer0/model/workflow.py +274 -0
- clm/layer0/narrative.py +431 -0
- clm/layer0/ontology/collection.py +125 -0
- clm/layer0/parser.py +429 -0
- clm/layer0/relations.py +73 -0
- clm/layer0/rule_evaluator.py +251 -0
- clm/layer0/schema/__init__.py +38 -0
- clm/layer0/schema/ddl.py +97 -0
- clm/layer0/templates/abstract.yaml +99 -0
- clm/layer0/templates/balanced.yaml +253 -0
- clm/layer0/templates/concrete.yaml +172 -0
- clm/layer0/type_registry.py +223 -0
- clm/layer0/types.py +204 -0
- clm/layer0/utils.py +886 -0
- clm/layer0/verifier_core.py +262 -0
- clm/layer1/__init__.py +31 -0
- clm/layer1/action_dispatcher.py +275 -0
- clm/layer1/adapter_registry.py +233 -0
- clm/layer1/algebra_baldwin_generators.py +192 -0
- clm/layer1/arrow_compiler.py +525 -0
- clm/layer1/batch_processor.py +152 -0
- clm/layer1/cel/__init__.py +27 -0
- clm/layer1/cel/builder.py +173 -0
- clm/layer1/engine.py +759 -0
- clm/layer1/engine_async.py +89 -0
- clm/layer1/eval_callback.py +49 -0
- clm/layer1/gate_composer.py +229 -0
- clm/layer1/hoare_sandwich.py +298 -0
- clm/layer1/identity_guard.py +348 -0
- clm/layer1/matrix.py +138 -0
- clm/layer1/neural_runtime.py +221 -0
- clm/layer1/operations/__init__.py +123 -0
- clm/layer1/operations/augment.py +156 -0
- clm/layer1/operations/base.py +109 -0
- clm/layer1/operations/builtins.py +338 -0
- clm/layer1/operations/dispatch.py +144 -0
- clm/layer1/operations/exclude.py +107 -0
- clm/layer1/operations/handle.py +197 -0
- clm/layer1/operations/identity.py +139 -0
- clm/layer1/operations/invert.py +66 -0
- clm/layer1/operations/loader.py +116 -0
- clm/layer1/operations/manipulators/__init__.py +23 -0
- clm/layer1/operations/manipulators/document.py +211 -0
- clm/layer1/operations/manipulators/media.py +134 -0
- clm/layer1/operations/manipulators/structured.py +159 -0
- clm/layer1/operations/manipulators/tabular.py +334 -0
- clm/layer1/operations/port.py +65 -0
- clm/layer1/operations/services.py +529 -0
- clm/layer1/operations/split.py +117 -0
- clm/layer1/operations/substitute.py +75 -0
- clm/layer1/petri.py +1052 -0
- clm/layer1/pocketflow_orchestrator.py +408 -0
- clm/layer1/pocketflow_scheduler.py +519 -0
- clm/layer1/protocols.py +197 -0
- clm/layer1/runtime_adapter.py +718 -0
- clm/layer1/runtimes/__init__.py +71 -0
- clm/layer1/runtimes/_generated_assertions.py +125 -0
- clm/layer1/runtimes/_generated_types.py +42 -0
- clm/layer1/runtimes/base.py +265 -0
- clm/layer1/runtimes/binary.py +101 -0
- clm/layer1/runtimes/factory.py +203 -0
- clm/layer1/runtimes/javascript.py +321 -0
- clm/layer1/runtimes/javascript_runtime.js +33 -0
- clm/layer1/runtimes/js_scripts/isomorphism.mjs +22 -0
- clm/layer1/runtimes/js_scripts/loader.mjs +52 -0
- clm/layer1/runtimes/lambda_calc.py +126 -0
- clm/layer1/runtimes/python.py +288 -0
- clm/layer1/runtimes/python_runtime.py +41 -0
- clm/layer1/runtimes/script.py +114 -0
- clm/layer1/sandbox.py +334 -0
- clm/layer1/savepoint_guard.py +136 -0
- clm/layer1/sparse_net.py +122 -0
- clm/layer1/spec_tester.py +165 -0
- clm/layer1/store.py +295 -0
- clm/layer1/test_runner.py +498 -0
- clm/layer1/type_inspector.py +166 -0
- clm/layer1/verifier.py +772 -0
- clm/layer1/vm.py +91 -0
- clm/layer2/__init__.py +78 -0
- clm/layer2/card_collection.py +678 -0
- clm/layer2/classifier.py +479 -0
- clm/layer2/engine/__init__.py +1 -0
- clm/layer2/engine/abstract_sql_engine.py +320 -0
- clm/layer2/engine/base.py +193 -0
- clm/layer2/engine/duckdb_engine.py +576 -0
- clm/layer2/engine/sqlite_engine.py +518 -0
- clm/layer2/fiber/__init__.py +31 -0
- clm/layer2/fiber/context.py +89 -0
- clm/layer2/fiber/disposable.py +163 -0
- clm/layer2/fiber/fiber.py +125 -0
- clm/layer2/fiber/hoare.py +191 -0
- clm/layer2/file_io.py +376 -0
- clm/layer2/file_registrar.py +77 -0
- clm/layer2/ingest.py +202 -0
- clm/layer2/lineage.py +322 -0
- clm/layer2/mcard_fs.py +262 -0
- clm/layer2/merkle.py +11 -0
- clm/layer2/mime.py +244 -0
- clm/layer2/mime_detector.py +245 -0
- clm/layer2/private_collection.py +118 -0
- clm/layer2/shared_store.py +11 -0
- clm/layer2/storage/__init__.py +190 -0
- clm/layer2/storage/db_bitemporal.py +141 -0
- clm/layer2/storage/db_driver.py +331 -0
- clm/layer2/storage/db_hyperlinks.py +214 -0
- clm/layer2/storage/db_pool.py +139 -0
- clm/layer2/storage/db_repository.py +229 -0
- clm/layer2/storage/savepoint.py +78 -0
- clm/layer2/storage/sqlite_wal.py +74 -0
- clm/layer2/tridb.py +245 -0
- clm/layer2/vcard_receipt.py +71 -0
- clm/layer2/vfs.py +17 -0
- clm/layer3/__init__.py +141 -0
- clm/layer3/agency/__init__.py +19 -0
- clm/layer3/agency/config.py +220 -0
- clm/layer3/agency/providers/__init__.py +15 -0
- clm/layer3/agency/providers/base.py +91 -0
- clm/layer3/agency/providers/mlc_llm.py +169 -0
- clm/layer3/agency/providers/ollama.py +203 -0
- clm/layer3/agency/router.py +71 -0
- clm/layer3/agency/runtime.py +524 -0
- clm/layer3/gateway.py +353 -0
- clm/layer3/open_interpreter_adapter.py +159 -0
- clm/layer3/rag/__init__.py +70 -0
- clm/layer3/rag/cli.py +228 -0
- clm/layer3/rag/config.py +153 -0
- clm/layer3/rag/embeddings/__init__.py +16 -0
- clm/layer3/rag/embeddings/base.py +79 -0
- clm/layer3/rag/embeddings/ollama.py +211 -0
- clm/layer3/rag/embeddings/vision.py +290 -0
- clm/layer3/rag/engine.py +276 -0
- clm/layer3/rag/graph/__init__.py +22 -0
- clm/layer3/rag/graph/community.py +198 -0
- clm/layer3/rag/graph/engine.py +433 -0
- clm/layer3/rag/graph/extractor.py +255 -0
- clm/layer3/rag/graph/schema.py +115 -0
- clm/layer3/rag/graph/store.py +641 -0
- clm/layer3/rag/indexer.py +333 -0
- clm/layer3/rag/llm_providers.json +52 -0
- clm/layer3/rag/semantic_versioning.py +313 -0
- clm/layer3/rag/vector/__init__.py +28 -0
- clm/layer3/rag/vector/handle_vector_store.py +749 -0
- clm/layer3/rag/vector/schema.py +151 -0
- clm/layer3/rag/vector/store.py +632 -0
- clm/layer3/reticulum/__init__.py +55 -0
- clm/layer3/reticulum/bridge.py +74 -0
- clm/layer3/reticulum/daemon.py +88 -0
- clm/layer3/reticulum/discovery.py +164 -0
- clm/layer3/reticulum/identity.py +107 -0
- clm/layer3/reticulum/media.py +148 -0
- clm/layer3/reticulum/transport.py +133 -0
- clm/layer3/satori/__init__.py +209 -0
- clm/layer3/satori/alpha_conversion.py +168 -0
- clm/layer3/satori/beta_reduction.py +359 -0
- clm/layer3/satori/continuation.py +38 -0
- clm/layer3/satori/eta_conversion.py +205 -0
- clm/layer3/satori/free_variables.py +163 -0
- clm/layer3/satori/io_effects.py +370 -0
- clm/layer3/satori/lambda_runtime.py +726 -0
- clm/layer3/satori/lambda_term.py +230 -0
- clm/layer3/satori/sheaf_audit.py +261 -0
- clm/layer3/satori/speech_act.py +279 -0
- clm/layer4/README.md +9 -0
- clm/layer4/__init__.py +67 -0
- clm/layer4/_sovereign_executor.py +580 -0
- clm/layer4/action_commands.py +719 -0
- clm/layer4/api.py +236 -0
- clm/layer4/bootstrap/__init__.py +13 -0
- clm/layer4/bootstrap/bootstrapper.py +118 -0
- clm/layer4/bootstrap/cleanup.py +44 -0
- clm/layer4/bootstrap/clm_hooks.py +49 -0
- clm/layer4/bootstrap/genesis.py +77 -0
- clm/layer4/bootstrap/injection.py +125 -0
- clm/layer4/cli.py +549 -0
- clm/layer4/collection_manager.py +553 -0
- clm/layer4/commands/__init__.py +81 -0
- clm/layer4/commands/action_cmd.py +42 -0
- clm/layer4/commands/check_cmd.py +82 -0
- clm/layer4/commands/envelope.py +73 -0
- clm/layer4/commands/evaluate_cmd.py +224 -0
- clm/layer4/commands/mcard_cmd.py +24 -0
- clm/layer4/commands/petri_cmd.py +92 -0
- clm/layer4/commands/tridb_cmd.py +319 -0
- clm/layer4/commands/version_cmd.py +27 -0
- clm/layer4/diff_commands.py +167 -0
- clm/layer4/eoa_sponsor.py +232 -0
- clm/layer4/events.py +69 -0
- clm/layer4/flux_gateway.py +133 -0
- clm/layer4/gateway.py +91 -0
- clm/layer4/loader.py +307 -0
- clm/layer4/mcard_commands.py +166 -0
- clm/layer4/membrane.py +273 -0
- clm/layer4/navigation.py +50 -0
- clm/layer4/protocol/protocol_entry_python.py +108 -0
- clm/layer4/protocol/python_sync.py +126 -0
- clm/layer4/runner.py +150 -0
- clm/layer4/satori.py +262 -0
- clm/layer4/server/python_sync_client.py +150 -0
- clm/layer4/server/python_sync_server.py +139 -0
- clm/layer4/server/service_api.py +315 -0
- clm/layer4/server/start_servers.py +107 -0
- clm/layer4/server/stop_servers.py +24 -0
- clm/layer4/simulated_optimization_fix.py +0 -0
- clm/layer4/storage.py +463 -0
- clm/layer4/tridb_admin.py +268 -0
- clm/layer4/websocket/test_ws_client.py +104 -0
- clm/layer4/websocket/test_ws_server.py +138 -0
- clm/layer4/websocket/ws_server_local.py +160 -0
- clm/layer5/__init__.py +51 -0
- clm/layer5/async_batch_writer.py +133 -0
- clm/layer5/learning.py +276 -0
- clm/layer5/telemetry_hook.py +107 -0
- clm/layer5/type_lattice.py +238 -0
- clm/py.typed +0 -0
- clm/schemas/__init__.py +652 -0
- clm/schemas/abstract_face_schema.yaml +77 -0
- clm/schemas/cel_schema.yaml +416 -0
- clm/schemas/lifecycle/__init__.py +45 -0
- clm/schemas/lifecycle/models.py +123 -0
- clm/schemas/lifecycle/service.py +364 -0
- clm/schemas/mcard_schema.sql +192 -0
- clm/schemas/mcard_vector_schema.sql +283 -0
- clm/schemas/mlp-context-v1.json +91 -0
- clm_kernel-0.0.0.dist-info/METADATA +121 -0
- clm_kernel-0.0.0.dist-info/RECORD +333 -0
- clm_kernel-0.0.0.dist-info/WHEEL +4 -0
- clm_kernel-0.0.0.dist-info/entry_points.txt +3 -0
- clm_kernel-0.0.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# mcard/model/schema.py
|
|
2
|
+
"""
|
|
3
|
+
DEPRECATED: This module is a backward-compatibility shim.
|
|
4
|
+
|
|
5
|
+
All schemas are now loaded from the singleton:
|
|
6
|
+
from clm.schemas import MCardSchema
|
|
7
|
+
schema = MCardSchema.get_instance()
|
|
8
|
+
|
|
9
|
+
The single source of truth is: schema/mcard_schema.sql
|
|
10
|
+
|
|
11
|
+
WHY THIS FILE EXISTS:
|
|
12
|
+
---------------------
|
|
13
|
+
This file exists only for backward compatibility with legacy imports like:
|
|
14
|
+
from clm.layer0.model.schema import CARD_TABLE_SCHEMA
|
|
15
|
+
|
|
16
|
+
New code should import directly from clm.schemas:
|
|
17
|
+
from clm.schemas import MCardSchema, get_schema
|
|
18
|
+
|
|
19
|
+
This file will be removed in a future version.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from clm.schemas import (
|
|
23
|
+
CARD_TABLE_SCHEMA,
|
|
24
|
+
HANDLE_HISTORY_SCHEMA,
|
|
25
|
+
HANDLE_INDEX_SCHEMA,
|
|
26
|
+
HANDLE_REGISTRY_SCHEMA,
|
|
27
|
+
MCardSchema,
|
|
28
|
+
get_schema,
|
|
29
|
+
init_all_tables,
|
|
30
|
+
init_core_tables,
|
|
31
|
+
init_handle_tables,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
__all__ = [
|
|
35
|
+
"CARD_TABLE_SCHEMA",
|
|
36
|
+
"HANDLE_REGISTRY_SCHEMA",
|
|
37
|
+
"HANDLE_HISTORY_SCHEMA",
|
|
38
|
+
"HANDLE_INDEX_SCHEMA",
|
|
39
|
+
"MCardSchema",
|
|
40
|
+
"get_schema",
|
|
41
|
+
"init_all_tables",
|
|
42
|
+
"init_core_tables",
|
|
43
|
+
"init_handle_tables",
|
|
44
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Content analysis utilities
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Content analysis utilities for detecting problematic content patterns.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
# Constants for sampling and limits
|
|
6
|
+
MAX_SAMPLE_BYTES = 64 * 1024 # 64KB
|
|
7
|
+
MAX_SAMPLE_LINES = 100
|
|
8
|
+
|
|
9
|
+
# Heuristic thresholds for problematic content detection
|
|
10
|
+
_EXT_LONG_LINE_KNOWN = {
|
|
11
|
+
".js",
|
|
12
|
+
".mjs",
|
|
13
|
+
".min.js",
|
|
14
|
+
".bundle.js",
|
|
15
|
+
".css",
|
|
16
|
+
".min.css",
|
|
17
|
+
".json",
|
|
18
|
+
".jsonl",
|
|
19
|
+
".xml",
|
|
20
|
+
".svg",
|
|
21
|
+
".html",
|
|
22
|
+
".htm",
|
|
23
|
+
".map", # source maps
|
|
24
|
+
".wasm",
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
_PROBLEMATIC_MAX_KNOWN_TYPE = 1024 * 1024 # 1MB
|
|
28
|
+
_PROBLEMATIC_SAMPLE_SIZE = 32 * 1024 # 32KB
|
|
29
|
+
_PROBLEMATIC_MIN_MULTI_POSITION = 100 * 1024 # 100KB
|
|
30
|
+
_PATHOLOGICAL_NO_BREAK_MIN = 8192
|
|
31
|
+
_PATHOLOGICAL_NO_BREAK_STRICT_MIN = 32 * 1024
|
|
32
|
+
_NULL_RATIO_THRESHOLD = 0.1
|
|
33
|
+
_CTRL_RATIO_THRESHOLD = 0.2
|
|
34
|
+
_MAX_LINE_LENGTH_KNOWN = 100000
|
|
35
|
+
_MAX_LINE_LENGTH_DEFAULT = 50000
|
|
36
|
+
_AVG_LINE_LENGTH_KNOWN = 20000
|
|
37
|
+
_AVG_LINE_LENGTH_DEFAULT = 5000
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class ContentAnalyzer:
|
|
41
|
+
"""Utility class for analyzing content patterns and detecting problematic content."""
|
|
42
|
+
|
|
43
|
+
@staticmethod
|
|
44
|
+
def is_known_long_line_extension(file_extension: str | None) -> bool:
|
|
45
|
+
"""Return True if extension is typically legitimate with long lines."""
|
|
46
|
+
if not file_extension:
|
|
47
|
+
return False
|
|
48
|
+
return file_extension.lower() in _EXT_LONG_LINE_KNOWN
|
|
49
|
+
|
|
50
|
+
@staticmethod
|
|
51
|
+
def is_unstructured_binary(sample: bytes) -> bool:
|
|
52
|
+
"""Heuristic check for unstructured binary content."""
|
|
53
|
+
if len(sample) < 512:
|
|
54
|
+
return False
|
|
55
|
+
null_count = sample.count(b"\x00")
|
|
56
|
+
control_chars = sum(1 for b in sample if b < 32 and b not in (9, 10, 13))
|
|
57
|
+
null_ratio = null_count / len(sample)
|
|
58
|
+
control_ratio = control_chars / len(sample)
|
|
59
|
+
return (
|
|
60
|
+
null_ratio > _NULL_RATIO_THRESHOLD or control_ratio > _CTRL_RATIO_THRESHOLD
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
@staticmethod
|
|
64
|
+
def has_pathological_lines(sample: bytes, is_known_type: bool) -> bool:
|
|
65
|
+
"""Detect pathological line structures (extremely long or no breaks)."""
|
|
66
|
+
max_line_len = (
|
|
67
|
+
_MAX_LINE_LENGTH_KNOWN if is_known_type else _MAX_LINE_LENGTH_DEFAULT
|
|
68
|
+
)
|
|
69
|
+
max_avg_len = (
|
|
70
|
+
_AVG_LINE_LENGTH_KNOWN if is_known_type else _AVG_LINE_LENGTH_DEFAULT
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
if (
|
|
74
|
+
b"\n" not in sample
|
|
75
|
+
and b"\r" not in sample
|
|
76
|
+
and len(sample) >= _PATHOLOGICAL_NO_BREAK_MIN
|
|
77
|
+
):
|
|
78
|
+
if (not is_known_type) or len(sample) >= _PATHOLOGICAL_NO_BREAK_STRICT_MIN:
|
|
79
|
+
return True
|
|
80
|
+
|
|
81
|
+
lines = sample.replace(b"\r\n", b"\n").replace(b"\r", b"\n").split(b"\n")
|
|
82
|
+
if not lines:
|
|
83
|
+
return False
|
|
84
|
+
|
|
85
|
+
if any(len(line) > max_line_len for line in lines):
|
|
86
|
+
return True
|
|
87
|
+
|
|
88
|
+
if len(lines) > 1:
|
|
89
|
+
avg_len = sum(len(line) for line in lines) / len(lines)
|
|
90
|
+
if avg_len > max_avg_len:
|
|
91
|
+
return True
|
|
92
|
+
|
|
93
|
+
return False
|
|
94
|
+
|
|
95
|
+
@staticmethod
|
|
96
|
+
def is_problematic_bytes(content: bytes, file_extension: str | None = None) -> bool:
|
|
97
|
+
"""
|
|
98
|
+
Byte-content heuristic for detecting problematic content patterns.
|
|
99
|
+
|
|
100
|
+
Args:
|
|
101
|
+
content: The byte content to analyze
|
|
102
|
+
file_extension: Optional file extension for type-aware thresholds
|
|
103
|
+
|
|
104
|
+
Returns:
|
|
105
|
+
True if content appears problematic
|
|
106
|
+
"""
|
|
107
|
+
if not content:
|
|
108
|
+
return False
|
|
109
|
+
|
|
110
|
+
is_known_type = ContentAnalyzer.is_known_long_line_extension(file_extension)
|
|
111
|
+
sample = content[:_PROBLEMATIC_SAMPLE_SIZE]
|
|
112
|
+
|
|
113
|
+
if ContentAnalyzer.is_unstructured_binary(sample):
|
|
114
|
+
return True
|
|
115
|
+
|
|
116
|
+
if ContentAnalyzer.has_pathological_lines(sample, is_known_type):
|
|
117
|
+
return True
|
|
118
|
+
|
|
119
|
+
return False
|
|
120
|
+
|
|
121
|
+
@staticmethod
|
|
122
|
+
def prepare_content_sample(
|
|
123
|
+
content: bytes,
|
|
124
|
+
max_bytes: int = MAX_SAMPLE_BYTES,
|
|
125
|
+
max_lines: int = MAX_SAMPLE_LINES,
|
|
126
|
+
) -> tuple[str, list[str], str]:
|
|
127
|
+
"""
|
|
128
|
+
Prepare content sample for analysis.
|
|
129
|
+
|
|
130
|
+
Truncates raw bytes *before* decoding to avoid O(n) decode on
|
|
131
|
+
multi-MB inputs. This matches the TypeScript DetectionStrategy
|
|
132
|
+
which caps the analytical sample to 8 192 bytes.
|
|
133
|
+
|
|
134
|
+
Args:
|
|
135
|
+
content: Raw byte content
|
|
136
|
+
max_bytes: Maximum bytes to sample
|
|
137
|
+
max_lines: Maximum lines to process
|
|
138
|
+
|
|
139
|
+
Returns:
|
|
140
|
+
Tuple of (content_sample, lines, first_line)
|
|
141
|
+
"""
|
|
142
|
+
# Cap bytes BEFORE decoding — avoids decoding multi-MB blobs.
|
|
143
|
+
sample_bytes = content[:max_bytes]
|
|
144
|
+
try:
|
|
145
|
+
content_sample = sample_bytes.decode("utf-8", errors="replace")
|
|
146
|
+
except Exception:
|
|
147
|
+
content_sample = str(sample_bytes, errors="replace")
|
|
148
|
+
|
|
149
|
+
lines = content_sample.split("\n")[:max_lines]
|
|
150
|
+
first_line = lines[0] if lines else ""
|
|
151
|
+
|
|
152
|
+
return content_sample, lines, first_line
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
# Content validation modules
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Base validator interface for content validation.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from abc import ABC, abstractmethod
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class ValidationError(Exception):
|
|
9
|
+
"""Exception raised for validation errors."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, message):
|
|
12
|
+
self.message = message
|
|
13
|
+
super().__init__(self.message)
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class BaseValidator(ABC):
|
|
17
|
+
"""Abstract base class for content validators."""
|
|
18
|
+
|
|
19
|
+
@abstractmethod
|
|
20
|
+
def validate(self, content: str | bytes, mime_type: str) -> None:
|
|
21
|
+
"""
|
|
22
|
+
Validate content for a specific MIME type.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
content: The content to validate
|
|
26
|
+
mime_type: The detected MIME type
|
|
27
|
+
|
|
28
|
+
Raises:
|
|
29
|
+
ValidationError: If content is invalid
|
|
30
|
+
"""
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
@abstractmethod
|
|
34
|
+
def can_validate(self, mime_type: str) -> bool:
|
|
35
|
+
"""
|
|
36
|
+
Check if this validator can handle the given MIME type.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
mime_type: The MIME type to check
|
|
40
|
+
|
|
41
|
+
Returns:
|
|
42
|
+
True if this validator can handle the MIME type
|
|
43
|
+
"""
|
|
44
|
+
pass
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Binary content validator.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .base_validator import BaseValidator, ValidationError
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BinaryValidator(BaseValidator):
|
|
9
|
+
"""Validator for binary content types."""
|
|
10
|
+
|
|
11
|
+
BINARY_MIME_TYPES = {
|
|
12
|
+
"image/png",
|
|
13
|
+
"image/jpeg",
|
|
14
|
+
"image/gif",
|
|
15
|
+
"image/bmp",
|
|
16
|
+
"application/pdf",
|
|
17
|
+
"application/zip",
|
|
18
|
+
"video/mp4",
|
|
19
|
+
"audio/wav",
|
|
20
|
+
"application/octet-stream",
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
def can_validate(self, mime_type: str) -> bool:
|
|
24
|
+
return mime_type in self.BINARY_MIME_TYPES or mime_type.startswith(
|
|
25
|
+
("image/", "audio/", "video/")
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
def validate(self, content: str | bytes, mime_type: str) -> None:
|
|
29
|
+
"""Validate binary content based on MIME type."""
|
|
30
|
+
if isinstance(content, str):
|
|
31
|
+
content = content.encode("utf-8")
|
|
32
|
+
|
|
33
|
+
if mime_type.startswith("image/"):
|
|
34
|
+
self._validate_image(content, mime_type)
|
|
35
|
+
elif mime_type == "application/pdf":
|
|
36
|
+
self._validate_pdf(content)
|
|
37
|
+
elif mime_type == "application/zip":
|
|
38
|
+
self._validate_zip(content)
|
|
39
|
+
|
|
40
|
+
def _validate_image(self, content: bytes, mime_type: str) -> None:
|
|
41
|
+
"""Validate image content."""
|
|
42
|
+
if mime_type == "image/png" and len(content) <= 8:
|
|
43
|
+
raise ValidationError("Invalid PNG content: truncated file")
|
|
44
|
+
elif mime_type == "image/jpeg" and len(content) <= 3:
|
|
45
|
+
raise ValidationError("Invalid JPEG content: truncated file")
|
|
46
|
+
elif mime_type == "image/gif" and len(content) <= 6:
|
|
47
|
+
raise ValidationError("Invalid GIF content: truncated file")
|
|
48
|
+
|
|
49
|
+
# Check for proper signature
|
|
50
|
+
signatures = {
|
|
51
|
+
"image/png": b"\x89PNG\r\n\x1a\n",
|
|
52
|
+
"image/jpeg": b"\xff\xd8\xff",
|
|
53
|
+
"image/gif": b"GIF8",
|
|
54
|
+
}
|
|
55
|
+
if mime_type in signatures and not content.startswith(signatures[mime_type]):
|
|
56
|
+
raise ValidationError(f"Invalid {mime_type} content: missing proper header")
|
|
57
|
+
|
|
58
|
+
def _validate_pdf(self, content: bytes) -> None:
|
|
59
|
+
"""Validate PDF content."""
|
|
60
|
+
if not content.startswith(b"%PDF-"):
|
|
61
|
+
raise ValidationError("Invalid PDF content")
|
|
62
|
+
|
|
63
|
+
def _validate_zip(self, content: bytes) -> None:
|
|
64
|
+
"""Validate ZIP content."""
|
|
65
|
+
if len(content) <= 4:
|
|
66
|
+
raise ValidationError("Invalid ZIP content")
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Text content validator.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
from defusedxml.ElementTree import ParseError
|
|
9
|
+
from defusedxml.ElementTree import fromstring as safe_fromstring
|
|
10
|
+
except ImportError:
|
|
11
|
+
from xml.etree.ElementTree import ParseError
|
|
12
|
+
from xml.etree.ElementTree import fromstring as safe_fromstring # nosec B314
|
|
13
|
+
|
|
14
|
+
from .base_validator import BaseValidator, ValidationError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class TextValidator(BaseValidator):
|
|
18
|
+
"""Validator for text-based content types."""
|
|
19
|
+
|
|
20
|
+
TEXT_MIME_TYPES = {
|
|
21
|
+
"text/plain",
|
|
22
|
+
"application/json",
|
|
23
|
+
"application/xml",
|
|
24
|
+
"text/xml",
|
|
25
|
+
"image/svg+xml",
|
|
26
|
+
"text/html",
|
|
27
|
+
"text/markdown",
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
def can_validate(self, mime_type: str) -> bool:
|
|
31
|
+
return mime_type in self.TEXT_MIME_TYPES
|
|
32
|
+
|
|
33
|
+
def validate(self, content: str | bytes, mime_type: str) -> None:
|
|
34
|
+
"""Validate text content based on MIME type."""
|
|
35
|
+
if isinstance(content, bytes):
|
|
36
|
+
try:
|
|
37
|
+
text_content = content.decode("utf-8")
|
|
38
|
+
except UnicodeDecodeError as e:
|
|
39
|
+
raise ValidationError("Invalid content: not valid UTF-8") from e
|
|
40
|
+
else:
|
|
41
|
+
text_content = content
|
|
42
|
+
|
|
43
|
+
if mime_type == "text/plain":
|
|
44
|
+
self._validate_plain_text(text_content)
|
|
45
|
+
elif mime_type == "application/json":
|
|
46
|
+
self._validate_json(text_content)
|
|
47
|
+
elif mime_type in ("application/xml", "text/xml", "image/svg+xml"):
|
|
48
|
+
self._validate_xml(text_content)
|
|
49
|
+
|
|
50
|
+
def _validate_plain_text(self, content: str) -> None:
|
|
51
|
+
"""Validate plain text content."""
|
|
52
|
+
if not content.strip():
|
|
53
|
+
raise ValidationError("Invalid content: empty text")
|
|
54
|
+
if len(content.strip()) < 3:
|
|
55
|
+
raise ValidationError("Invalid content: too short")
|
|
56
|
+
if all(ord(c) < 32 for c in content.strip()):
|
|
57
|
+
raise ValidationError("Invalid content: contains only control characters")
|
|
58
|
+
# Heuristic: valid plain text should contain some whitespace
|
|
59
|
+
if not any(c.isspace() for c in content):
|
|
60
|
+
# Check for multi-word content without spaces
|
|
61
|
+
if (
|
|
62
|
+
len(content.split()) == 1 and len(content) > 20
|
|
63
|
+
): # Arbitrary length check
|
|
64
|
+
raise ValidationError("Invalid content: likely not plain text")
|
|
65
|
+
|
|
66
|
+
def _validate_json(self, content: str) -> None:
|
|
67
|
+
"""Validate JSON content."""
|
|
68
|
+
try:
|
|
69
|
+
# Check for comments before attempting to parse
|
|
70
|
+
lines = content.split("\n")
|
|
71
|
+
if any(line.strip().startswith("//") for line in lines):
|
|
72
|
+
raise ValidationError("Invalid JSON content: contains comments")
|
|
73
|
+
json.loads(content)
|
|
74
|
+
except json.JSONDecodeError as e:
|
|
75
|
+
raise ValidationError("Invalid JSON content") from e
|
|
76
|
+
|
|
77
|
+
def _validate_xml(self, content: str) -> None:
|
|
78
|
+
"""Validate XML content."""
|
|
79
|
+
try:
|
|
80
|
+
safe_fromstring(content) # nosec B314
|
|
81
|
+
except ParseError as e:
|
|
82
|
+
raise ValidationError("Invalid XML content") from e
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Central registry for content validators.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from .base_validator import BaseValidator, ValidationError
|
|
6
|
+
from .binary_validator import BinaryValidator
|
|
7
|
+
from .text_validator import TextValidator
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class ValidationRegistry:
|
|
11
|
+
"""Central registry for managing content validators."""
|
|
12
|
+
|
|
13
|
+
def __init__(self):
|
|
14
|
+
self.validators: list[BaseValidator] = [
|
|
15
|
+
TextValidator(),
|
|
16
|
+
BinaryValidator(),
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
def validate(self, content: str | bytes, mime_type: str) -> None:
|
|
20
|
+
"""
|
|
21
|
+
Validate content using appropriate validator.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
content: The content to validate
|
|
25
|
+
mime_type: The detected MIME type
|
|
26
|
+
|
|
27
|
+
Raises:
|
|
28
|
+
ValidationError: If content is invalid
|
|
29
|
+
"""
|
|
30
|
+
if not content:
|
|
31
|
+
raise ValidationError("Empty content")
|
|
32
|
+
|
|
33
|
+
# Find appropriate validator
|
|
34
|
+
for validator in self.validators:
|
|
35
|
+
if validator.can_validate(mime_type):
|
|
36
|
+
validator.validate(content, mime_type)
|
|
37
|
+
return
|
|
38
|
+
|
|
39
|
+
# If no specific validator found, do basic validation
|
|
40
|
+
self._basic_validation(content)
|
|
41
|
+
|
|
42
|
+
def _basic_validation(self, content: str | bytes) -> None:
|
|
43
|
+
"""Basic validation for unknown content types."""
|
|
44
|
+
if isinstance(content, bytes) and not content.strip():
|
|
45
|
+
raise ValidationError("Invalid content: empty byte array")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# Global registry instance
|
|
49
|
+
validation_registry = ValidationRegistry()
|