indw 1.0__py3-none-any.whl

app/cli.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import argparse
+import sys
+
+from app.commands import (
+    register_audit,
+    register_benchmark,
+    register_doctor,
+    register_merge,
+    register_test,
+    register_validate,
+)
+
+
+def build_parser() -> argparse.ArgumentParser:
+    ap = argparse.ArgumentParser(prog="indw", description="INDW — Instant Data Workflow")
+    sub = ap.add_subparsers(dest="command", required=True)
+    register_merge(sub)
+    register_test(sub)
+    register_validate(sub)
+    register_audit(sub)
+    register_benchmark(sub)
+    register_doctor(sub)
+    return ap
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    return int(args._handler(args))
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

app/commands/__init__.py

@@ -0,0 +1,16 @@
+from app.commands.audit import register as register_audit
+from app.commands.benchmark import register as register_benchmark
+from app.commands.doctor import register as register_doctor
+from app.commands.merge import register as register_merge
+from app.commands.test import register as register_test
+
+from app.commands.validate import register as register_validate
+
+__all__ = [
+    "register_audit",
+    "register_benchmark",
+    "register_doctor",
+    "register_merge",
+    "register_test",
+    "register_validate",
+]

app/commands/audit.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser("audit", help="run pipeline audit reports")
+    p.add_argument(
+        "--kind",
+        choices=("pipeline", "dask", "production", "library", "stage0"),
+        default="pipeline",
+    )
+    p.add_argument("--work-dir", type=Path, default=None)
+    p.add_argument("--workers", type=int, default=4)
+    p.set_defaults(_handler=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    from app.workflows import run_audit
+
+    return run_audit(kind=args.kind, work_dir=args.work_dir, workers=args.workers)

app/commands/benchmark.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+import argparse
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser("benchmark", help="production scale benchmark")
+    p.add_argument("--workers", default="1 2 4", help="worker counts")
+    p.set_defaults(_handler=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    from app.workflows import run_benchmark
+
+    return run_benchmark(workers=args.workers)

app/commands/doctor.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import platform
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser("doctor", help="check install and backend availability")
+    p.set_defaults(_handler=run)
+
+
+def run(_args: argparse.Namespace) -> int:
+    import indw
+    from indw.schedule.backends.config import pipeline_execution_backend
+    from indw.schedule.backends.factory import resolve_execution_backend
+
+    print(f"indw={indw.__version__} python={platform.python_version()} platform={platform.platform()}")
+    print(f"backend={pipeline_execution_backend()} resolved={resolve_execution_backend().name}")
+    for pkg in ("orjson", "trafilatura", "dask"):
+        print(f"{pkg}={'ok' if importlib.util.find_spec(pkg) else 'missing'}")
+    return 0

app/commands/merge.py

@@ -0,0 +1,43 @@
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser("merge", help="run quality merge on raw corpus")
+    p.add_argument("raw_dir", type=Path)
+    p.add_argument("out_path", type=Path)
+    p.add_argument("--work-dir", type=Path, default=None)
+    p.add_argument("--workers", type=int, default=1)
+    p.add_argument("--chunk-size", type=int, default=500)
+    p.add_argument("--fresh", action="store_true")
+    p.add_argument(
+        "--backend",
+        choices=("local", "thread", "multiprocess", "dask"),
+        default=None,
+        help="execution backend (INSTANT_PIPELINE_BACKEND)",
+    )
+    p.set_defaults(_handler=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    from indw.filter.spec.quality import QualityPipelineConfig
+    from indw.schedule.core import merge_with_quality
+
+    if args.backend:
+        os.environ["INSTANT_PIPELINE_BACKEND"] = args.backend
+    os.environ.setdefault("INSTANT_MERGE_HW_PROBE", "0")
+    cfg = QualityPipelineConfig()
+    merge_with_quality(
+        args.raw_dir,
+        args.out_path,
+        quality_config=cfg,
+        work_dir=args.work_dir,
+        fresh=args.fresh,
+        resume=not args.fresh,
+        workers=args.workers,
+        chunk_size=args.chunk_size,
+    )
+    return 0

app/commands/test.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+import argparse
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser("test", help="run framework test suite")
+    p.add_argument(
+        "--profile",
+        choices=("unit", "critical", "parity", "integration", "smoke"),
+        default="unit",
+    )
+    p.add_argument("pytest_args", nargs="*", help="extra pytest arguments")
+    p.set_defaults(_handler=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    from app.workflows import run_tests
+
+    return run_tests(args.profile, extra_args=args.pytest_args or None)

app/commands/validate.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+import argparse
+
+
+def register(sub: argparse._SubParsersAction) -> None:
+    p = sub.add_parser("validate", help="run parity and acceptance validation")
+    p.add_argument("pytest_args", nargs="*", help="extra pytest arguments")
+    p.set_defaults(_handler=run)
+
+
+def run(args: argparse.Namespace) -> int:
+    from app.workflows import run_tests
+
+    return run_tests("parity", extra_args=args.pytest_args or None)

app/workflows.py

@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+_ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_profiles() -> dict[str, dict[str, Any]]:
+    return {
+        "unit": {"markers": "not integration and not slow", "parallel": True, "paths": ["tests/"]},
+        "critical": {"markers": "critical and not integration", "parallel": False, "paths": ["tests/subsystems/"]},
+        "parity": {
+            "markers": "integration",
+            "parallel": False,
+            "paths": [
+                "tests/subsystems/test_stage_pool_parity.py",
+                "tests/subsystems/test_parallel_merge_parity.py",
+                "tests/subsystems/test_tier_admission_parity.py",
+                "tests/subsystems/test_execution_backend.py",
+            ],
+        },
+        "integration": {"markers": "integration or slow", "parallel": False, "paths": ["tests/"]},
+        "smoke": {"markers": "smoke", "parallel": False, "paths": ["tests/"]},
+    }
+
+
+def run_tests(profile: str = "unit", *, extra_args: list[str] | None = None) -> int:
+    profiles = test_profiles()
+    if profile not in profiles:
+        raise ValueError(f"unknown profile {profile}; choose from {sorted(profiles)}")
+    spec = profiles[profile]
+    cmd = [sys.executable, "-m", "pytest", *spec["paths"], "-m", spec["markers"], "--tb=short", "--strict-markers"]
+    if spec.get("parallel"):
+        cmd.extend(["-n", "auto", "--dist", "loadfile", "-q"])
+    else:
+        cmd.append("-v")
+    if extra_args:
+        cmd.extend(extra_args)
+    return subprocess.run(cmd, cwd=_ROOT).returncode
+
+
+def run_benchmark(*, workers: str = "1 2 4") -> int:
+    script = _ROOT / "scripts" / "production_scale_audit.py"
+    if not script.is_file():
+        print("benchmark script missing", file=sys.stderr)
+        return 1
+    cmd = [sys.executable, str(script), "--workers", *workers.split()]
+    return subprocess.run(cmd, cwd=_ROOT).returncode
+
+
+def run_audit(*, kind: str = "pipeline", work_dir: Path | None = None, workers: int = 4) -> int:
+    scripts = {
+        "pipeline": ("scripts/pipeline_audit.py", []),
+        "dask": ("scripts/dask_integration_report.py", []),
+        "production": ("scripts/production_scale_audit.py", ["--workers", "1", "2"]),
+        "library": ("scripts/library_migration_report.py", []),
+        "stage0": ("scripts/stage0_production_verify.py", ["--workers", str(workers)]),
+    }
+    rel, extra = scripts.get(kind, scripts["pipeline"])
+    script = _ROOT / rel
+    if not script.is_file():
+        print(f"audit script not found: {kind}", file=sys.stderr)
+        return 1
+    cmd = [sys.executable, str(script), *extra]
+    if work_dir is not None and kind == "pipeline":
+        cmd.extend(["--work-dir", str(work_dir)])
+    return subprocess.run(cmd, cwd=_ROOT).returncode

indw/__init__.py

@@ -0,0 +1,42 @@
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version('indw')
+except PackageNotFoundError:
+    __version__ = '0.0.0'
+
+_LAZY_EXPORTS = {
+    'CorpusRegistry': ('indw.store.corpus.registry', 'CorpusRegistry'),
+    'DatasetDownloader': ('indw.ingest.download', 'DatasetDownloader'),
+    'FastDatasetPipeline': ('indw.ingest.run', 'FastDatasetPipeline'),
+    'setup_dataset_logging': ('indw.ingest.log', 'setup_dataset_logging'),
+    'ScriptProfile': ('indw.filter.language.script', 'ScriptProfile'),
+    'analyze_script_profile': ('indw.filter.language.script', 'analyze_script_profile'),
+    'MultilingualPolicyConfig': ('indw.filter.language.script_policy', 'MultilingualPolicyConfig'),
+    'MixtureOrchestrationConfig': ('indw.schedule.mix.config', 'MixtureOrchestrationConfig'),
+    'CorpusMixturePlan': ('indw.schedule.mix.plan', 'CorpusMixturePlan'),
+    'adapt_mixture_from_telemetry': ('indw.schedule.mix.telemetry', 'adapt_mixture_from_telemetry'),
+    'build_corpus_mixture_plan': ('indw.schedule.mix.mixture_planner', 'build_corpus_mixture_plan'),
+    'QualityPipelineConfig': ('indw.filter.spec.quality', 'QualityPipelineConfig'),
+    'merge_with_quality': ('indw.schedule.core', 'merge_with_quality'),
+    'QualityGate': ('indw.filter.gate.quality', 'QualityGate'),
+    'export_token_bins_fast': ('indw.store.export.fast_export', 'export_token_bins_fast'),
+    'build_pretrain_dataloader': ('indw.store.export.memmap_stream', 'build_pretrain_dataloader'),
+    'build_val_dataloader': ('indw.store.export.memmap_stream', 'build_val_dataloader'),
+}
+
+__all__ = [
+    '__version__',
+    *sorted(_LAZY_EXPORTS),
+]
+
+
+def __getattr__(name: str):
+    if name in _LAZY_EXPORTS:
+        import importlib
+
+        mod_path, attr = _LAZY_EXPORTS[name]
+        val = getattr(importlib.import_module(mod_path), attr)
+        globals()[name] = val
+        return val
+    raise AttributeError(f'module {__name__!r} has no attribute {name!r}')

indw/_compat.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import importlib
+import importlib.abc
+import importlib.util
+import sys
+
+_installed = False
+
+
+class _IndwCompatFinder(importlib.abc.MetaPathFinder):
+    def find_spec(self, fullname, path, target=None):
+        if fullname != "data" and not fullname.startswith("data."):
+            return None
+        indw_name = "indw" if fullname == "data" else "indw" + fullname[4:]
+        if fullname in sys.modules:
+            return importlib.util.spec_from_loader(fullname, loader=None)
+        try:
+            mod = importlib.import_module(indw_name)
+        except ModuleNotFoundError:
+            return None
+        sys.modules[fullname] = mod
+        return importlib.util.spec_from_loader(fullname, loader=None)
+
+
+def install_compat() -> None:
+    global _installed
+    if _installed:
+        return
+    for finder in sys.meta_path:
+        if isinstance(finder, _IndwCompatFinder):
+            _installed = True
+            return
+    sys.meta_path.insert(0, _IndwCompatFinder())
+    _installed = True

indw/clean/__init__.py

@@ -0,0 +1,29 @@
+__all__ = [
+    'CorpusCleaningPipeline',
+    'CleaningResult',
+    'extract_row_text',
+    'final_pass_jsonl_row',
+    'process_jsonl_row',
+    'row_text_key',
+]
+
+_LAZY = {
+    'CorpusCleaningPipeline': ('indw.clean.corpus', 'CorpusCleaningPipeline'),
+    'CleaningResult': ('indw.clean.corpus', 'CleaningResult'),
+    'extract_row_text': ('indw.clean.corpus', 'extract_row_text'),
+    'final_pass_jsonl_row': ('indw.clean.corpus', 'final_pass_jsonl_row'),
+    'process_jsonl_row': ('indw.clean.corpus', 'process_jsonl_row'),
+    'row_text_key': ('indw.clean.corpus', 'row_text_key'),
+}
+
+
+def __getattr__(name: str):
+    if name in _LAZY:
+        module_path, attr = _LAZY[name]
+        import importlib
+
+        mod = importlib.import_module(module_path)
+        val = getattr(mod, attr)
+        globals()[name] = val
+        return val
+    raise AttributeError(f'module {__name__!r} has no attribute {name!r}')

indw/clean/artifact/calibrate.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from indw.clean.artifact.discovery_config import DiscoveryConfig
+from indw.clean.artifact.discovery_corpus import CorpusStatsAccumulator
+from indw.clean.artifact.discovery_registry import DynamicArtifactRegistry
+
+@dataclass
+class ShadowDisagreement:
+    doc_id: str = ''
+    legacy_ratio: float = 0.0
+    discovery_ratio: float = 0.0
+    delta: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            'doc_id': self.doc_id,
+            'legacy_ratio': round(self.legacy_ratio, 4),
+            'discovery_ratio': round(self.discovery_ratio, 4),
+            'delta': round(self.delta, 4),
+        }
+
+@dataclass
+class CalibrationReport:
+    batch_id: int = 0
+    docs_seen: int = 0
+    promoted: int = 0
+    demoted: int = 0
+    registry_size: int = 0
+    shadow_disagreements: list[ShadowDisagreement] = field(default_factory=list)
+    trim_threshold: float = 0.92
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            'batch_id': self.batch_id,
+            'docs_seen': self.docs_seen,
+            'promoted': self.promoted,
+            'demoted': self.demoted,
+            'registry_size': self.registry_size,
+            'trim_threshold': self.trim_threshold,
+            'shadow_disagreements': [d.to_dict() for d in self.shadow_disagreements[-500:]],
+        }
+
+def batch_calibrate(
+    accumulator: CorpusStatsAccumulator,
+    registry: DynamicArtifactRegistry,
+    config: DiscoveryConfig,
+    *,
+    corpus_dir: str = '',
+    shadow_disagreements: list[ShadowDisagreement] | None = None,
+) -> CalibrationReport:
+    accumulator.end_batch(decay=config.decay)
+    cal = registry.calibrate(accumulator)
+    report = CalibrationReport(
+        batch_id=accumulator.batch_id,
+        docs_seen=accumulator.docs_seen,
+        promoted=cal['promoted'],
+        demoted=cal['demoted'],
+        registry_size=cal['total'],
+        shadow_disagreements=shadow_disagreements or [],
+        trim_threshold=config.min_trim_confidence,
+    )
+    if corpus_dir:
+        out = Path(corpus_dir) / 'discovery_calibration.json'
+        out.parent.mkdir(parents=True, exist_ok=True)
+        existing: list[dict] = []
+        if out.exists():
+            try:
+                existing = json.loads(out.read_text(encoding='utf-8'))
+                if not isinstance(existing, list):
+                    existing = [existing]
+            except (json.JSONDecodeError, OSError):
+                existing = []
+        existing.append(report.to_dict())
+        out.write_text(json.dumps(existing[-50:], indent=2), encoding='utf-8')
+    return report

indw/clean/artifact/confidence.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from indw.clean.artifact.discovery_corpus import CorpusStatsAccumulator
+from indw.clean.artifact.decompose import DocumentUnit, position_bin
+from indw.clean.artifact.discovery_registry import ArtifactEntry, DynamicArtifactRegistry
+from indw.clean.artifact.safeguards import is_protected_unit
+from indw.clean.document.value import analyze_content_value
+
+@dataclass
+class FusedConfidence:
+    artifact_confidence: float = 0.0
+    knowledge_confidence: float = 0.0
+    frequency_confidence: float = 0.0
+    position_confidence: float = 0.0
+    structural_confidence: float = 0.0
+    novelty_confidence: float = 0.0
+    repetition_confidence: float = 0.0
+    coverage_confidence: float = 0.0
+    entropy_confidence: float = 0.0
+    trim_tier: str = 'keep'
+    would_trim: bool = False
+
+    def to_dict(self) -> dict[str, float | bool | str]:
+        return {
+            'artifact_confidence': round(self.artifact_confidence, 4),
+            'knowledge_confidence': round(self.knowledge_confidence, 4),
+            'frequency_confidence': round(self.frequency_confidence, 4),
+            'position_confidence': round(self.position_confidence, 4),
+            'structural_confidence': round(self.structural_confidence, 4),
+            'novelty_confidence': round(self.novelty_confidence, 4),
+            'repetition_confidence': round(self.repetition_confidence, 4),
+            'coverage_confidence': round(self.coverage_confidence, 4),
+            'entropy_confidence': round(self.entropy_confidence, 4),
+            'trim_tier': self.trim_tier,
+            'would_trim': self.would_trim,
+        }
+
+class ConfidenceFusion:
+    def __init__(
+        self,
+        *,
+        min_trim_confidence: float = 0.92,
+        medium_trim_confidence: float = 0.72,
+        knowledge_dampen: float = 0.55,
+    ) -> None:
+        self.min_trim_confidence = min_trim_confidence
+        self.medium_trim_confidence = medium_trim_confidence
+        self.knowledge_dampen = knowledge_dampen
+
+    def fuse_unit(
+        self,
+        unit: DocumentUnit,
+        entry: ArtifactEntry | None,
+        *,
+        doc_text: str,
+        count_in_doc: int = 1,
+        doc_len: int = 0,
+    ) -> FusedConfidence:
+        if entry is None:
+            return FusedConfidence()
+
+        if is_protected_unit(unit.text, kind=unit.kind, in_fence=unit.in_fence):
+            return FusedConfidence(knowledge_confidence=1.0, trim_tier='protected')
+
+        ctx_start = max(0, unit.start - 200)
+        ctx_end = min(len(doc_text), unit.end + 200)
+        context = doc_text[ctx_start:ctx_end]
+        cv = analyze_content_value(context)
+        knowledge = cv.overall_value_score
+        if cv.evidence and cv.evidence.preserve:
+            knowledge = max(knowledge, 0.85)
+
+        repetition = 1.0 - min(1.0, 1.0 / max(count_in_doc, 1))
+        coverage = entry.frequency_confidence
+        entropy_signal = 1.0 - entry.novelty_confidence
+
+        artifact = entry.artifact_confidence
+        artifact = min(1.0, artifact * 0.75 + repetition * 0.15 + coverage * 0.10)
+        if knowledge > self.knowledge_dampen:
+            artifact *= max(0.0, 1.0 - (knowledge - self.knowledge_dampen))
+
+        tier = 'keep'
+        would_trim = False
+        if artifact >= self.min_trim_confidence and knowledge < self.knowledge_dampen and entry.novelty_confidence < 0.35:
+            if not unit.in_fence and unit.kind != 'code':
+                tier = 'high'
+                would_trim = True
+        elif artifact >= self.medium_trim_confidence and knowledge < 0.45 and entry.novelty_confidence < 0.25:
+            bin_idx = position_bin(unit.start, max(doc_len, 1))
+            if bin_idx in (0, 4) and unit.kind in ('line', 'header', 'footer'):
+                tier = 'medium'
+                would_trim = True
+
+        return FusedConfidence(
+            artifact_confidence=artifact,
+            knowledge_confidence=knowledge,
+            frequency_confidence=entry.frequency_confidence,
+            position_confidence=entry.position_confidence,
+            structural_confidence=entry.structural_confidence,
+            novelty_confidence=entry.novelty_confidence,
+            repetition_confidence=repetition,
+            coverage_confidence=coverage,
+            entropy_confidence=entropy_signal,
+            trim_tier=tier,
+            would_trim=would_trim,
+        )
+
+    def fuse_document(
+        self,
+        units: list[DocumentUnit],
+        registry: DynamicArtifactRegistry,
+        accumulator: CorpusStatsAccumulator,
+        doc_text: str,
+        *,
+        key_counts: dict[str, int] | None = None,
+    ) -> list[tuple[DocumentUnit, FusedConfidence]]:
+        counts: dict[str, int] = key_counts or {}
+        doc_len = len(doc_text)
+        out: list[tuple[DocumentUnit, FusedConfidence]] = []
+        for unit in units:
+            from indw.clean.artifact.discovery_corpus import fragment_key
+
+            key = fragment_key(unit.text, unit.layout)
+            cnt = counts.get(key, 1)
+            entry = registry.lookup(unit.text, accumulator, layout=unit.layout, count_in_doc=cnt)
+            fused = self.fuse_unit(
+                unit, entry, doc_text=doc_text, count_in_doc=cnt, doc_len=doc_len,
+            )
+            out.append((unit, fused))
+        return out