modwire 11.0.0__tar.gz → 11.0.2__tar.gz

{modwire-11.0.0 → modwire-11.0.2}/.dev/modwire.egg-info/SOURCES.txt

@@ -10,7 +10,6 @@ setup.py
 uv.lock
 .github/workflows/ci.yml
 .github/workflows/release.yml
-docs/target-language-ownership.md
 scripts/check_extractors.py
 scripts/generate_docs.py
 scripts/verify_consumer.py
@@ -143,6 +142,10 @@ src/modwire/cli/initialization/services/initialization_service.py
 src/modwire/cli/pipeline/__init__.py
 src/modwire/cli/pipeline/application.py
 src/modwire/cli/pipeline/domain.py
+src/modwire/cli/pipeline/batching/balanced_planner.py
+src/modwire/cli/pipeline/batching/fixed_planner.py
+src/modwire/cli/pipeline/batching/plan.py
+src/modwire/cli/pipeline/batching/planner.py
 src/modwire/cli/pipeline/models/__init__.py
 src/modwire/cli/pipeline/models/command_request.py
 src/modwire/cli/pipeline/models/extractor_batch_input.py
@@ -168,6 +171,10 @@ src/modwire/cli/pipeline/services/source_manifest_builder.py
 src/modwire/cli/pipeline/services/success.py
 src/modwire/cli/pipeline/services/unknown_files_warning.py
 src/modwire/cli/pipeline/services/unused_exports_insight.py
+src/modwire/cli/pipeline/transport/json_lines_output_reader.py
+src/modwire/cli/pipeline/transport/json_output_reader.py
+src/modwire/cli/pipeline/transport/output_reader.py
+src/modwire/cli/pipeline/transport/source_file_factory.py
 src/modwire/cli/resources/__init__.py
 src/modwire/cli/resources/application.py
 src/modwire/cli/resources/models/__init__.py
@@ -199,16 +206,88 @@ src/modwire/extraction/extractors/models/extraction_request.py
 src/modwire/extraction/extractors/models/extractor_resource.py
 src/modwire/extraction/extractors/models/extractor_resource_set.py
 src/modwire/extraction/extractors/models/extractor_runtime.py
+src/modwire/extraction/extractors/models/parsed_source_batch.py
+src/modwire/extraction/extractors/models/parsed_source_file.py
 src/modwire/extraction/extractors/php/__init__.py
 src/modwire/extraction/extractors/php/extractor.py
 src/modwire/extraction/extractors/python/__init__.py
-src/modwire/extraction/extractors/python/assigned_value_reader.py
-src/modwire/extraction/extractors/python/call_context.py
-src/modwire/extraction/extractors/python/call_reader.py
-src/modwire/extraction/extractors/python/domain.py
-src/modwire/extraction/extractors/python/expression_reference_reader.py
 src/modwire/extraction/extractors/python/extractor.py
-src/modwire/extraction/extractors/python/syntax_parser.py
+src/modwire/extraction/extractors/python/observations/call_candidate.py
+src/modwire/extraction/extractors/python/observations/call_observation.py
+src/modwire/extraction/extractors/python/observations/call_reference.py
+src/modwire/extraction/extractors/python/observations/callable_candidate.py
+src/modwire/extraction/extractors/python/observations/class_candidate.py
+src/modwire/extraction/extractors/python/observations/export_candidate.py
+src/modwire/extraction/extractors/python/observations/function_candidate.py
+src/modwire/extraction/extractors/python/observations/import_candidate.py
+src/modwire/extraction/extractors/python/observations/inheritance_candidate.py
+src/modwire/extraction/extractors/python/observations/property_candidate.py
+src/modwire/extraction/extractors/python/observations/source_context.py
+src/modwire/extraction/extractors/python/observations/source_observation.py
+src/modwire/extraction/extractors/python/observations/value_candidate.py
+src/modwire/extraction/extractors/python/pipeline/source_parser.py
+src/modwire/extraction/extractors/python/semantics/catalog.py
+src/modwire/extraction/extractors/python/semantics/contribution.py
+src/modwire/extraction/extractors/python/semantics/reader.py
+src/modwire/extraction/extractors/python/semantics/callables/classifier.py
+src/modwire/extraction/extractors/python/semantics/callables/constructor_rule.py
+src/modwire/extraction/extractors/python/semantics/callables/function_rule.py
+src/modwire/extraction/extractors/python/semantics/callables/instance_method_rule.py
+src/modwire/extraction/extractors/python/semantics/callables/static_method_rule.py
+src/modwire/extraction/extractors/python/semantics/callables/type_method_rule.py
+src/modwire/extraction/extractors/python/semantics/calls/classifier.py
+src/modwire/extraction/extractors/python/semantics/calls/constructor_target_rule.py
+src/modwire/extraction/extractors/python/semantics/calls/dynamic_target_rule.py
+src/modwire/extraction/extractors/python/semantics/calls/instance_target_rule.py
+src/modwire/extraction/extractors/python/semantics/calls/local_target_rule.py
+src/modwire/extraction/extractors/python/semantics/calls/qualified_target_rule.py
+src/modwire/extraction/extractors/python/semantics/calls/reference_reader.py
+src/modwire/extraction/extractors/python/semantics/calls/unresolved_target_rule.py
+src/modwire/extraction/extractors/python/semantics/classes/abstract_rule.py
+src/modwire/extraction/extractors/python/semantics/classes/classifier.py
+src/modwire/extraction/extractors/python/semantics/classes/concrete_rule.py
+src/modwire/extraction/extractors/python/semantics/contributors/call_contributor.py
+src/modwire/extraction/extractors/python/semantics/contributors/callable_contributor.py
+src/modwire/extraction/extractors/python/semantics/contributors/class_contributor.py
+src/modwire/extraction/extractors/python/semantics/contributors/export_contributor.py
+src/modwire/extraction/extractors/python/semantics/contributors/function_contributor.py
+src/modwire/extraction/extractors/python/semantics/contributors/import_contributor.py
+src/modwire/extraction/extractors/python/semantics/contributors/inheritance_contributor.py
+src/modwire/extraction/extractors/python/semantics/contributors/metric_contributor.py
+src/modwire/extraction/extractors/python/semantics/contributors/property_contributor.py
+src/modwire/extraction/extractors/python/semantics/contributors/value_contributor.py
+src/modwire/extraction/extractors/python/semantics/exports/classifier.py
+src/modwire/extraction/extractors/python/semantics/exports/explicit_rule.py
+src/modwire/extraction/extractors/python/semantics/exports/implicit_rule.py
+src/modwire/extraction/extractors/python/semantics/policies/annotation_reader.py
+src/modwire/extraction/extractors/python/semantics/policies/assigned_value_reader.py
+src/modwire/extraction/extractors/python/semantics/policies/declaration_identity_factory.py
+src/modwire/extraction/extractors/python/semantics/policies/expression_reference_reader.py
+src/modwire/extraction/extractors/python/semantics/policies/import_path_policy.py
+src/modwire/extraction/extractors/python/semantics/policies/module_identity_policy.py
+src/modwire/extraction/extractors/python/semantics/policies/parameter_reader.py
+src/modwire/extraction/extractors/python/semantics/policies/type_alias_classifier.py
+src/modwire/extraction/extractors/python/semantics/policies/visibility_policy.py
+src/modwire/extraction/extractors/python/semantics/properties/annotated_class_rule.py
+src/modwire/extraction/extractors/python/semantics/properties/annotated_instance_rule.py
+src/modwire/extraction/extractors/python/semantics/properties/assigned_class_rule.py
+src/modwire/extraction/extractors/python/semantics/properties/assigned_instance_rule.py
+src/modwire/extraction/extractors/python/semantics/properties/classifier.py
+src/modwire/extraction/extractors/python/source_facts/call_contributor.py
+src/modwire/extraction/extractors/python/source_facts/callable_contributor.py
+src/modwire/extraction/extractors/python/source_facts/class_contributor.py
+src/modwire/extraction/extractors/python/source_facts/contribution.py
+src/modwire/extraction/extractors/python/source_facts/export_contributor.py
+src/modwire/extraction/extractors/python/source_facts/function_contributor.py
+src/modwire/extraction/extractors/python/source_facts/import_contributor.py
+src/modwire/extraction/extractors/python/source_facts/inheritance_contributor.py
+src/modwire/extraction/extractors/python/source_facts/metric_contributor.py
+src/modwire/extraction/extractors/python/source_facts/property_contributor.py
+src/modwire/extraction/extractors/python/source_facts/reader.py
+src/modwire/extraction/extractors/python/source_facts/value_contributor.py
+src/modwire/extraction/extractors/python/traversal/context.py
+src/modwire/extraction/extractors/python/traversal/observation_reader.py
+src/modwire/extraction/extractors/python/traversal/ordered_observation_visitor.py
 src/modwire/extraction/extractors/resources/__init__.py
 src/modwire/extraction/extractors/resources/php/build.php
 src/modwire/extraction/extractors/resources/php/build.py

{modwire-11.0.0 → modwire-11.0.2}/Makefile

@@ -3,6 +3,7 @@ BUILD_DIR := $(DEV_DIR)/build
 DIST_DIR := $(DEV_DIR)/dist
 EGG_INFO_DIR := src/modwire.egg-info
 PROJECT_ROOT := $(CURDIR)
+BIG_PROJECT_ARGS ?=
 
 export PYTHONPYCACHEPREFIX := $(PROJECT_ROOT)/$(DEV_DIR)/cache/python
 export UV_CACHE_DIR := $(PROJECT_ROOT)/$(DEV_DIR)/cache/uv
@@ -26,7 +27,7 @@ test:
 	uv run pytest
 
 big-projects:
-	uv run python -m tests.extraction.big_projects.run
+	uv run python -u -m tests.extraction.big_projects.run $(BIG_PROJECT_ARGS)
 
 scan-benchmark-fixture:
 	uv run python -m tests.extraction.scan_benchmark.prepare --root .dev/benchmarks/scan-project --source-template "$(SCAN_BENCHMARK_SOURCE)"

{modwire-11.0.0 → modwire-11.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: modwire
-Version: 11.0.0
+Version: 11.0.2
 Summary: Code extraction, architecture analysis and the Modwire CLI.
 Author: Tomasz Szpak
 License-Expression: MIT

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/architecture/boundaries/services/no_cycles_flow_analyzer.py

@@ -38,11 +38,11 @@ class NoCyclesFlowAnalyzer(FlowAnalyzerInterface, BaseFlowAnalyzer):
 
     def module_adjacency(self, architecture_map: ArchitectureMap) -> dict[str, set[str]]:
         adjacency: dict[str, set[str]] = {}
-        for dependency in architecture_map.code_map.dependency_edges().all():
-            if dependency.edge.to_id is None:
+        for edge in architecture_map.code_map.cm.dependency_graph.edges:
+            if edge.to_id is None:
                 continue
-            source = self.module_for(architecture_map, dependency.edge.from_id)
-            target = self.module_for(architecture_map, dependency.edge.to_id)
+            source = self.module_for(architecture_map, edge.from_id)
+            target = self.module_for(architecture_map, edge.to_id)
             if not source or not target or source == target:
                 continue
             adjacency.setdefault(source, set()).add(target)

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/architecture/boundaries/services/no_reentry_flow_analyzer.py

@@ -43,10 +43,9 @@ class NoReentryFlowAnalyzer(FlowAnalyzerInterface, BaseFlowAnalyzer):
         return self.dedupe(violations)
 
     def roots(self, architecture_map: ArchitectureMap) -> tuple[str, ...]:
+        graph = architecture_map.code_map.cm.dependency_graph
         roots = tuple(
-            source_id
-            for source_id in architecture_map.code_map.source_ids()
-            if architecture_map.code_map.incoming_dependencies(FileId(source_id)).count() == 0
+            source_id for source_id in architecture_map.code_map.source_ids() if not graph.incoming(FileId(source_id))
         )
         if roots:
             return tuple(sorted(roots))
@@ -66,16 +65,17 @@ class NoReentryFlowAnalyzer(FlowAnalyzerInterface, BaseFlowAnalyzer):
             return
         visited.add(state)
         source_module = self.module_for(architecture_map, source_id)
+        graph = architecture_map.code_map.cm.dependency_graph
         dependencies = sorted(
-            architecture_map.code_map.outgoing_dependencies(FileId(source_id)).all(),
-            key=lambda dependency: (
-                dependency.edge.to_id or "",
-                dependency.edge.specifier,
-                dependency.edge.kind,
+            graph.outgoing(FileId(source_id)),
+            key=lambda edge: (
+                edge.to_id or "",
+                edge.specifier,
+                edge.kind,
             ),
         )
-        for dependency in dependencies:
-            target_id = dependency.edge.to_id
+        for edge in dependencies:
+            target_id = edge.to_id
             if target_id is None:
                 continue
             target_module = self.module_for(architecture_map, target_id)

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/architecture/insights/services/clusters_reporter.py

@@ -26,14 +26,37 @@ class ClustersReporter(InsightReporterInterface):
 
     def collect(self, architecture_map: ArchitectureMap) -> ClustersReport:
         source_ids = architecture_map.code_map.source_ids()
+        tracked_source_ids = set(source_ids)
         file_sets: dict[str, list[str]] = {}
+        cluster_by_source_id: dict[str, str] = {}
         for source_id in source_ids:
-            file_sets.setdefault(self.cluster_name(source_id), []).append(source_id)
+            cluster_name = self.cluster_name(source_id)
+            file_sets.setdefault(cluster_name, []).append(source_id)
+            cluster_by_source_id[source_id] = cluster_name
+
+        incoming_by_cluster = dict.fromkeys(file_sets, 0)
+        outgoing_by_cluster = dict.fromkeys(file_sets, 0)
+        pressure_by_file: dict[str, int] = dict.fromkeys(source_ids, 0)
+        for edge in architecture_map.code_map.cm.dependency_graph.edges:
+            if edge.from_id in tracked_source_ids:
+                pressure_by_file[edge.from_id] += 1
+            if edge.to_id is None or edge.to_id not in tracked_source_ids:
+                continue
+
+            pressure_by_file[edge.to_id] += 1
+            target_cluster = cluster_by_source_id[edge.to_id]
+            source_cluster = cluster_by_source_id.get(edge.from_id)
+            if source_cluster == target_cluster:
+                continue
+            incoming_by_cluster[target_cluster] += 1
+            if source_cluster is not None:
+                outgoing_by_cluster[source_cluster] += 1
+
         clusters: list[ClustersReportItem] = []
         for name, files in sorted(file_sets.items()):
             file_tuple = tuple(sorted(files))
-            incoming_count = self.incoming_count(architecture_map, file_tuple)
-            outgoing_count = self.outgoing_count(architecture_map, file_tuple)
+            incoming_count = incoming_by_cluster[name]
+            outgoing_count = outgoing_by_cluster[name]
             clusters.append(
                 ClustersReportItem(
                     name=name,
@@ -41,7 +64,11 @@ class ClustersReporter(InsightReporterInterface):
                     incoming_count=incoming_count,
                     outgoing_count=outgoing_count,
                     pressure_score=incoming_count + outgoing_count,
-                    top_files=self.top_files(architecture_map, file_tuple),
+                    top_files=tuple(
+                        sorted(files, key=lambda source_id: (-pressure_by_file[source_id], source_id))[
+                            : self.top_file_limit
+                        ]
+                    ),
                 )
             )
         return self.report_type(
@@ -78,7 +105,5 @@ class ClustersReporter(InsightReporterInterface):
         )
 
     def file_pressure(self, architecture_map: ArchitectureMap, source_id: str) -> int:
-        return (
-            architecture_map.code_map.incoming_dependencies(FileId(source_id)).count()
-            + architecture_map.code_map.outgoing_dependencies(FileId(source_id)).count()
-        )
+        graph = architecture_map.code_map.cm.dependency_graph
+        return len(graph.incoming(FileId(source_id))) + len(graph.outgoing(FileId(source_id)))

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/architecture/insights/services/coherence_reporter.py

@@ -21,12 +21,13 @@ class CoherenceReporter(InsightReporterInterface):
 
     def collect(self, architecture_map: ArchitectureMap) -> CoherenceReport:
         source_ids = set(architecture_map.code_map.source_ids())
+        graph = architecture_map.code_map.cm.dependency_graph
         roots: list[str] = []
         leaves: list[str] = []
         isolated: list[str] = []
         for source_id in sorted(source_ids):
-            has_incoming = architecture_map.code_map.incoming_dependencies(FileId(source_id)).count() > 0
-            has_outgoing = architecture_map.code_map.outgoing_dependencies(FileId(source_id)).count() > 0
+            has_incoming = len(graph.incoming(FileId(source_id))) > 0
+            has_outgoing = len(graph.outgoing(FileId(source_id))) > 0
             if not has_incoming:
                 roots.append(source_id)
             if not has_outgoing:
@@ -34,9 +35,7 @@ class CoherenceReporter(InsightReporterInterface):
             if not has_incoming and (not has_outgoing):
                 isolated.append(source_id)
         external_dependencies = {
-            edge_result.edge.specifier
-            for edge_result in architecture_map.code_map.external_dependency_edges().all()
-            if edge_result.edge.resolution == "external"
+            edge.specifier for edge in graph.external_edges(source_ids) if edge.resolution == "external"
         }
         return self.report_type(
             roots=tuple(roots),

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/architecture/insights/services/hotspots_reporter.py

@@ -30,8 +30,9 @@ class HotspotsReporter(InsightReporterInterface):
         return self.report_type(hotspots=hotspots)
 
     def hotspot_for(self, architecture_map: ArchitectureMap, source_id: str) -> HotspotsReportItem:
-        incoming_count = architecture_map.code_map.incoming_dependencies(FileId(source_id)).count()
-        outgoing_count = architecture_map.code_map.outgoing_dependencies(FileId(source_id)).count()
+        graph = architecture_map.code_map.cm.dependency_graph
+        incoming_count = len(graph.incoming(FileId(source_id)))
+        outgoing_count = len(graph.outgoing(FileId(source_id)))
         return HotspotsReportItem(
             source_id=source_id,
             incoming_count=incoming_count,

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/cli/facade.py

@@ -8,6 +8,7 @@ from ..architecture.config.models.architecture_config import ArchitectureConfig
 from ..architecture.facade import ArchitectureFacade
 from ..architecture.report.models.report_node import ReportNode
 from ..extraction.extractors.models.extraction_request import ExtractionRequest
+from ..extraction.extractors.models.parsed_source_batch import ParsedSourceBatch
 from ..extraction.facade import ExtractionFacade
 from ..shared.code.models.code_map import CodeMap
 from ..shared.code.models.queryable_code_map import QueryableCodeMap
@@ -99,8 +100,8 @@ class CliFacade:
     def read_sources(self, language: str) -> ExtractorCommandInput | None:
         return self.pipeline.read_sources(language)
 
-    def write_sources(self, result: dict[str, object]) -> int:
-        return self.pipeline.write_sources(result)
+    def write_sources(self, document: str) -> int:
+        return self.pipeline.write_sources(document)
 
     def generate_documentation(self, readme: str, public_interfaces: tuple[type[object], ...], check: bool) -> int:
         return self.documentation.generate(Path(readme), public_interfaces, check)
@@ -110,16 +111,21 @@ class CliFacade:
         if request is None:
             return 1
         if request.batch:
-            result: dict[str, object] = {
-                source.source_id: self.extraction.parse_source(
-                    language, source.content, str(source.path), str(source.root), source.source_id
-                )
-                for source in request.sources
-            }
-            return self.write_sources(result)
+            return self.write_sources(
+                ParsedSourceBatch(
+                    sources={
+                        source.source_id: self.extraction.parse_source(
+                            language, source.content, str(source.path), str(source.root), source.source_id
+                        )
+                        for source in request.sources
+                    }
+                ).document()
+            )
         source = request.sources[0]
         return self.write_sources(
-            self.extraction.parse_source(language, source.content, str(source.path), str(source.root), source.source_id)
+            self.extraction.parse_source(
+                language, source.content, str(source.path), str(source.root), source.source_id
+            ).model_dump_json()
         )
 
     def run(self, argv: Sequence[str]) -> int:

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/cli/pipeline/application.py

@@ -53,8 +53,8 @@ class PipelineApplication:
     def read_sources(self, language: str) -> ExtractorCommandInput | None:
         return self.extractor_command.read(language)
 
-    def write_sources(self, result: dict[str, object]) -> int:
-        return self.extractor_command.write(result)
+    def write_sources(self, document: str) -> int:
+        return self.extractor_command.write(document)
 
     def run(self, reports: tuple[ReportNode, ...], *, summary: bool) -> int:
         context = ReportPipelineContext(reports=reports, summary=summary)

modwire-11.0.2/src/modwire/cli/pipeline/batching/balanced_planner.py

@@ -0,0 +1,34 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from wireup import injectable
+
+from ....extraction.extractors.models.batch_config import BatchConfig
+from .plan import SourceBatchPlan
+from .planner import SourceBatchPlanner
+
+
+@injectable(as_type=SourceBatchPlanner, qualifier="balanced")
+@dataclass(frozen=True)
+class BalancedSourceBatchPlanner(SourceBatchPlanner):
+    def plan(self, sources: tuple[Path, ...], config: BatchConfig) -> SourceBatchPlan:
+        parallel = (
+            config.parallel_threshold > 0 and len(sources) >= config.parallel_threshold and config.max_workers > 1
+        )
+        if not parallel:
+            return SourceBatchPlan(
+                parallel=False,
+                batches=tuple(sources[start : start + config.size] for start in range(0, len(sources), config.size)),
+            )
+        if config.parallel_size <= 0:
+            raise ValueError("Balanced source batching requires a positive parallel size.")
+        worker_count = min(config.max_workers, len(sources))
+        minimum_batch_count = (len(sources) + config.parallel_size - 1) // config.parallel_size
+        batch_count = max(worker_count, minimum_batch_count)
+        complete_wave_count = ((batch_count + worker_count - 1) // worker_count) * worker_count
+        if complete_wave_count <= len(sources):
+            batch_count = complete_wave_count
+        return SourceBatchPlan(
+            parallel=True,
+            batches=tuple(sources[index::batch_count] for index in range(batch_count)),
+        )

modwire-11.0.2/src/modwire/cli/pipeline/batching/fixed_planner.py

@@ -0,0 +1,22 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+from wireup import injectable
+
+from ....extraction.extractors.models.batch_config import BatchConfig
+from .plan import SourceBatchPlan
+from .planner import SourceBatchPlanner
+
+
+@injectable(as_type=SourceBatchPlanner, qualifier="fixed")
+@dataclass(frozen=True)
+class FixedSourceBatchPlanner(SourceBatchPlanner):
+    def plan(self, sources: tuple[Path, ...], config: BatchConfig) -> SourceBatchPlan:
+        parallel = (
+            config.parallel_threshold > 0 and len(sources) >= config.parallel_threshold and config.max_workers > 1
+        )
+        batch_size = config.parallel_size if parallel and config.parallel_size > 0 else config.size
+        return SourceBatchPlan(
+            parallel=parallel,
+            batches=tuple(sources[start : start + batch_size] for start in range(0, len(sources), batch_size)),
+        )

modwire-11.0.2/src/modwire/cli/pipeline/batching/plan.py

@@ -0,0 +1,8 @@
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class SourceBatchPlan:
+    parallel: bool
+    batches: tuple[tuple[Path, ...], ...]

modwire-11.0.2/src/modwire/cli/pipeline/batching/planner.py

@@ -0,0 +1,11 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+
+from ....extraction.extractors.models.batch_config import BatchConfig
+from .plan import SourceBatchPlan
+
+
+class SourceBatchPlanner(ABC):
+    @abstractmethod
+    def plan(self, sources: tuple[Path, ...], config: BatchConfig) -> SourceBatchPlan:
+        raise NotImplementedError

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/cli/pipeline/models/extractor_batch_input.py

@@ -1,5 +1,6 @@
+from ....shared.code.models.identity import FileId
 from ....shared.values.models.value_model import ValueModel
 
 
 class ExtractorBatchInput(ValueModel):
-    paths: dict[str, str]
+    paths: dict[FileId, str]

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/cli/pipeline/models/extractor_source_input.py

@@ -1,10 +1,11 @@
 from pathlib import Path
 
+from ....shared.code.models.identity import FileId
 from ....shared.values.models.value_model import ValueModel
 
 
 class ExtractorSourceInput(ValueModel):
-    source_id: str
+    source_id: FileId
     path: Path
     root: Path
     content: str

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/cli/pipeline/services/batch_source_reader.py

@@ -3,12 +3,12 @@ import json
 import os
 import shutil
 import subprocess
+from collections.abc import Hashable, Mapping
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from importlib import resources
 from itertools import repeat
 from pathlib import Path
-from typing import Any, cast
 
 from wireup import injectable
 
@@ -21,9 +21,12 @@ from ....shared.code.models.identity import FileId, ModuleId
 from ....shared.code.models.scan_policy import ScanPolicy
 from ....shared.code.models.source_extraction import SourceExtraction
 from ....shared.code.models.source_file import SourceFile
+from ..batching.planner import SourceBatchPlanner
 from ..domain import SourceReader
 from ..models.source_entry import SourceEntry
 from ..models.source_inventory import SourceInventory
+from ..transport.output_reader import ExtractorBatchOutputReader
+from ..transport.source_file_factory import SourceFileFactory
 from .source_manifest_builder import SourceManifestBuilder
 
 
@@ -31,8 +34,11 @@ from .source_manifest_builder import SourceManifestBuilder
 @dataclass(frozen=True)
 class BatchSourceReader(SourceReader):
     code: CodeApplication
+    files: SourceFileFactory
     paths: PathMatcher
     manifests: SourceManifestBuilder
+    outputs: Mapping[Hashable, ExtractorBatchOutputReader]
+    planners: Mapping[Hashable, SourceBatchPlanner]
 
     def ensure_available(self, runtime: ExtractorRuntime) -> None:
         executable = runtime.command[0]
@@ -102,16 +108,15 @@ class BatchSourceReader(SourceReader):
             missing = sorted(str(source_id) for source_id in requested - {entry.source_id for entry in entries})
             raise ValueError(f"Source inventory does not contain requested identities: {missing}")
         root = Path(request.root).resolve()
-        source_paths = [Path(entry.path) for entry in entries]
+        source_paths = tuple(Path(entry.path) for entry in entries)
         files: dict[FileId, SourceFile] = {}
-        batches = self._source_batches(request, source_paths)
-        if self._uses_parallel_batches(request, len(source_paths)):
-            max_workers = max(1, request.batch_config.max_workers)
-            with ThreadPoolExecutor(max_workers=max_workers) as executor:
-                for extracted in executor.map(self._extract_batch, repeat(request), repeat(root), batches):
+        plan = self.planners[request.batch_config.planner].plan(source_paths, request.batch_config)
+        if plan.parallel:
+            with ThreadPoolExecutor(max_workers=request.batch_config.max_workers) as executor:
+                for extracted in executor.map(self._extract_batch, repeat(request), repeat(root), plan.batches):
                     self._merge_files(files, extracted)
         else:
-            for batch_paths in batches:
+            for batch_paths in plan.batches:
                 self._merge_files(files, self._extract_batch(request, root, batch_paths))
         expected_digests = {entry.source_id: entry.content_digest for entry in entries}
         for source_path in source_paths:
@@ -120,22 +125,6 @@ class BatchSourceReader(SourceReader):
                 raise RuntimeError(f"Source file changed during extraction: {source_id}")
         return {entry.source_id: files[entry.source_id] for entry in entries}
 
-    def _source_batches(self, request: ExtractionRequest, source_paths: list[Path]) -> list[list[Path]]:
-        batch_size = self._batch_size(request, len(source_paths))
-        return [source_paths[start : start + batch_size] for start in range(0, len(source_paths), batch_size)]
-
-    def _batch_size(self, request: ExtractionRequest, source_count: int) -> int:
-        if self._uses_parallel_batches(request, source_count) and request.batch_config.parallel_size:
-            return max(1, request.batch_config.parallel_size)
-        return max(1, request.batch_config.size)
-
-    def _uses_parallel_batches(self, request: ExtractionRequest, source_count: int) -> bool:
-        return (
-            request.batch_config.parallel_threshold > 0
-            and source_count >= request.batch_config.parallel_threshold
-            and (request.batch_config.max_workers > 1)
-        )
-
     def _discover_source_files(
         self, request: ExtractionRequest, root: Path, policy: ScanPolicy, limit: int | None
     ) -> tuple[list[Path], int, int]:
@@ -180,7 +169,7 @@ class BatchSourceReader(SourceReader):
         )
 
     def _extract_batch(
-        self, request: ExtractionRequest, root: Path, source_paths: list[Path]
+        self, request: ExtractionRequest, root: Path, source_paths: tuple[Path, ...]
     ) -> dict[FileId, SourceFile]:
         if not source_paths:
             return {}
@@ -203,20 +192,17 @@ class BatchSourceReader(SourceReader):
             raise RuntimeError(
                 f"{runtime.descriptor.language} extractor failed with exit code {completed.returncode}: {message}"
             )
-        extracted = self._parse_batch_output(request, completed.stdout)
+        extracted = self.outputs[request.batch_config.output_format].read(completed.stdout)
         source_paths_by_id = {
             self.code.file_id(str(root), str(source_path)): source_path for source_path in source_paths
         }
         extracted_files: dict[FileId, SourceFile] = {}
-        for raw_file_id, source_file in extracted.items():
-            file_id = FileId(raw_file_id)
+        for file_id, parsed_source_file in extracted.items():
             source_path = source_paths_by_id[file_id]
-            extracted_files[file_id] = SourceFile.model_validate(
-                {
-                    **source_file,
-                    "file_id": file_id,
-                    "module_id": self.code.module_id(str(root), str(source_path)),
-                }
+            extracted_files[file_id] = self.files.create(
+                file_id,
+                self.code.module_id(str(root), str(source_path)),
+                parsed_source_file,
             )
         return extracted_files
 
@@ -226,28 +212,6 @@ class BatchSourceReader(SourceReader):
                 raise DuplicateIdentityError("file", file_id, file_id, file_id)
             files[file_id] = source_file
 
-    def _parse_batch_output(self, request: ExtractionRequest, output: str) -> dict[str, Any]:
-        if request.batch_config.output_format == "jsonl":
-            result: dict[str, Any] = {}
-            for line in output.splitlines():
-                if not line.strip():
-                    continue
-                item: Any = json.loads(line)
-                if not isinstance(item, list):
-                    raise RuntimeError("Extractor returned invalid JSONL batch output.")
-                item_list = cast(list[Any], item)
-                if len(item_list) != 2:
-                    raise RuntimeError("Extractor returned invalid JSONL batch output.")
-                source_id, source_file = item_list
-                if not isinstance(source_id, str):
-                    raise RuntimeError("Extractor returned a non-string source id.")
-                result[source_id] = source_file
-            return result
-        parsed: Any = json.loads(output)
-        if not isinstance(parsed, dict):
-            raise RuntimeError("Extractor returned invalid JSON batch output.")
-        return cast(dict[str, Any], parsed)
-
     def _source_id_for_path(self, request: ExtractionRequest, root: Path, path: Path) -> FileId:
         return self.code.file_id(str(root), str(path))
 

{modwire-11.0.0 → modwire-11.0.2}/src/modwire/cli/pipeline/services/extractor_command.py

@@ -44,6 +44,6 @@ class ExtractorCommand:
             ),
         )
 
-    def write(self, result: dict[str, object]) -> int:
-        print(json.dumps(result))
+    def write(self, document: str) -> int:
+        print(document)
         return 0