mainframe-migration-toolkit 0.2.0__tar.gz → 0.3.0__tar.gz

{mainframe_migration_toolkit-0.2.0 → mainframe_migration_toolkit-0.3.0}/.claude/skills/analyze-mainframe-similarity/SKILL.md

@@ -8,11 +8,13 @@ description: Cluster COBOL and JCL source by deterministic textual similarity, i
 1. Run the deterministic scan:
 
    ```text
-   python -m mainframe_toolkit similarity scan <source-root> --threshold <threshold> --json similarity.json --csv similarity.csv --review-request similarity-review-request.json
+   python -m mainframe_toolkit similarity scan <source-root> --algorithm advanced --threshold <threshold> --json similarity.json --csv similarity.csv --review-request similarity-review-request.json
    ```
 
    Use the requested threshold, or 80 when absent. Never edit deterministic scores or cluster membership manually.
 
+   Inspect each pair's `componentes` (`conteudo`, `ordem`, `renomeacao`, `estrutura`, `literais`, and `clone_parcial`). Also inspect `clones_parciais`, which are evidence of shared blocks but do not automatically join clusters. Use `--cluster-policy cohesive` when every pair in a cluster must meet the threshold; keep `connected` for broader codebase discovery.
+
 2. For each non-singleton cluster in the review request, read every original file and its normalized evidence. Check all pair relationships; connected-component chaining alone is not proof of equivalence. Ignore comments, sequence columns, copyright blocks, whitespace, generated names, and environment-only identifiers. Compare JCL execution flow/DD semantics or COBOL inputs, outputs, branches, calculations, side effects, error behavior, and record layouts.
 
 3. Assign exactly one verdict per requested cluster:

{mainframe_migration_toolkit-0.2.0 → mainframe_migration_toolkit-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: mainframe-migration-toolkit
-Version: 0.2.0
+Version: 0.3.0
 Summary: Runtime and deterministic tools for COBOL/JCL to PySpark migrations
 Author: Mainframe Migration Toolkit
 Requires-Python: >=3.10

{mainframe_migration_toolkit-0.2.0 → mainframe_migration_toolkit-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "mainframe-migration-toolkit"
-version = "0.2.0"
+version = "0.3.0"
 description = "Runtime and deterministic tools for COBOL/JCL to PySpark migrations"
 requires-python = ">=3.10"
 authors = [{ name = "Mainframe Migration Toolkit" }]

{mainframe_migration_toolkit-0.2.0 → mainframe_migration_toolkit-0.3.0}/src/mainframe_toolkit/__init__.py

@@ -89,4 +89,4 @@ __all__ = [
     "write_s3_parquet",
 ]
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"

{mainframe_migration_toolkit-0.2.0 → mainframe_migration_toolkit-0.3.0}/src/mainframe_toolkit/cli.py

@@ -84,6 +84,10 @@ def _similarity_scan(args: argparse.Namespace) -> int:
         threshold=args.threshold,
         shingle_size=args.shingle_size,
         exhaustive_limit=args.exhaustive_limit,
+        algorithm=args.algorithm,
+        partial_threshold=args.partial_threshold,
+        min_clone_tokens=args.min_clone_tokens,
+        cluster_policy=args.cluster_policy,
     )
     write_results(report, args.json, args.csv)
     write_ai_review_payload(report, args.review_request)
@@ -293,6 +297,10 @@ def _parser() -> argparse.ArgumentParser:
     scan.add_argument("--threshold", type=float, default=80.0)
     scan.add_argument("--shingle-size", type=int, default=3)
     scan.add_argument("--exhaustive-limit", type=int, default=80)
+    scan.add_argument("--algorithm", choices=("advanced", "classic"), default="advanced")
+    scan.add_argument("--partial-threshold", type=float, default=75.0)
+    scan.add_argument("--min-clone-tokens", type=int, default=30)
+    scan.add_argument("--cluster-policy", choices=("cohesive", "connected"), default="connected")
     scan.add_argument("--json", default="similarity.json")
     scan.add_argument("--csv", default="similarity.csv")
     scan.add_argument("--review-request", default="similarity-review-request.json")

{mainframe_migration_toolkit-0.2.0 → mainframe_migration_toolkit-0.3.0}/src/mainframe_toolkit/similarity.py

@@ -81,6 +81,45 @@ _LSH_BANDS = 12
 _MAX_LSH_BUCKET = 128
 _MAX_REPORTED_PAIRS = 5_000
 
+_COBOL_STRUCTURAL_WORDS = frozenset(
+    {
+        "ACCEPT", "ADD", "CALL", "CANCEL", "CLOSE", "COMPUTE", "CONTINUE",
+        "DELETE", "DISPLAY", "DIVIDE", "ELSE", "END-CALL", "END-COMPUTE",
+        "END-EVALUATE", "END-IF", "END-PERFORM", "END-READ", "END-SEARCH",
+        "END-STRING", "END-WRITE", "ENTRY", "EVALUATE", "EXIT", "GOBACK",
+        "GO", "IF", "INITIALIZE", "INSPECT", "MERGE", "MOVE", "MULTIPLY",
+        "OPEN", "PERFORM", "READ", "RELEASE", "RETURN", "REWRITE", "SEARCH",
+        "SET", "SORT", "START", "STOP", "STRING", "SUBTRACT", "UNSTRING",
+        "WHEN", "WRITE", "IDENTIFICATION", "ENVIRONMENT", "DATA", "PROCEDURE",
+        "DIVISION", "SECTION", "DECLARATIVES", "END-DECLARATIVES",
+    }
+)
+_COBOL_LANGUAGE_WORDS = _COBOL_STRUCTURAL_WORDS | frozenset(
+    {
+        "PROGRAM-ID", "FILE-CONTROL", "SELECT", "ASSIGN", "FD", "SD", "PIC",
+        "PICTURE", "VALUE", "VALUES", "OCCURS", "REDEFINES", "RENAMES", "USAGE",
+        "COMP", "COMP-1", "COMP-2", "COMP-3", "BINARY", "PACKED-DECIMAL",
+        "DISPLAY-1", "WORKING-STORAGE", "LINKAGE", "FILE", "INPUT-OUTPUT",
+        "USING", "GIVING", "THRU", "THROUGH", "UNTIL", "VARYING", "TIMES",
+        "FROM", "BY", "TO", "INTO", "OF", "IN", "ON", "AT", "END", "NOT",
+        "AND", "OR", "TRUE", "FALSE", "ZERO", "ZEROS", "ZEROES", "SPACE",
+        "SPACES", "HIGH-VALUES", "LOW-VALUES", "ALL", "CORRESPONDING", "ROUNDED",
+        "SIZE", "ERROR", "INVALID", "KEY", "WITH", "TEST", "AFTER", "BEFORE",
+        "ASCENDING", "DESCENDING", "DUPLICATES", "NO", "ADVANCING", "UPON",
+    }
+)
+_JCL_LANGUAGE_WORDS = _JCL_OPERATIONS | frozenset(
+    {
+        "PGM", "PROC", "PARM", "COND", "DSN", "DSNAME", "DISP", "UNIT",
+        "SPACE", "DCB", "SYSOUT", "CLASS", "MSGCLASS", "REGION", "TIME",
+        "RESTART", "SYMBOLS", "MEMBER", "ORDER", "INCLUDE", "SET", "IF",
+        "THEN", "ELSE", "ENDIF", "PASS", "KEEP", "CATLG", "DELETE", "SHR",
+        "OLD", "NEW", "MOD", "DUMMY", "DATA", "DLM",
+    }
+)
+_IDENTIFIER_RE = re.compile(r"[A-Z_$#@&][A-Z0-9_$#@&.-]*\Z")
+_NUMBER_RE = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)\Z")
+
 
 @dataclass(frozen=True, slots=True)
 class SourceDocument:
@@ -91,6 +130,9 @@ class SourceDocument:
     content: str = field(repr=False)
     normalized: str = field(repr=False)
     tokens: tuple[str, ...] = field(repr=False)
+    abstract_tokens: tuple[str, ...] = field(repr=False)
+    structure_tokens: tuple[str, ...] = field(repr=False)
+    literal_tokens: tuple[str, ...] = field(repr=False)
     digest: str
     source_id: str
 
@@ -100,22 +142,30 @@ class SimilarityReport:
     threshold: float
     shingle_size: int
     exhaustive_limit: int
+    algorithm: str
+    partial_threshold: float
+    cluster_policy: str
     records: list[dict[str, Any]]
     clusters: list[dict[str, Any]]
     statistics: dict[str, Any]
+    partial_clones: list[dict[str, Any]] = field(default_factory=list)
     documents: tuple[SourceDocument, ...] = field(default_factory=tuple, repr=False)
 
     def to_dict(self) -> dict[str, Any]:
         return {
-            "schema_version": 1,
+            "schema_version": 2,
             "configuracao": {
                 "limiar_percentual": self.threshold,
                 "tamanho_shingle": self.shingle_size,
                 "limite_comparacao_exaustiva": self.exhaustive_limit,
+                "algoritmo": self.algorithm,
+                "limiar_clone_parcial": self.partial_threshold,
+                "politica_cluster": self.cluster_policy,
             },
             "resumo": deepcopy(self.statistics),
             "arquivos": deepcopy(self.records),
             "clusters": deepcopy(self.clusters),
+            "clones_parciais": deepcopy(self.partial_clones),
         }
 
     def __getitem__(self, key: str) -> Any:
@@ -409,6 +459,38 @@ def tokenize_normalized(text: str) -> tuple[str, ...]:
     return tuple(_TOKEN_RE.findall(text))
 
 
+def _is_literal(token: str) -> bool:
+    return (
+        len(token) >= 2
+        and token[0] == token[-1]
+        and token[0] in {"'", '"'}
+    ) or _NUMBER_RE.fullmatch(token) is not None
+
+
+def _semantic_features(tokens: Sequence[str], tipo: str) -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]:
+    language_words = _COBOL_LANGUAGE_WORDS if tipo == "cobol" else _JCL_LANGUAGE_WORDS
+    structural_words = _COBOL_STRUCTURAL_WORDS if tipo == "cobol" else _JCL_LANGUAGE_WORDS
+    identifiers: dict[str, str] = {}
+    abstract: list[str] = []
+    structure: list[str] = []
+    literals: list[str] = []
+
+    for token in tokens:
+        upper = token.upper()
+        if _is_literal(token):
+            literals.append(token)
+            abstract.append(token)
+        elif upper in language_words or not _IDENTIFIER_RE.fullmatch(upper):
+            abstract.append(upper)
+        else:
+            placeholder = identifiers.setdefault(upper, f"__ID{len(identifiers) + 1}__")
+            abstract.append(placeholder)
+        if upper in structural_words:
+            structure.append(upper)
+
+    return tuple(abstract), tuple(structure), tuple(literals)
+
+
 def _stable_digest(value: str, *, size: int = 16) -> str:
     return hashlib.blake2b(value.encode("utf-8"), digest_size=size).hexdigest()
 
@@ -423,6 +505,8 @@ def _make_document(
     kind = _canonical_kind(tipo)
     relative = Path(relative_path).as_posix()
     normalized = normalize_source(content, kind)
+    tokens = tokenize_normalized(normalized)
+    abstract_tokens, structure_tokens, literal_tokens = _semantic_features(tokens, kind)
     return SourceDocument(
         tipo=kind,
         nome=name,
@@ -430,7 +514,10 @@ def _make_document(
         relative_path=relative,
         content=content,
         normalized=normalized,
-        tokens=tokenize_normalized(normalized),
+        tokens=tokens,
+        abstract_tokens=abstract_tokens,
+        structure_tokens=structure_tokens,
+        literal_tokens=literal_tokens,
         digest=_stable_digest(normalized),
         source_id=_stable_digest(f"{kind}\0{relative}", size=10),
     )
@@ -538,6 +625,34 @@ def _dice(left: Counter[Any], right: Counter[Any]) -> float:
     return (2.0 * overlap) / total
 
 
+def _containment(left: Counter[Any], right: Counter[Any]) -> float:
+    smallest = min(sum(left.values()), sum(right.values()))
+    if smallest == 0:
+        return 1.0 if not left and not right else 0.0
+    return sum((left & right).values()) / smallest
+
+
+@dataclass(frozen=True, slots=True)
+class PairScore:
+    percentage: float
+    content: float
+    order: float
+    renamed: float
+    structure: float
+    literals: float
+    partial: float
+
+    def to_dict(self) -> dict[str, float]:
+        return {
+            "conteudo": self.content,
+            "ordem": self.order,
+            "renomeacao": self.renamed,
+            "estrutura": self.structure,
+            "literais": self.literals,
+            "clone_parcial": self.partial,
+        }
+
+
 def _similarity_tokens(
     left: Sequence[str], right: Sequence[str], shingle_size: int
 ) -> float:
@@ -553,21 +668,73 @@ def _similarity_tokens(
     return round(100.0 * ((0.25 * ordered) + (0.75 * lexical)), 2)
 
 
+def _advanced_similarity(left: SourceDocument, right: SourceDocument, shingle_size: int) -> PairScore:
+    if left.tokens == right.tokens:
+        return PairScore(100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 100.0)
+
+    width = max(1, min(shingle_size, len(left.tokens), len(right.tokens)))
+    left_words = _feature_counter(left.tokens, 1)
+    right_words = _feature_counter(right.tokens, 1)
+    left_ordered = _feature_counter(left.tokens, width)
+    right_ordered = _feature_counter(right.tokens, width)
+    content = 100.0 * _dice(left_words, right_words)
+    order = 100.0 * _dice(left_ordered, right_ordered)
+    renamed = 100.0 * _dice(
+        _feature_counter(left.abstract_tokens, width),
+        _feature_counter(right.abstract_tokens, width),
+    )
+    structure = 100.0 * _dice(
+        _feature_counter(left.structure_tokens, 2),
+        _feature_counter(right.structure_tokens, 2),
+    )
+    literals = 100.0 * _dice(
+        _feature_counter(left.literal_tokens, 1),
+        _feature_counter(right.literal_tokens, 1),
+    )
+    partial = 100.0 * max(
+        _containment(left_ordered, right_ordered),
+        _containment(
+            _feature_counter(left.abstract_tokens, width),
+            _feature_counter(right.abstract_tokens, width),
+        ),
+    )
+    percentage = (
+        (0.15 * content)
+        + (0.15 * order)
+        + (0.35 * renamed)
+        + (0.20 * structure)
+        + (0.15 * literals)
+    )
+    return PairScore(
+        *(
+            round(value, 2)
+            for value in (percentage, content, order, renamed, structure, literals, partial)
+        )
+    )
+
+
+def _classic_pair_score(left: SourceDocument, right: SourceDocument, shingle_size: int) -> PairScore:
+    percentage = _similarity_tokens(left.tokens, right.tokens, shingle_size)
+    return PairScore(percentage, percentage, percentage, percentage, percentage, percentage, percentage)
+
+
 def similarity_percent(
     left: str,
     right: str,
     *,
     tipo: str | None = None,
     shingle_size: int = 3,
+    algorithm: str = "advanced",
 ) -> float:
     if shingle_size < 1:
         raise ValueError("shingle_size deve ser positivo")
-    if tipo is not None:
-        left = normalize_source(left, tipo)
-        right = normalize_source(right, tipo)
-    return _similarity_tokens(
-        tokenize_normalized(left), tokenize_normalized(right), shingle_size
-    )
+    if algorithm not in {"advanced", "classic"}:
+        raise ValueError("algorithm deve ser advanced ou classic")
+    kind = _canonical_kind(tipo or "cobol")
+    left_document = _make_document(kind, "LEFT", "LEFT", left)
+    right_document = _make_document(kind, "RIGHT", "RIGHT", right)
+    scorer = _advanced_similarity if algorithm == "advanced" else _classic_pair_score
+    return scorer(left_document, right_document, shingle_size).percentage
 
 
 def _shingle_hashes(tokens: Sequence[str], width: int) -> set[int]:
@@ -620,6 +787,7 @@ def _candidate_pairs(
     *,
     shingle_size: int,
     exhaustive_limit: int,
+    algorithm: str,
 ) -> tuple[set[tuple[int, int]], str]:
     candidates: set[tuple[int, int]] = set()
     indices_by_kind: dict[str, list[int]] = defaultdict(list)
@@ -641,17 +809,21 @@ def _candidate_pairs(
                 anchor = min(bucket)
                 candidates.update((anchor, item) for item in sorted(bucket) if item != anchor)
 
-        band_buckets: dict[tuple[int, tuple[int, ...]], list[int]] = defaultdict(list)
+        band_buckets: dict[tuple[str, int, tuple[int, ...]], list[int]] = defaultdict(list)
         rows = _MINHASH_SIZE // _LSH_BANDS
         # One representative per exact digest keeps duplicate-heavy scans linear.
         representatives = sorted(min(bucket) for bucket in exact_groups.values())
         for index in representatives:
-            signature = _minhash_signature(
-                _shingle_hashes(documents[index].tokens, shingle_size)
-            )
-            for band in range(_LSH_BANDS):
-                start = band * rows
-                band_buckets[(band, signature[start : start + rows])].append(index)
+            feature_streams = [("raw", documents[index].tokens)]
+            if algorithm == "advanced":
+                feature_streams.append(("abstract", documents[index].abstract_tokens))
+            for feature_name, feature_tokens in feature_streams:
+                signature = _minhash_signature(
+                    _shingle_hashes(feature_tokens, shingle_size)
+                )
+                for band in range(_LSH_BANDS):
+                    start = band * rows
+                    band_buckets[(feature_name, band, signature[start : start + rows])].append(index)
         for bucket in band_buckets.values():
             if len(bucket) > 1:
                 _add_bucket_pairs(bucket, candidates)
@@ -709,6 +881,10 @@ def analyze_documents(
     threshold: float = 80.0,
     shingle_size: int = 3,
     exhaustive_limit: int = 80,
+    algorithm: str = "advanced",
+    partial_threshold: float = 75.0,
+    min_clone_tokens: int = 30,
+    cluster_policy: str = "connected",
 ) -> SimilarityReport:
     if not 0 <= threshold <= 100:
         raise ValueError("threshold deve estar entre 0 e 100")
@@ -716,6 +892,14 @@ def analyze_documents(
         raise ValueError("shingle_size deve ser positivo")
     if exhaustive_limit < 2:
         raise ValueError("exhaustive_limit deve ser pelo menos 2")
+    if algorithm not in {"advanced", "classic"}:
+        raise ValueError("algorithm deve ser advanced ou classic")
+    if not 0 <= partial_threshold <= 100:
+        raise ValueError("partial_threshold deve estar entre 0 e 100")
+    if min_clone_tokens < 1:
+        raise ValueError("min_clone_tokens deve ser positivo")
+    if cluster_policy not in {"cohesive", "connected"}:
+        raise ValueError("cluster_policy deve ser cohesive ou connected")
 
     ordered = sorted(
         documents,
@@ -733,24 +917,55 @@ def analyze_documents(
         ordered,
         shingle_size=shingle_size,
         exhaustive_limit=exhaustive_limit,
+        algorithm=algorithm,
     )
-    score_cache: dict[tuple[int, int], float] = {}
+    score_cache: dict[tuple[int, int], PairScore] = {}
 
-    def score(left: int, right: int) -> float:
+    def score(left: int, right: int) -> PairScore:
         key = _pair_key(left, right)
         if key not in score_cache:
-            score_cache[key] = _similarity_tokens(
-                ordered[key[0]].tokens, ordered[key[1]].tokens, shingle_size
-            )
+            scorer = _advanced_similarity if algorithm == "advanced" else _classic_pair_score
+            score_cache[key] = scorer(ordered[key[0]], ordered[key[1]], shingle_size)
         return score_cache[key]
 
     union_find = _UnionFind(len(ordered))
-    qualifying_pairs = 0
+    component_members: dict[int, set[int]] = {index: {index} for index in range(len(ordered))}
+    qualifying_edges: list[tuple[float, int, int]] = []
     for left, right in sorted(candidates):
-        percentage = score(left, right)
+        percentage = score(left, right).percentage
         if percentage >= threshold:
-            union_find.union(left, right)
-            qualifying_pairs += 1
+            qualifying_edges.append((percentage, left, right))
+
+    rejected_by_cohesion = 0
+    cohesion_limit_rejections = 0
+    for _, left, right in sorted(qualifying_edges, key=lambda item: (-item[0], item[1], item[2])):
+        left_root = union_find.find(left)
+        right_root = union_find.find(right)
+        if left_root == right_root:
+            continue
+        left_members = component_members[left_root]
+        right_members = component_members[right_root]
+        should_merge = True
+        if cluster_policy == "cohesive":
+            combined = left_members | right_members
+            identical = len({ordered[index].digest for index in combined}) == 1
+            if len(combined) > exhaustive_limit and not identical:
+                should_merge = False
+                cohesion_limit_rejections += 1
+            elif not identical:
+                should_merge = all(
+                    score(member_left, member_right).percentage >= threshold
+                    for member_left in sorted(left_members)
+                    for member_right in sorted(right_members)
+                )
+        if not should_merge:
+            rejected_by_cohesion += 1
+            continue
+        union_find.union(left_root, right_root)
+        merged_root = union_find.find(left_root)
+        old_root = right_root if merged_root == left_root else left_root
+        component_members[merged_root] = left_members | right_members
+        component_members.pop(old_root, None)
 
     components: dict[int, list[int]] = defaultdict(list)
     for index in range(len(ordered)):
@@ -794,13 +1009,15 @@ def analyze_documents(
             if pair_index >= _MAX_REPORTED_PAIRS:
                 exact_interval = False
                 break
-            percentage = score(left, right)
+            pair_score = score(left, right)
+            percentage = pair_score.percentage
             pair_scores.append(percentage)
             pair_details.append(
                 {
                     "origem": ordered[left].nome,
                     "destino": ordered[right].nome,
                     "percentual": percentage,
+                    "componentes": pair_score.to_dict(),
                 }
             )
         percentage_range = _format_range(pair_scores)
@@ -846,6 +1063,28 @@ def analyze_documents(
         count * (count - 1) // 2
         for count in Counter(document.tipo for document in ordered).values()
     )
+    partial_clones: list[dict[str, Any]] = []
+    if algorithm == "advanced":
+        for (left, right), pair_score in sorted(score_cache.items()):
+            minimum_tokens = min(len(ordered[left].tokens), len(ordered[right].tokens))
+            if (
+                pair_score.percentage < threshold
+                and pair_score.partial >= partial_threshold
+                and minimum_tokens >= min_clone_tokens
+            ):
+                partial_clones.append(
+                    {
+                        "tipo": ordered[left].tipo,
+                        "origem": ordered[left].nome,
+                        "destino": ordered[right].nome,
+                        "percentual_global": pair_score.percentage,
+                        "percentual_contencao": pair_score.partial,
+                        "componentes": pair_score.to_dict(),
+                    }
+                )
+    partial_clones.sort(
+        key=lambda item: (-item["percentual_contencao"], item["tipo"], item["origem"], item["destino"])
+    )
     statistics = {
         "arquivos": len(ordered),
         "clusters": len(clusters),
@@ -853,16 +1092,25 @@ def analyze_documents(
         "pares_possiveis": possible_pairs,
         "pares_candidatos": len(candidates),
         "pares_comparados": len(score_cache),
-        "pares_acima_limiar": qualifying_pairs,
+        "pares_acima_limiar": len(qualifying_edges),
         "modo_candidatos": mode,
+        "algoritmo": algorithm,
+        "clones_parciais": len(partial_clones),
+        "politica_cluster": cluster_policy,
+        "unioes_rejeitadas_por_coesao": rejected_by_cohesion,
+        "unioes_rejeitadas_por_limite": cohesion_limit_rejections,
     }
     return SimilarityReport(
         threshold=float(threshold),
         shingle_size=shingle_size,
         exhaustive_limit=exhaustive_limit,
+        algorithm=algorithm,
+        partial_threshold=float(partial_threshold),
+        cluster_policy=cluster_policy,
         records=records,
         clusters=clusters,
         statistics=statistics,
+        partial_clones=partial_clones,
         documents=tuple(ordered),
     )
 
@@ -880,6 +1128,10 @@ def analyze_codebase(
     threshold: float = 80.0,
     shingle_size: int = 3,
     exhaustive_limit: int = 80,
+    algorithm: str = "advanced",
+    partial_threshold: float = 75.0,
+    min_clone_tokens: int = 30,
+    cluster_policy: str = "connected",
     ignored_directories: Iterable[str] = _IGNORED_DIRECTORIES,
 ) -> SimilarityReport:
     return analyze_documents(
@@ -887,6 +1139,10 @@ def analyze_codebase(
         threshold=threshold,
         shingle_size=shingle_size,
         exhaustive_limit=exhaustive_limit,
+        algorithm=algorithm,
+        partial_threshold=partial_threshold,
+        min_clone_tokens=min_clone_tokens,
+        cluster_policy=cluster_policy,
     )
 
 
@@ -972,7 +1228,7 @@ def build_ai_review_payload(
         )
 
     return {
-        "schema_version": 1,
+        "schema_version": 2,
         "tarefa": (
             "Revise cada cluster pela semantica do codigo e retorne um parecer "
             "similar, identico ou diferente, com justificativa breve."
@@ -988,6 +1244,7 @@ def build_ai_review_payload(
             ]
         },
         "clusters": clusters,
+        "clones_parciais": deepcopy(payload.get("clones_parciais", [])),
     }