mainframe-migration-toolkit 0.2.0__py3-none-any.whl

mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/JsonReportWriterTest.java

@@ -0,0 +1,43 @@
+package io.mainframe.migration.validator;
+
+import org.junit.jupiter.api.Test;
+
+import java.io.StringWriter;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class JsonReportWriterTest {
+    @Test
+    void outputIsStableAndUsesLfOnEveryPlatform() throws Exception {
+        ValidationReport report = new ValidationReport(
+                "DIFFERENT",
+                "expected.csv",
+                "actual.csv",
+                "CSV",
+                "CSV",
+                List.of(),
+                List.of("trim"),
+                1,
+                1,
+                1,
+                1,
+                1,
+                1,
+                false,
+                List.of(new Difference(1, 1, "name", " Ana ", "Bia", "VALUE_MISMATCH")));
+        JsonReportWriter writer = new JsonReportWriter();
+        StringWriter first = new StringWriter();
+        StringWriter second = new StringWriter();
+
+        writer.write(report, first);
+        writer.write(report, second);
+
+        assertEquals(first.toString(), second.toString());
+        assertFalse(first.toString().contains("\r"));
+        assertTrue(first.toString().endsWith("\n"));
+        assertTrue(first.toString().contains("\"line\" : 1"));
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/KeyedValidationTest.java

@@ -0,0 +1,89 @@
+package io.mainframe.migration.validator;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class KeyedValidationTest {
+    @TempDir
+    Path temporaryDirectory;
+
+    @Test
+    void comparesRowsByCompositeKeyRegardlessOfPhysicalOrder() throws Exception {
+        Path expected = csv("expected.csv", "account,seq,value\nB,2,x\nA,1,y\n");
+        Path actual = csv("actual.csv", "account,seq,value\nA,1,y\nB,2,x\n");
+
+        ValidationReport report = validate(expected, actual, List.of("account", "seq"), Set.of());
+
+        assertTrue(report.matches());
+        assertEquals(List.of("account", "seq"), report.keys());
+        assertEquals(2, report.expectedRows());
+    }
+
+    @Test
+    void ordersDifferencesByNormalizedKey() throws Exception {
+        Path expected = csv("expected.csv", "id,value\nB,expected-b\n A ,expected-a\n");
+        Path actual = csv("actual.csv", "id,value\nA,actual-a\nB,actual-b\n");
+
+        ValidationReport report = validate(
+                expected, actual, List.of("id"), Set.of(Normalization.TRIM));
+
+        assertEquals(2, report.differenceCount());
+        assertEquals(" A ", report.differences().get(0).key().get("id"));
+        assertEquals("B", report.differences().get(1).key().get("id"));
+        assertEquals("value", report.differences().get(0).column());
+    }
+
+    @Test
+    void reportsDuplicateKeysOnEitherSide() throws Exception {
+        Path expected = csv("expected.csv", "id,value\n1,A\n1,A\n2,B\n");
+        Path actual = csv("actual.csv", "id,value\n2,B\n1,A\n1,A\n");
+
+        ValidationReport report = validate(expected, actual, List.of("id"), Set.of());
+
+        assertEquals(2, report.differenceCount());
+        assertEquals("DUPLICATE_EXPECTED_KEY", report.differences().get(0).reason());
+        assertEquals("DUPLICATE_ACTUAL_KEY", report.differences().get(1).reason());
+        assertEquals("1", report.differences().get(0).key().get("id"));
+    }
+
+    @Test
+    void keyModeWorksAcrossSparkPartDirectories() throws Exception {
+        Path expected = Files.createDirectory(temporaryDirectory.resolve("expected-dir"));
+        Files.writeString(expected.resolve("part-00000.csv"), "id,value\n2,B\n");
+        Files.writeString(expected.resolve("part-00001.csv"), "id,value\n1,A\n");
+        Path actual = Files.createDirectory(temporaryDirectory.resolve("actual-dir"));
+        Files.writeString(actual.resolve("part-00000.csv"), "id,value\n1,A\n2,B\n");
+
+        ValidationReport report = validate(expected, actual, List.of("id"), Set.of());
+
+        assertTrue(report.matches());
+    }
+
+    private ValidationReport validate(
+            Path expected,
+            Path actual,
+            List<String> keys,
+            Set<Normalization> normalizations) throws Exception {
+        InputOptions left = new InputOptions(
+                expected, DataFormat.AUTO, ',', true, StandardCharsets.UTF_8);
+        InputOptions right = new InputOptions(
+                actual, DataFormat.AUTO, ',', true, StandardCharsets.UTF_8);
+        return new ValidationService().validate(
+                new ValidationOptions(left, right, normalizations, 100, keys));
+    }
+
+    private Path csv(String name, String content) throws Exception {
+        Path path = temporaryDirectory.resolve(name);
+        Files.writeString(path, content, StandardCharsets.UTF_8);
+        return path;
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/ParquetValidationTest.java

@@ -0,0 +1,146 @@
+package io.mainframe.migration.validator;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.parquet.avro.AvroParquetWriter;
+import org.apache.parquet.hadoop.ParquetWriter;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
+import org.apache.parquet.io.LocalOutputFile;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ParquetValidationTest {
+    private static final Schema SCHEMA = new Schema.Parser().parse("""
+            {
+              "type": "record",
+              "name": "Result",
+              "fields": [
+                {"name": "id", "type": "long"},
+                {"name": "amount", "type": "double"},
+                {"name": "name", "type": ["null", "string"], "default": null}
+              ]
+            }
+            """);
+
+    @TempDir
+    Path temporaryDirectory;
+
+    @Test
+    void comparesParquetCellsInRecordOrder() throws Exception {
+        Path expected = parquet("expected.parquet", List.of(
+                row(1, 10.5, "Ana"),
+                row(2, 20.0, "Bia")));
+        Path actual = parquet("actual.parquet", List.of(
+                row(1, 10.5, "Ana"),
+                row(2, 20.0, "Bea")));
+
+        ValidationReport report = validate(
+                input(expected, DataFormat.AUTO, true),
+                input(actual, DataFormat.AUTO, true),
+                Set.of());
+
+        assertEquals(1, report.differenceCount());
+        Difference difference = report.differences().get(0);
+        assertEquals(2, difference.line());
+        assertEquals(3, difference.columnIndex());
+        assertEquals("name", difference.column());
+        assertEquals("Bia", difference.expected());
+        assertEquals("Bea", difference.actual());
+    }
+
+    @Test
+    void readsSchemaFromEmptyParquetFiles() throws Exception {
+        Path expected = parquet("expected.parquet", List.of());
+        Path actual = parquet("actual.parquet", List.of());
+
+        ValidationReport report = validate(
+                input(expected, DataFormat.PARQUET, true),
+                input(actual, DataFormat.PARQUET, true),
+                Set.of());
+
+        assertTrue(report.matches());
+        assertEquals(0, report.expectedRows());
+        assertEquals(3, report.expectedColumns());
+    }
+
+    @Test
+    void comparesCsvAndParquetWithDecimalNormalization() throws Exception {
+        Path expected = temporaryDirectory.resolve("expected.csv");
+        Files.writeString(expected, "id,amount,name\n1,1.2300,Ana\n", StandardCharsets.UTF_8);
+        Path actual = parquet("actual.parquet", List.of(row(1, 1.23, "Ana")));
+
+        ValidationReport report = validate(
+                input(expected, DataFormat.CSV, true),
+                input(actual, DataFormat.PARQUET, true),
+                Set.of(Normalization.DECIMAL));
+
+        assertTrue(report.matches());
+    }
+
+    @Test
+    void readsSparkParquetPartDirectoriesInLexicalOrder() throws Exception {
+        Path expected = Files.createDirectory(temporaryDirectory.resolve("expected-dir"));
+        parquet(expected.resolve("part-00001.snappy.parquet"), List.of(row(2, 2.0, "Bia")));
+        parquet(expected.resolve("part-00000.snappy.parquet"), List.of(row(1, 1.0, "Ana")));
+        Files.writeString(expected.resolve("_SUCCESS"), "");
+        Path actual = Files.createDirectory(temporaryDirectory.resolve("actual-dir"));
+        parquet(actual.resolve("part-00000.parquet"), List.of(
+                row(1, 1.0, "Ana"),
+                row(2, 2.0, "Bia")));
+
+        ValidationReport report = validate(
+                input(expected, DataFormat.AUTO, true),
+                input(actual, DataFormat.AUTO, true),
+                Set.of());
+
+        assertTrue(report.matches());
+        assertEquals(2, report.expectedRows());
+        assertEquals("PARQUET", report.expectedFormat());
+    }
+
+    private ValidationReport validate(
+            InputOptions expected,
+            InputOptions actual,
+            Set<Normalization> normalizations) throws Exception {
+        return new ValidationService().validate(new ValidationOptions(expected, actual, normalizations, 100));
+    }
+
+    private InputOptions input(Path path, DataFormat format, boolean header) {
+        return new InputOptions(path, format, ',', header, StandardCharsets.UTF_8);
+    }
+
+    private GenericRecord row(long id, double amount, String name) {
+        GenericRecord record = new GenericData.Record(SCHEMA);
+        record.put("id", id);
+        record.put("amount", amount);
+        record.put("name", name);
+        return record;
+    }
+
+    private Path parquet(String name, List<GenericRecord> records) throws Exception {
+        Path path = temporaryDirectory.resolve(name);
+        return parquet(path, records);
+    }
+
+    private Path parquet(Path path, List<GenericRecord> records) throws Exception {
+        try (ParquetWriter<GenericRecord> writer = AvroParquetWriter.<GenericRecord>builder(new LocalOutputFile(path))
+                .withSchema(SCHEMA)
+                .withCompressionCodec(CompressionCodecName.UNCOMPRESSED)
+                .build()) {
+            for (GenericRecord record : records) {
+                writer.write(record);
+            }
+        }
+        return path;
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/ValidationServiceTest.java

@@ -0,0 +1,137 @@
+package io.mainframe.migration.validator;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ValidationServiceTest {
+    @TempDir
+    Path temporaryDirectory;
+
+    @Test
+    void reportsEveryCellWithItsLineAndColumn() throws Exception {
+        Path expected = csv("expected.csv", "id;amount\n1;10.00\n2;ABC\n");
+        Path actual = csv("actual.csv", "id;amount\n1;10.01\n2;ABC\n3;X\n");
+
+        ValidationReport report = validate(expected, actual, ';', true, Set.of(), 20);
+
+        assertEquals("DIFFERENT", report.status());
+        assertEquals(2, report.expectedRows());
+        assertEquals(3, report.actualRows());
+        assertEquals(3, report.differenceCount());
+        Difference first = report.differences().get(0);
+        assertEquals(1, first.line());
+        assertEquals(2, first.columnIndex());
+        assertEquals("amount", first.column());
+        assertEquals("10.00", first.expected());
+        assertEquals("10.01", first.actual());
+        assertEquals("VALUE_MISMATCH", first.reason());
+        Difference missingRowCell = report.differences().get(1);
+        assertEquals(3, missingRowCell.line());
+        assertNull(missingRowCell.expected());
+        assertEquals("3", missingRowCell.actual());
+        assertEquals("UNEXPECTED_ACTUAL_CELL", missingRowCell.reason());
+    }
+
+    @Test
+    void comparesNamedColumnsBeforeData() throws Exception {
+        Path expected = csv("expected.csv", "id,name\n1,Ana\n");
+        Path actual = csv("actual.csv", "id,full_name\n1,Ana\n");
+
+        ValidationReport report = validate(expected, actual, ',', true, Set.of(), 20);
+
+        assertEquals(1, report.differenceCount());
+        Difference difference = report.differences().get(0);
+        assertEquals(0, difference.line());
+        assertEquals(2, difference.columnIndex());
+        assertEquals("name", difference.expected());
+        assertEquals("full_name", difference.actual());
+        assertEquals("COLUMN_NAME_MISMATCH", difference.reason());
+    }
+
+    @Test
+    void appliesTrimAndDecimalNormalizations() throws Exception {
+        Path expected = csv("expected.csv", "value\n 001.2300 \n-0.00\ntext \n");
+        Path actual = csv("actual.csv", "value\n1.23\n0\ntext\n");
+
+        ValidationReport report = validate(
+                expected,
+                actual,
+                ',',
+                true,
+                Set.of(Normalization.TRIM, Normalization.DECIMAL),
+                20);
+
+        assertTrue(report.matches());
+        assertEquals("MATCH", report.status());
+        assertEquals(0, report.differenceCount());
+        assertEquals(java.util.List.of("trim", "decimal"), report.normalizations());
+    }
+
+    @Test
+    void noHeaderKeepsFirstRecordAsData() throws Exception {
+        Path expected = csv("expected.csv", "1,Ana\n2,Bia\n");
+        Path actual = csv("actual.csv", "1,Ana\n2,Bia\n");
+
+        ValidationReport report = validate(expected, actual, ',', false, Set.of(), 20);
+
+        assertTrue(report.matches());
+        assertEquals(2, report.expectedRows());
+        assertEquals(2, report.expectedColumns());
+    }
+
+    @Test
+    void limitsDetailsButCountsAllDifferences() throws Exception {
+        Path expected = csv("expected.csv", "a,b\n1,2\n3,4\n");
+        Path actual = csv("actual.csv", "a,b\nx,y\nz,w\n");
+
+        ValidationReport report = validate(expected, actual, ',', true, Set.of(), 2);
+
+        assertEquals(4, report.differenceCount());
+        assertEquals(2, report.reportedDifferenceCount());
+        assertEquals(2, report.differences().size());
+        assertTrue(report.truncated());
+    }
+
+    @Test
+    void leavesNonNumericCellsUntouchedDuringDecimalNormalization() throws Exception {
+        Path expected = csv("expected.csv", "v\n001A\n");
+        Path actual = csv("actual.csv", "v\n1A\n");
+
+        ValidationReport report = validate(
+                expected, actual, ',', true, Set.of(Normalization.DECIMAL), 20);
+
+        assertFalse(report.matches());
+        assertEquals("001A", report.differences().get(0).expected());
+    }
+
+    private ValidationReport validate(
+            Path expected,
+            Path actual,
+            char delimiter,
+            boolean header,
+            Set<Normalization> normalizations,
+            int maxDifferences) throws Exception {
+        InputOptions expectedOptions = new InputOptions(
+                expected, DataFormat.AUTO, delimiter, header, StandardCharsets.UTF_8);
+        InputOptions actualOptions = new InputOptions(
+                actual, DataFormat.AUTO, delimiter, header, StandardCharsets.UTF_8);
+        return new ValidationService().validate(
+                new ValidationOptions(expectedOptions, actualOptions, normalizations, maxDifferences));
+    }
+
+    private Path csv(String name, String contents) throws Exception {
+        Path path = temporaryDirectory.resolve(name);
+        Files.writeString(path, contents, StandardCharsets.UTF_8);
+        return path;
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/ValidatorCliTest.java

@@ -0,0 +1,102 @@
+package io.mainframe.migration.validator;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import picocli.CommandLine;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ValidatorCliTest {
+    @TempDir
+    Path temporaryDirectory;
+
+    @Test
+    void returnsZeroForMatchAndWritesJsonToStdout() throws Exception {
+        Path expected = write("expected.csv", "v\n 01.00 \n");
+        Path actual = write("actual.csv", "v\n1\n");
+        StringWriter stdout = new StringWriter();
+        CommandLine command = ValidatorCli.commandLine();
+        command.setOut(new PrintWriter(stdout, true));
+
+        int exitCode = command.execute(
+                expected.toString(),
+                actual.toString(),
+                "--normalize", "trim,decimal");
+
+        assertEquals(ValidatorCli.EXIT_MATCH, exitCode);
+        assertTrue(stdout.toString().contains("\"status\" : \"MATCH\""));
+    }
+
+    @Test
+    void returnsOneForDifferencesAndWritesReportFile() throws Exception {
+        Path expected = write("expected.data", "id|name\n1|Ana\n");
+        Path actual = write("actual.data", "id|name\n1|Bia\n");
+        Path report = temporaryDirectory.resolve("reports/result.json");
+        CommandLine command = ValidatorCli.commandLine();
+
+        int exitCode = command.execute(
+                expected.toString(),
+                actual.toString(),
+                "--format", "csv",
+                "--delimiter", "pipe",
+                "--report", report.toString());
+
+        assertEquals(ValidatorCli.EXIT_DIFFERENT, exitCode);
+        assertTrue(Files.readString(report).contains("\"status\" : \"DIFFERENT\""));
+    }
+
+    @Test
+    void returnsTwoForInvalidInput() {
+        StringWriter stderr = new StringWriter();
+        CommandLine command = ValidatorCli.commandLine();
+        command.setErr(new PrintWriter(stderr, true));
+
+        int exitCode = command.execute("missing.csv", "also-missing.csv");
+
+        assertEquals(ValidatorCli.EXIT_INPUT_ERROR, exitCode);
+        assertTrue(stderr.toString().contains("Arquivo ou diretório não encontrado"));
+    }
+
+    @Test
+    void supportsNoHeaderAndTabAlias() throws Exception {
+        Path expected = write("expected.csv", "1\tAna\n");
+        Path actual = write("actual.csv", "1\tAna\n");
+        StringWriter stdout = new StringWriter();
+        CommandLine command = ValidatorCli.commandLine();
+        command.setOut(new PrintWriter(stdout, true));
+
+        int exitCode = command.execute(
+                expected.toString(), actual.toString(), "--delimiter", "tab", "--no-header");
+
+        assertEquals(ValidatorCli.EXIT_MATCH, exitCode);
+        assertTrue(stdout.toString().contains("\"expectedRows\" : 1"), stdout::toString);
+    }
+
+    @Test
+    void acceptsRepeatedAndCommaSeparatedKeys() throws Exception {
+        Path expected = write("expected.csv", "tenant,id,value\nB,2,x\nA,1,y\n");
+        Path actual = write("actual.csv", "tenant,id,value\nA,1,y\nB,2,x\n");
+        StringWriter stdout = new StringWriter();
+        CommandLine command = ValidatorCli.commandLine();
+        command.setOut(new PrintWriter(stdout, true));
+
+        int exitCode = command.execute(
+                expected.toString(), actual.toString(), "--key", "tenant,id");
+
+        assertEquals(ValidatorCli.EXIT_MATCH, exitCode);
+        assertTrue(stdout.toString().contains("\"tenant\""), stdout::toString);
+        assertTrue(stdout.toString().contains("\"id\""), stdout::toString);
+    }
+
+    private Path write(String name, String content) throws Exception {
+        Path path = temporaryDirectory.resolve(name);
+        Files.writeString(path, content);
+        return path;
+    }
+}