mainframe-migration-toolkit 0.2.0__py3-none-any.whl

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/InputFileSet.java

@@ -0,0 +1,45 @@
+package io.mainframe.migration.validator;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Locale;
+import java.util.stream.Stream;
+
+final class InputFileSet {
+    private InputFileSet() {
+    }
+
+    static List<Path> files(InputOptions options) throws IOException {
+        if (!Files.isDirectory(options.path())) {
+            return List.of(options.path());
+        }
+        DataFormat format = options.resolvedFormat();
+        try (Stream<Path> entries = Files.list(options.path())) {
+            List<Path> parts = entries
+                    .filter(Files::isRegularFile)
+                    .filter(path -> isPart(path, format))
+                    .sorted(Comparator.comparing(path -> path.getFileName().toString()))
+                    .toList();
+            if (parts.isEmpty()) {
+                throw new IllegalArgumentException(
+                        "Nenhuma parte " + format.name() + " encontrada em: " + options.path());
+            }
+            return parts;
+        }
+    }
+
+    private static boolean isPart(Path path, DataFormat format) {
+        String name = path.getFileName().toString().toLowerCase(Locale.ROOT);
+        if (!name.startsWith("part-")) {
+            return false;
+        }
+        return switch (format) {
+            case CSV -> name.endsWith(".csv");
+            case PARQUET -> name.endsWith(".parquet") || name.endsWith(".parq");
+            case AUTO -> false;
+        };
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/InputOptions.java

@@ -0,0 +1,23 @@
+package io.mainframe.migration.validator;
+
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.util.Objects;
+
+public record InputOptions(
+        Path path,
+        DataFormat format,
+        char delimiter,
+        boolean header,
+        Charset charset) {
+
+    public InputOptions {
+        Objects.requireNonNull(path, "path");
+        Objects.requireNonNull(format, "format");
+        Objects.requireNonNull(charset, "charset");
+    }
+
+    public DataFormat resolvedFormat() {
+        return format.resolve(path);
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/JsonReportWriter.java

@@ -0,0 +1,40 @@
+package io.mainframe.migration.validator;
+
+import com.fasterxml.jackson.core.util.DefaultIndenter;
+import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+
+import java.io.IOException;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public final class JsonReportWriter {
+    private static final ObjectWriter JSON_WRITER;
+
+    static {
+        DefaultPrettyPrinter printer = new DefaultPrettyPrinter();
+        DefaultIndenter indenter = new DefaultIndenter("  ", "\n");
+        printer.indentObjectsWith(indenter);
+        printer.indentArraysWith(indenter);
+        JSON_WRITER = new ObjectMapper().writer(printer);
+    }
+
+    public void write(ValidationReport report, Writer output) throws IOException {
+        output.write(JSON_WRITER.writeValueAsString(report));
+        output.write('\n');
+        output.flush();
+    }
+
+    public void write(ValidationReport report, Path output) throws IOException {
+        Path parent = output.toAbsolutePath().getParent();
+        if (parent != null) {
+            Files.createDirectories(parent);
+        }
+        try (Writer writer = Files.newBufferedWriter(output, StandardCharsets.UTF_8)) {
+            write(report, writer);
+        }
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/MultiFileTabularReader.java

@@ -0,0 +1,80 @@
+package io.mainframe.migration.validator;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+
+final class MultiFileTabularReader implements TabularReader {
+    private final InputOptions options;
+    private final List<Path> files;
+    private int nextFile;
+    private TabularReader current;
+    private List<String> columns = List.of();
+    private boolean namedColumns;
+
+    MultiFileTabularReader(InputOptions options, List<Path> files) throws IOException {
+        this.options = options;
+        this.files = files;
+        openNext();
+    }
+
+    @Override
+    public List<String> columns() {
+        return columns;
+    }
+
+    @Override
+    public boolean hasNamedColumns() {
+        return namedColumns;
+    }
+
+    @Override
+    public List<String> readRow() throws IOException {
+        while (current != null) {
+            List<String> row = current.readRow();
+            if (row != null) {
+                return row;
+            }
+            current.close();
+            current = null;
+            openNext();
+        }
+        return null;
+    }
+
+    private void openNext() throws IOException {
+        while (nextFile < files.size()) {
+            TabularReader candidate = TabularReaderFactory.openFile(options, files.get(nextFile++));
+            if (columns.isEmpty()) {
+                if (candidate.columns().isEmpty()) {
+                    candidate.close();
+                    continue;
+                }
+                columns = candidate.columns();
+                namedColumns = candidate.hasNamedColumns();
+                current = candidate;
+                return;
+            }
+            if (namedColumns && candidate.hasNamedColumns()
+                    && !candidate.columns().isEmpty()
+                    && !columns.equals(candidate.columns())) {
+                candidate.close();
+                throw new IllegalArgumentException("Partes com colunas diferentes em: " + options.path());
+            }
+            if (candidate.columns().isEmpty()) {
+                candidate.close();
+                continue;
+            }
+            current = candidate;
+            return;
+        }
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (current != null) {
+            current.close();
+            current = null;
+        }
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/Normalization.java

@@ -0,0 +1,16 @@
+package io.mainframe.migration.validator;
+
+import java.util.Locale;
+
+public enum Normalization {
+    TRIM,
+    DECIMAL;
+
+    public static Normalization parse(String value) {
+        try {
+            return valueOf(value.trim().toUpperCase(Locale.ROOT));
+        } catch (RuntimeException exception) {
+            throw new IllegalArgumentException("Normalização inválida: " + value + ". Use trim ou decimal.");
+        }
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ParquetTabularReader.java

@@ -0,0 +1,61 @@
+package io.mainframe.migration.validator;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.parquet.avro.AvroParquetReader;
+import org.apache.parquet.hadoop.ParquetFileReader;
+import org.apache.parquet.hadoop.ParquetReader;
+import org.apache.parquet.io.LocalInputFile;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+
+final class ParquetTabularReader implements TabularReader {
+    private final ParquetReader<GenericRecord> reader;
+    private final List<String> columns;
+
+    ParquetTabularReader(Path path) throws IOException {
+        LocalInputFile input = new LocalInputFile(path);
+        try (ParquetFileReader metadataReader = ParquetFileReader.open(input)) {
+            columns = metadataReader.getFooter().getFileMetaData().getSchema().getFields().stream()
+                    .map(type -> type.getName())
+                    .toList();
+        }
+        reader = AvroParquetReader.genericRecordReader(new LocalInputFile(path));
+    }
+
+    @Override
+    public List<String> columns() {
+        return columns;
+    }
+
+    @Override
+    public boolean hasNamedColumns() {
+        return true;
+    }
+
+    @Override
+    public List<String> readRow() throws IOException {
+        GenericRecord record = reader.read();
+        if (record == null) {
+            return null;
+        }
+        List<String> values = new ArrayList<>(columns.size());
+        for (int index = 0; index < columns.size(); index++) {
+            Schema.Field field = record.getSchema().getField(columns.get(index));
+            if (field == null && index < record.getSchema().getFields().size()) {
+                field = record.getSchema().getFields().get(index);
+            }
+            Object value = field == null ? null : record.get(field.pos());
+            values.add(AvroValueFormatter.format(value, field == null ? null : field.schema()));
+        }
+        return values;
+    }
+
+    @Override
+    public void close() throws IOException {
+        reader.close();
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/TabularReader.java

@@ -0,0 +1,13 @@
+package io.mainframe.migration.validator;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+
+interface TabularReader extends Closeable {
+    List<String> columns();
+
+    boolean hasNamedColumns();
+
+    List<String> readRow() throws IOException;
+}

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/TabularReaderFactory.java

@@ -0,0 +1,29 @@
+package io.mainframe.migration.validator;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+final class TabularReaderFactory {
+    private TabularReaderFactory() {
+    }
+
+    static TabularReader open(InputOptions options) throws IOException {
+        List<Path> files = InputFileSet.files(options);
+        if (Files.isDirectory(options.path())) {
+            return new MultiFileTabularReader(options, files);
+        }
+        return openFile(options, files.get(0));
+    }
+
+    static TabularReader openFile(InputOptions options, Path file) throws IOException {
+        InputOptions fileOptions = new InputOptions(
+                file, options.resolvedFormat(), options.delimiter(), options.header(), options.charset());
+        return switch (fileOptions.resolvedFormat()) {
+            case CSV -> new CsvTabularReader(fileOptions);
+            case PARQUET -> new ParquetTabularReader(file);
+            case AUTO -> throw new IllegalStateException("Formato AUTO não resolvido");
+        };
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidationOptions.java

@@ -0,0 +1,40 @@
+package io.mainframe.migration.validator;
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+public record ValidationOptions(
+        InputOptions expected,
+        InputOptions actual,
+        Set<Normalization> normalizations,
+        int maxDifferences,
+        List<String> keys) {
+
+    public ValidationOptions {
+        Objects.requireNonNull(expected, "expected");
+        Objects.requireNonNull(actual, "actual");
+        normalizations = normalizations == null || normalizations.isEmpty()
+                ? Set.of()
+                : Set.copyOf(EnumSet.copyOf(normalizations));
+        if (maxDifferences < 1) {
+            throw new IllegalArgumentException("maxDifferences deve ser maior que zero");
+        }
+        keys = keys == null ? List.of() : keys.stream().map(String::strip).toList();
+        if (keys.stream().anyMatch(String::isEmpty)) {
+            throw new IllegalArgumentException("Nome de chave vazio");
+        }
+        if (keys.stream().distinct().count() != keys.size()) {
+            throw new IllegalArgumentException("Colunas de chave duplicadas: " + keys);
+        }
+    }
+
+    public ValidationOptions(
+            InputOptions expected,
+            InputOptions actual,
+            Set<Normalization> normalizations,
+            int maxDifferences) {
+        this(expected, actual, normalizations, maxDifferences, List.of());
+    }
+}

mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidationReport.java

@@ -0,0 +1,44 @@
+package io.mainframe.migration.validator;
+
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+import java.util.List;
+
+@JsonPropertyOrder({
+        "status",
+        "expected",
+        "actual",
+        "expectedFormat",
+        "actualFormat",
+        "keys",
+        "normalizations",
+        "expectedRows",
+        "actualRows",
+        "expectedColumns",
+        "actualColumns",
+        "differenceCount",
+        "reportedDifferenceCount",
+        "truncated",
+        "differences"
+})
+public record ValidationReport(
+        String status,
+        String expected,
+        String actual,
+        String expectedFormat,
+        String actualFormat,
+        List<String> keys,
+        List<String> normalizations,
+        long expectedRows,
+        long actualRows,
+        int expectedColumns,
+        int actualColumns,
+        long differenceCount,
+        int reportedDifferenceCount,
+        boolean truncated,
+        List<Difference> differences) {
+
+    public boolean matches() {
+        return differenceCount == 0;
+    }
+}