mainframe-migration-toolkit 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. mainframe_migration_toolkit-0.2.0.dist-info/METADATA +16 -0
  2. mainframe_migration_toolkit-0.2.0.dist-info/RECORD +52 -0
  3. mainframe_migration_toolkit-0.2.0.dist-info/WHEEL +4 -0
  4. mainframe_migration_toolkit-0.2.0.dist-info/entry_points.txt +3 -0
  5. mainframe_toolkit/__init__.py +92 -0
  6. mainframe_toolkit/__main__.py +5 -0
  7. mainframe_toolkit/_workspace/.claude/skills/analyze-mainframe-similarity/SKILL.md +30 -0
  8. mainframe_toolkit/_workspace/.claude/skills/migrate-mainframe-job/SKILL.md +63 -0
  9. mainframe_toolkit/_workspace/.claude/skills/validate-golden-dataset/SKILL.md +12 -0
  10. mainframe_toolkit/_workspace/AGENTS.md +10 -0
  11. mainframe_toolkit/_workspace/CLAUDE.md +2 -0
  12. mainframe_toolkit/_workspace/validator-java/.mvn/wrapper/maven-wrapper.properties +3 -0
  13. mainframe_toolkit/_workspace/validator-java/mvnw +295 -0
  14. mainframe_toolkit/_workspace/validator-java/mvnw.cmd +189 -0
  15. mainframe_toolkit/_workspace/validator-java/pom.xml +116 -0
  16. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/AvroValueFormatter.java +152 -0
  17. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/CsvTabularReader.java +111 -0
  18. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/DataFormat.java +62 -0
  19. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/Difference.java +34 -0
  20. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/InputFileSet.java +45 -0
  21. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/InputOptions.java +23 -0
  22. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/JsonReportWriter.java +40 -0
  23. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/MultiFileTabularReader.java +80 -0
  24. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/Normalization.java +16 -0
  25. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ParquetTabularReader.java +61 -0
  26. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/TabularReader.java +13 -0
  27. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/TabularReaderFactory.java +29 -0
  28. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidationOptions.java +40 -0
  29. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidationReport.java +44 -0
  30. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidationService.java +395 -0
  31. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidatorCli.java +190 -0
  32. mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValueNormalizer.java +27 -0
  33. mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/DirectoryValidationTest.java +70 -0
  34. mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/JsonReportWriterTest.java +43 -0
  35. mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/KeyedValidationTest.java +89 -0
  36. mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/ParquetValidationTest.java +146 -0
  37. mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/ValidationServiceTest.java +137 -0
  38. mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/ValidatorCliTest.java +102 -0
  39. mainframe_toolkit/cli.py +361 -0
  40. mainframe_toolkit/cobol.py +106 -0
  41. mainframe_toolkit/copybook.py +558 -0
  42. mainframe_toolkit/errors.py +15 -0
  43. mainframe_toolkit/external.py +48 -0
  44. mainframe_toolkit/io.py +202 -0
  45. mainframe_toolkit/jcl.py +126 -0
  46. mainframe_toolkit/pipeline.py +322 -0
  47. mainframe_toolkit/sequential.py +259 -0
  48. mainframe_toolkit/similarity.py +1171 -0
  49. mainframe_toolkit/sorting.py +60 -0
  50. mainframe_toolkit/specs.py +312 -0
  51. mainframe_toolkit/synthetic.py +108 -0
  52. mainframe_toolkit/workspace.py +133 -0
@@ -0,0 +1,16 @@
1
+ Metadata-Version: 2.5
2
+ Name: mainframe-migration-toolkit
3
+ Version: 0.2.0
4
+ Summary: Runtime and deterministic tools for COBOL/JCL to PySpark migrations
5
+ Author: Mainframe Migration Toolkit
6
+ Requires-Python: >=3.10
7
+ Provides-Extra: all
8
+ Requires-Dist: pyarrow>=15; extra == 'all'
9
+ Requires-Dist: pyspark<5,>=3.5; extra == 'all'
10
+ Provides-Extra: dev
11
+ Requires-Dist: pytest-cov>=5; extra == 'dev'
12
+ Requires-Dist: pytest>=8.2; extra == 'dev'
13
+ Provides-Extra: parquet
14
+ Requires-Dist: pyarrow>=15; extra == 'parquet'
15
+ Provides-Extra: spark
16
+ Requires-Dist: pyspark<5,>=3.5; extra == 'spark'
@@ -0,0 +1,52 @@
1
+ mainframe_toolkit/__init__.py,sha256=iBuG-E3iICCMZUrJeVZ4o2iM7zOY5acOrJIUm3YGNkw,2168
2
+ mainframe_toolkit/__main__.py,sha256=4BTAfHHF2u3j_oUg_Wluqu0HyxnXfAHBQUcnPQ-e1xs,50
3
+ mainframe_toolkit/cli.py,sha256=fpHDNn2aoScDqzIHCDtEan6HVMrxxSlTpThAX8a_UDk,14495
4
+ mainframe_toolkit/cobol.py,sha256=EOeTBh74IwwLtmZKF0Ov8fnuB1gAWGHc4qyA7Kfa6ug,3658
5
+ mainframe_toolkit/copybook.py,sha256=omXj6X1BhtaXvKUsDYR6siBQqgIQUzL4hH3eoBF5fRA,18248
6
+ mainframe_toolkit/errors.py,sha256=u1cBP7Zu8bUK9Xn-T2Qb45-ikw0cVqEfjhvIImdmCAs,395
7
+ mainframe_toolkit/external.py,sha256=IGZLHyu9qfhzqZYsUcfITtlIOhzfZ9c9BwSKaH4xwRQ,1942
8
+ mainframe_toolkit/io.py,sha256=hQTbu2Gp-zYM4cBFtUzdZjg_ajPCbENoXNgZJnWDPkI,6605
9
+ mainframe_toolkit/jcl.py,sha256=UB3t33UcpEeGkIVSQo4meKWN6968J5vCX9Bei6WHl0U,4604
10
+ mainframe_toolkit/pipeline.py,sha256=cIivUpc2FY158CsrE_I76SOzgfTGjUeO_tcLfP4LfV4,12078
11
+ mainframe_toolkit/sequential.py,sha256=iJYANVtGnFH42hp6mXyRfk7Oy8fJWrceKSQbL--Nkq0,9182
12
+ mainframe_toolkit/similarity.py,sha256=GUxSt_vySk_n5yFsRqqg0XKzRLCdeZj8C4Vk6TjdU34,38319
13
+ mainframe_toolkit/sorting.py,sha256=HBtEQxS05-9m-DnhJyKI7hfrQ5e_RM3nCopRdH6HCzA,2041
14
+ mainframe_toolkit/specs.py,sha256=8c-ZRUJQvaJokZGsypmModaM-e5rOBm3nt60eGc5E5w,12311
15
+ mainframe_toolkit/synthetic.py,sha256=RKdqkmqaxssktWJN6H85Ang-bN5uEXTJDyU3nXgEKTE,4276
16
+ mainframe_toolkit/workspace.py,sha256=5tyak9Tmyopc5in0c_UFeUCJ6JT00KXH5CIGGA_FUsM,4740
17
+ mainframe_toolkit/_workspace/AGENTS.md,sha256=Jl0fjSNtzzVVpx5st0zA2mDVSkfV4a_n1Z1ivr55PXA,1565
18
+ mainframe_toolkit/_workspace/CLAUDE.md,sha256=XnHDtjkvRbD50sNxe-Ki7JuUHXo7YPlRTY5UdHngJag,12
19
+ mainframe_toolkit/_workspace/validator-java/mvnw,sha256=yuls74nr6jUxIh9K4Xwjz47fZ9AOroMG1Bhq4bvtTQI,11790
20
+ mainframe_toolkit/_workspace/validator-java/mvnw.cmd,sha256=Ru7bhBm9FP5w1bspFte29RgG5Rs51bdqQmEDhMqSnBw,8481
21
+ mainframe_toolkit/_workspace/validator-java/pom.xml,sha256=2HW7K2QRFWRG1UT_ws1Xd4HhYWrA4FqzJ6wutD5EH3g,3961
22
+ mainframe_toolkit/_workspace/validator-java/.mvn/wrapper/maven-wrapper.properties,sha256=jypEOIEDSCcNnZj41tUwVB3bcnHjc9sgforDgeDjW2I,168
23
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/AvroValueFormatter.java,sha256=CTwiBZEI8B31Gd5mknlaYwQouc_8ie0CCNqlBhbhR_8,6165
24
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/CsvTabularReader.java,sha256=az7e6IYWIAbG1tmM-AHiRvXxrVW8E91xlD2U6I08Ctk,3446
25
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/DataFormat.java,sha256=vbVyeuKJ_oxMrDxH0Hk4WFTlGfvt00A6JFPxrypLQ0M,2289
26
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/Difference.java,sha256=fAWJ64edhpZ03_AMVdRhO_se6vxJiGyeaKtHYxQKKBs,927
27
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/InputFileSet.java,sha256=ktAgQKPW0yzJP7btzKqbF7Q-Qy7zAZBoPwRVnwwF3eg,1535
28
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/InputOptions.java,sha256=BnfW6-KChmAIQu8nh98OJbNG1c3ZX3BjrQYP4kNPImc,547
29
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/JsonReportWriter.java,sha256=Vg5qBwAdKdmGIJD4ygHv6arQwh42VvFe4McqY9aqjMY,1376
30
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/MultiFileTabularReader.java,sha256=FvIppE8HZe6BmEYTA4AYgDH6aDyRcAMYmpOcx0mhsMg,2335
31
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/Normalization.java,sha256=vE25GgHSabyKKg4ioFf23Krt0DJMQcN-zNKteULKlqQ,434
32
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ParquetTabularReader.java,sha256=T1L0cS0DipALIuVpGNe7U-ZpeTMIPF624KgzbI30NBw,2069
33
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/TabularReader.java,sha256=JSGorThD2LKwuJ4Jpwg4MRnnWi-tVa9QZgcPRSQ33Ks,275
34
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/TabularReaderFactory.java,sha256=8Po2kyidjstrsevSlHwX1XobuekZVlwRkGelphYnP7U,1065
35
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidationOptions.java,sha256=MfdfOsqhnswRt9vRJeSXdhNlSQBbJnrGJ7nz7QyC-9A,1420
36
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidationReport.java,sha256=6jAlflov1O92rri-eB1BfNL_CjP66rwyFXkTncLblZc,1048
37
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidationService.java,sha256=3saqmhdMExgajxf0jB44dWrhWiMKlHjckoQJS9O-oL4,15999
38
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValidatorCli.java,sha256=tjb6XgsMLG7sYtX16VRT_o-uNkJHfiT165lm0tfSeBo,7971
39
+ mainframe_toolkit/_workspace/validator-java/src/main/java/io/mainframe/migration/validator/ValueNormalizer.java,sha256=A6VSnTAiPNHWKp79vraC0ZJ5eQtP2Fja2MS35N7ZTYY,888
40
+ mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/DirectoryValidationTest.java,sha256=84wZ3DsJiZeWXc7Rutw_6GIrWP_pJ5hXhmHQ3b-ESOg,3075
41
+ mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/JsonReportWriterTest.java,sha256=PjuWz0wMUDFsZGpn-7McuJWLm_pnozl69Djzw_KyqwI,1372
42
+ mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/KeyedValidationTest.java,sha256=FXsg3LdL1NgMKwE7exPs2jLBGzy19avUHmzknq8yFS8,3640
43
+ mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/ParquetValidationTest.java,sha256=1OhMG2v4msDmoYzBs3iux3ZFyrKnBG9fJ7ZGvBq4GBw,5569
44
+ mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/ValidationServiceTest.java,sha256=HgV51RgTTUNBA5pf1ILrmNJhOKIk4PZE1lq7OdIPSKY,5350
45
+ mainframe_toolkit/_workspace/validator-java/src/test/java/io/mainframe/migration/validator/ValidatorCliTest.java,sha256=jn-WL2u6-ZnJAgOjqhJN1RwlPYUOHjQzAe6zhbqY0hM,3866
46
+ mainframe_toolkit/_workspace/.claude/skills/analyze-mainframe-similarity/SKILL.md,sha256=DblDy-Q7N0IOt_1OeeO2cv66c9YqajrabrV9HdfkhBA,1926
47
+ mainframe_toolkit/_workspace/.claude/skills/migrate-mainframe-job/SKILL.md,sha256=PfwPGi-uM9rCXfaSfdUDwo6gYxiX8cgXW-7ShMh8FHg,5123
48
+ mainframe_toolkit/_workspace/.claude/skills/validate-golden-dataset/SKILL.md,sha256=A5l8g7k6LT1pNKxJdKNT-3Tj5CI9UCSwZ7ZbJqg92dQ,1001
49
+ mainframe_migration_toolkit-0.2.0.dist-info/METADATA,sha256=lwCG7Zdl9IypviltG-esz3NSbfTvOe0Zkc54VHNH4D4,568
50
+ mainframe_migration_toolkit-0.2.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
51
+ mainframe_migration_toolkit-0.2.0.dist-info/entry_points.txt,sha256=V5_IEuCYCDUOEVCjnqrOz2eipU1EdlcKtMUJS23qMP0,98
52
+ mainframe_migration_toolkit-0.2.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.32.0
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
@@ -0,0 +1,3 @@
1
+ [console_scripts]
2
+ mainframe-toolkit = mainframe_toolkit.cli:main
3
+ mft = mainframe_toolkit.cli:main
@@ -0,0 +1,92 @@
1
+ """Reusable runtime for specification-led COBOL/JCL migrations."""
2
+
3
+ from .cobol import (
4
+ cobol_decimal,
5
+ decode_overpunch,
6
+ decode_packed_decimal,
7
+ encode_packed_decimal,
8
+ is_numeric,
9
+ quantize_cobol,
10
+ )
11
+ from .copybook import CopybookParseError, layout_to_dict, parse_copybook, parse_copybook_text
12
+ from .external import ExternalProgramRegistry, external_program
13
+ from .io import (
14
+ configure_s3,
15
+ read_csv,
16
+ read_dataset,
17
+ read_delimited_rows,
18
+ read_parquet,
19
+ read_s3_csv,
20
+ read_s3_parquet,
21
+ s3_uri,
22
+ write_csv,
23
+ write_dataset,
24
+ write_parquet,
25
+ write_s3_csv,
26
+ write_s3_parquet,
27
+ )
28
+ from .jcl import JCLJob, JCLStep, parse_jcl
29
+ from .pipeline import (
30
+ DatasetRegistry,
31
+ PipelineContext,
32
+ PipelineProgram,
33
+ ProgramResult,
34
+ PipelineRunner,
35
+ PipelineRunError,
36
+ ProgramReturnCodeError,
37
+ )
38
+ from .sequential import FieldSpec, RecordLayout, convert_sequential, iter_sequential
39
+ from .sorting import SortKey, sort_dataframe, sort_records
40
+ from .synthetic import synthetic_dataframe, synthetic_rows
41
+ from .workspace import WorkspaceInitResult, init_workspace
42
+
43
+ __all__ = [
44
+ "CopybookParseError",
45
+ "DatasetRegistry",
46
+ "ExternalProgramRegistry",
47
+ "FieldSpec",
48
+ "JCLJob",
49
+ "JCLStep",
50
+ "PipelineContext",
51
+ "PipelineProgram",
52
+ "ProgramResult",
53
+ "ProgramReturnCodeError",
54
+ "PipelineRunError",
55
+ "PipelineRunner",
56
+ "RecordLayout",
57
+ "SortKey",
58
+ "WorkspaceInitResult",
59
+ "cobol_decimal",
60
+ "configure_s3",
61
+ "convert_sequential",
62
+ "decode_overpunch",
63
+ "decode_packed_decimal",
64
+ "encode_packed_decimal",
65
+ "external_program",
66
+ "is_numeric",
67
+ "init_workspace",
68
+ "iter_sequential",
69
+ "layout_to_dict",
70
+ "parse_copybook",
71
+ "parse_copybook_text",
72
+ "parse_jcl",
73
+ "quantize_cobol",
74
+ "read_csv",
75
+ "read_dataset",
76
+ "read_delimited_rows",
77
+ "read_parquet",
78
+ "read_s3_csv",
79
+ "read_s3_parquet",
80
+ "s3_uri",
81
+ "sort_dataframe",
82
+ "sort_records",
83
+ "synthetic_dataframe",
84
+ "synthetic_rows",
85
+ "write_csv",
86
+ "write_dataset",
87
+ "write_parquet",
88
+ "write_s3_csv",
89
+ "write_s3_parquet",
90
+ ]
91
+
92
+ __version__ = "0.2.0"
@@ -0,0 +1,5 @@
1
+ from .cli import main
2
+
3
+
4
+ raise SystemExit(main())
5
+
@@ -0,0 +1,30 @@
1
+ ---
2
+ name: analyze-mainframe-similarity
3
+ description: Cluster COBOL and JCL source by deterministic textual similarity, inspect every proposed cluster, and add an AI verdict and concise justification to JSON and CSV outputs. Use for duplicate or near-duplicate codebase analysis.
4
+ ---
5
+
6
+ # Analyze mainframe similarity
7
+
8
+ 1. Run the deterministic scan:
9
+
10
+ ```text
11
+ python -m mainframe_toolkit similarity scan <source-root> --threshold <threshold> --json similarity.json --csv similarity.csv --review-request similarity-review-request.json
12
+ ```
13
+
14
+ Use the requested threshold, or 80 when absent. Never edit deterministic scores or cluster membership manually.
15
+
16
+ 2. For each non-singleton cluster in the review request, read every original file and its normalized evidence. Check all pair relationships; connected-component chaining alone is not proof of equivalence. Ignore comments, sequence columns, copyright blocks, whitespace, generated names, and environment-only identifiers. Compare JCL execution flow/DD semantics or COBOL inputs, outputs, branches, calculations, side effects, error behavior, and record layouts.
17
+
18
+ 3. Assign exactly one verdict per requested cluster:
19
+
20
+ - `identico`: same executable semantics; differences are non-semantic.
21
+ - `similar`: materially shared flow/business implementation but at least one real semantic variation.
22
+ - `diferente`: superficial boilerplate or chaining grouped programs with different behavior.
23
+
24
+ 4. Write the requested review JSON with a short evidence-based Portuguese justification, then apply it:
25
+
26
+ ```text
27
+ python -m mainframe_toolkit similarity apply-review similarity.json similarity-review.json --json similarity-reviewed.json --csv similarity-reviewed.csv
28
+ ```
29
+
30
+ 5. Verify both reviewed files contain `parecer_ia` and `justificativa_ia` for every row. Keep singletons as `diferente` unless direct inspection establishes a duplicate omitted by the threshold.
@@ -0,0 +1,63 @@
1
+ ---
2
+ name: migrate-mainframe-job
3
+ description: Convert one specification-led JCL job and its COBOL programs to an incrementally validated PySpark pipeline with audited in-memory handoffs and final golden-dataset equality. Use when asked to migrate, continue, repair, or finish a mainframe job in this repository.
4
+ ---
5
+
6
+ # Migrate mainframe job
7
+
8
+ Migrate `$ARGUMENTS`, or the single job requested by the user. Do not widen the job scope.
9
+
10
+ ## Preflight
11
+
12
+ 1. Install the repo package if its import fails:
13
+
14
+ ```text
15
+ python -m pip install -e ".[all]"
16
+ ```
17
+
18
+ 2. Run `python -m mainframe_toolkit inspect . --output .migration-work/project-inspection.json`. Resolve structural errors before conversion; unresolved dependencies may proceed only through the synthetic/external rules below.
19
+ 3. Read the selected `spec/jcl/<JOB>-jcl.json` and source JCL completely. Reconcile their ordered steps, conditions, DD concatenation, SORT/IDCAMS control cards, input/output datasets, GDGs, and return-code behavior. The spec defines intended business behavior; source resolves mechanics the spec leaves open.
20
+ 4. Read every referenced `spec/cobol/<PROGRAM>-cbl.json`, COBOL source, used copybook, and relevant entry in `external-programs/*.json`. Do not infer a record layout from narrative text when a copybook exists. Distinguish physical COBOL width from any expanded textual transport width.
21
+ 5. Inventory every supplied input and golden file before generating anything. Freeze clock-dependent behavior behind an injected processing date.
22
+
23
+ ## Target shape
24
+
25
+ Create only the files needed under `converted/<JOB>/`:
26
+
27
+ ```text
28
+ <job_lower>.py
29
+ programs/__init__.py
30
+ programs/<program_lower>.py
31
+ tests/...
32
+ RELATORIO_FINAL.md # only after acceptance
33
+ ```
34
+
35
+ Create one module per COBOL program. Represent utility steps separately when they implement a transformation. Every module exposes `run(context)` and returns a mapping of logical dataset names to PySpark DataFrames, or a `ProgramResult` when a return code matters.
36
+
37
+ The JCL-named entrypoint contains no transformation, validation, lookup, calculation, filtering, joining, or formatting rules. It only initializes Spark and `PipelineContext`, imports modules, declares `PipelineProgram` order/conditions, loads runtime arguments, and calls `PipelineRunner`. It must accept `--through <PROGRAM>` so prefixes are executable.
38
+
39
+ Use `mainframe_toolkit.io` for local/S3 CSV and Parquet, `sequential`/`copybook` for fixed records, `sorting` for stable multi-key sort, `external` for subprogram adapters, and COBOL decimal helpers instead of cloning generic logic into modules.
40
+
41
+ ## Ordered conversion gate
42
+
43
+ For each executable step in JCL order:
44
+
45
+ 1. Implement only that program's behavior from its spec and source. Preserve COBOL decimal/truncation, key, mutation, EOF, return-code, and error semantics unless the specification explicitly overrides them.
46
+ 2. Add its import and `PipelineProgram` entry to the JCL-named file immediately.
47
+ 3. Run fast module tests, then execute the entire pipeline prefix with `--through <PROGRAM>` using supplied inputs.
48
+ 4. If a required input, dataset, clock value, or external implementation is absent, create the smallest deterministic substitute that makes semantic sense. Keep relationships valid, use a stable seed, place it under `converted/<JOB>/synthetic/`, and write a machine-readable provenance manifest. Never overwrite or shadow supplied data.
49
+ 5. Confirm the prefix has no import, analysis, Spark, schema, conversion, or runtime errors. Confirm each program output exists both in the in-memory registry and its audit path. Fix the current or earlier responsible module before moving to the next step.
50
+
51
+ Do not use the final golden comparison for a partial prefix unless that prefix is itself the documented final producer.
52
+
53
+ ## Final equivalence loop
54
+
55
+ 1. Execute the full pipeline with a clean run ID. Make final row ordering explicit using contract keys before export; Spark partition order is not a business order.
56
+ 2. Build the Java validator if needed, then invoke it through `python -m mainframe_toolkit golden validate ...` with the final output, matching golden file, format/delimiter, and ordering keys.
57
+ 3. Acceptance is `status=MATCH`, zero differing cells, equal row/column counts, and exit code 0.
58
+ 4. On mismatch, start from the first reported row/key/column. Trace that field backward through the per-program audit outputs and JCL data flow; identify the earliest program that creates the wrong value, row, multiplicity, or ordering. Correct that module, rerun its prefix, then rerun the full pipeline and validator.
59
+ 5. Repeat until 100% equal. Do not weaken comparison options, discard differing rows, alter the golden dataset, or encode golden values as business rules.
60
+
61
+ ## Final artifact
62
+
63
+ Only after acceptance, create `converted/<JOB>/RELATORIO_FINAL.md` with: programs converted in order; material spec/source decisions; supplied versus synthetic inputs and external adapters; tests/checkpoints executed; exact final comparison result; and the shortest commands needed to run the job and validator. Do not add deployment guidance.
@@ -0,0 +1,12 @@
1
+ ---
2
+ name: validate-golden-dataset
3
+ description: Compare a converted pipeline's final CSV or Parquet output cell by cell against its golden dataset and localize mismatches using audit lineage. Use for final equivalence checks or divergence repair.
4
+ ---
5
+
6
+ # Validate golden dataset
7
+
8
+ Build `validator-java` when its executable JAR is absent, then run `python -m mainframe_toolkit golden validate` for `$ARGUMENTS`. Supply the real delimiter/header/null/decimal options and explicit ordering keys whenever record order is not contractual.
9
+
10
+ Require equal schema, column order, row count, and normalized cell values. Exit code 0 and zero differences are the only passing result.
11
+
12
+ For a mismatch, use the report's first row/key/column and the audit manifest to trace the field backward through program outputs. Fix the earliest converted program that introduces the divergence, rerun its full prefix, then the full job and validator. Repeat without changing the golden dataset or relaxing comparison semantics.
@@ -0,0 +1,10 @@
1
+ # Mainframe migration toolkit
2
+
3
+ - Treat `spec/` as the intended behavior, the JCL/COBOL/copybook sources as the mechanical detail, and `golden-dataset/` as the final observable acceptance criterion. Record material conflicts in the final report.
4
+ - Convert a job strictly in JCL execution order. After each program, import it from the JCL-named entrypoint and execute the complete prefix through that program. Fix every syntax or runtime failure before continuing.
5
+ - Keep intermediate datasets as PySpark DataFrames in `PipelineContext`; never use an audit file as the handoff between programs. Publish every program output so the runtime also materializes an audit copy.
6
+ - Put business rules only in program modules. The JCL-named entrypoint may initialize Spark/context, import programs, declare their order/conditions, parse runtime arguments, and run the pipeline.
7
+ - Generate deterministic synthetic data or an external-program adapter only for an unavailable dependency. Never replace supplied data. Preserve referential integrity and identify every synthetic resource in the final report.
8
+ - Completion requires a full run and 100% cell-level equality with the golden dataset. Use explicit ordering keys when row order is not part of the contract. Trace mismatches to the responsible program, fix it, and rerun until equal.
9
+ - Do not add deployment, infrastructure, production hardening, architecture prose, README files, or per-step reports. Generate only functional core code, tests, audit artifacts, and the final `RELATORIO_FINAL.md` for each converted job.
10
+
@@ -0,0 +1,2 @@
1
+ @AGENTS.md
2
+
@@ -0,0 +1,3 @@
1
+ wrapperVersion=3.3.4
2
+ distributionType=only-script
3
+ distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip
@@ -0,0 +1,295 @@
1
+ #!/bin/sh
2
+ # ----------------------------------------------------------------------------
3
+ # Licensed to the Apache Software Foundation (ASF) under one
4
+ # or more contributor license agreements. See the NOTICE file
5
+ # distributed with this work for additional information
6
+ # regarding copyright ownership. The ASF licenses this file
7
+ # to you under the Apache License, Version 2.0 (the
8
+ # "License"); you may not use this file except in compliance
9
+ # with the License. You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing,
14
+ # software distributed under the License is distributed on an
15
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16
+ # KIND, either express or implied. See the License for the
17
+ # specific language governing permissions and limitations
18
+ # under the License.
19
+ # ----------------------------------------------------------------------------
20
+
21
+ # ----------------------------------------------------------------------------
22
+ # Apache Maven Wrapper startup batch script, version 3.3.4
23
+ #
24
+ # Optional ENV vars
25
+ # -----------------
26
+ # JAVA_HOME - location of a JDK home dir, required when download maven via java source
27
+ # MVNW_REPOURL - repo url base for downloading maven distribution
28
+ # MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven
29
+ # MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output
30
+ # ----------------------------------------------------------------------------
31
+
32
+ set -euf
33
+ [ "${MVNW_VERBOSE-}" != debug ] || set -x
34
+
35
+ # OS specific support.
36
+ native_path() { printf %s\\n "$1"; }
37
+ case "$(uname)" in
38
+ CYGWIN* | MINGW*)
39
+ [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")"
40
+ native_path() { cygpath --path --windows "$1"; }
41
+ ;;
42
+ esac
43
+
44
+ # set JAVACMD and JAVACCMD
45
+ set_java_home() {
46
+ # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched
47
+ if [ -n "${JAVA_HOME-}" ]; then
48
+ if [ -x "$JAVA_HOME/jre/sh/java" ]; then
49
+ # IBM's JDK on AIX uses strange locations for the executables
50
+ JAVACMD="$JAVA_HOME/jre/sh/java"
51
+ JAVACCMD="$JAVA_HOME/jre/sh/javac"
52
+ else
53
+ JAVACMD="$JAVA_HOME/bin/java"
54
+ JAVACCMD="$JAVA_HOME/bin/javac"
55
+
56
+ if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then
57
+ echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2
58
+ echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2
59
+ return 1
60
+ fi
61
+ fi
62
+ else
63
+ JAVACMD="$(
64
+ 'set' +e
65
+ 'unset' -f command 2>/dev/null
66
+ 'command' -v java
67
+ )" || :
68
+ JAVACCMD="$(
69
+ 'set' +e
70
+ 'unset' -f command 2>/dev/null
71
+ 'command' -v javac
72
+ )" || :
73
+
74
+ if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then
75
+ echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2
76
+ return 1
77
+ fi
78
+ fi
79
+ }
80
+
81
+ # hash string like Java String::hashCode
82
+ hash_string() {
83
+ str="${1:-}" h=0
84
+ while [ -n "$str" ]; do
85
+ char="${str%"${str#?}"}"
86
+ h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296))
87
+ str="${str#?}"
88
+ done
89
+ printf %x\\n $h
90
+ }
91
+
92
+ verbose() { :; }
93
+ [ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; }
94
+
95
+ die() {
96
+ printf %s\\n "$1" >&2
97
+ exit 1
98
+ }
99
+
100
+ trim() {
101
+ # MWRAPPER-139:
102
+ # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds.
103
+ # Needed for removing poorly interpreted newline sequences when running in more
104
+ # exotic environments such as mingw bash on Windows.
105
+ printf "%s" "${1}" | tr -d '[:space:]'
106
+ }
107
+
108
+ scriptDir="$(dirname "$0")"
109
+ scriptName="$(basename "$0")"
110
+
111
+ # parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties
112
+ while IFS="=" read -r key value; do
113
+ case "${key-}" in
114
+ distributionUrl) distributionUrl=$(trim "${value-}") ;;
115
+ distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;;
116
+ esac
117
+ done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties"
118
+ [ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties"
119
+
120
+ case "${distributionUrl##*/}" in
121
+ maven-mvnd-*bin.*)
122
+ MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/
123
+ case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in
124
+ *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;;
125
+ :Darwin*x86_64) distributionPlatform=darwin-amd64 ;;
126
+ :Darwin*arm64) distributionPlatform=darwin-aarch64 ;;
127
+ :Linux*x86_64*) distributionPlatform=linux-amd64 ;;
128
+ *)
129
+ echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2
130
+ distributionPlatform=linux-amd64
131
+ ;;
132
+ esac
133
+ distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip"
134
+ ;;
135
+ maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;;
136
+ *) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;;
137
+ esac
138
+
139
+ # apply MVNW_REPOURL and calculate MAVEN_HOME
140
+ # maven home pattern: ~/.m2/wrapper/dists/{apache-maven-<version>,maven-mvnd-<version>-<platform>}/<hash>
141
+ [ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}"
142
+ distributionUrlName="${distributionUrl##*/}"
143
+ distributionUrlNameMain="${distributionUrlName%.*}"
144
+ distributionUrlNameMain="${distributionUrlNameMain%-bin}"
145
+ MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}"
146
+ MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")"
147
+
148
+ exec_maven() {
149
+ unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || :
150
+ exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD"
151
+ }
152
+
153
+ if [ -d "$MAVEN_HOME" ]; then
154
+ verbose "found existing MAVEN_HOME at $MAVEN_HOME"
155
+ exec_maven "$@"
156
+ fi
157
+
158
+ case "${distributionUrl-}" in
159
+ *?-bin.zip | *?maven-mvnd-?*-?*.zip) ;;
160
+ *) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;;
161
+ esac
162
+
163
+ # prepare tmp dir
164
+ if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then
165
+ clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; }
166
+ trap clean HUP INT TERM EXIT
167
+ else
168
+ die "cannot create temp dir"
169
+ fi
170
+
171
+ mkdir -p -- "${MAVEN_HOME%/*}"
172
+
173
+ # Download and Install Apache Maven
174
+ verbose "Couldn't find MAVEN_HOME, downloading and installing it ..."
175
+ verbose "Downloading from: $distributionUrl"
176
+ verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName"
177
+
178
+ # select .zip or .tar.gz
179
+ if ! command -v unzip >/dev/null; then
180
+ distributionUrl="${distributionUrl%.zip}.tar.gz"
181
+ distributionUrlName="${distributionUrl##*/}"
182
+ fi
183
+
184
+ # verbose opt
185
+ __MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR=''
186
+ [ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v
187
+
188
+ # normalize http auth
189
+ case "${MVNW_PASSWORD:+has-password}" in
190
+ '') MVNW_USERNAME='' MVNW_PASSWORD='' ;;
191
+ has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;;
192
+ esac
193
+
194
+ if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then
195
+ verbose "Found wget ... using wget"
196
+ wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl"
197
+ elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then
198
+ verbose "Found curl ... using curl"
199
+ curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl"
200
+ elif set_java_home; then
201
+ verbose "Falling back to use Java to download"
202
+ javaSource="$TMP_DOWNLOAD_DIR/Downloader.java"
203
+ targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName"
204
+ cat >"$javaSource" <<-END
205
+ public class Downloader extends java.net.Authenticator
206
+ {
207
+ protected java.net.PasswordAuthentication getPasswordAuthentication()
208
+ {
209
+ return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() );
210
+ }
211
+ public static void main( String[] args ) throws Exception
212
+ {
213
+ setDefault( new Downloader() );
214
+ java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() );
215
+ }
216
+ }
217
+ END
218
+ # For Cygwin/MinGW, switch paths to Windows format before running javac and java
219
+ verbose " - Compiling Downloader.java ..."
220
+ "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java"
221
+ verbose " - Running Downloader.java ..."
222
+ "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")"
223
+ fi
224
+
225
+ # If specified, validate the SHA-256 sum of the Maven distribution zip file
226
+ if [ -n "${distributionSha256Sum-}" ]; then
227
+ distributionSha256Result=false
228
+ if [ "$MVN_CMD" = mvnd.sh ]; then
229
+ echo "Checksum validation is not supported for maven-mvnd." >&2
230
+ echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
231
+ exit 1
232
+ elif command -v sha256sum >/dev/null; then
233
+ if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then
234
+ distributionSha256Result=true
235
+ fi
236
+ elif command -v shasum >/dev/null; then
237
+ if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then
238
+ distributionSha256Result=true
239
+ fi
240
+ else
241
+ echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2
242
+ echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2
243
+ exit 1
244
+ fi
245
+ if [ $distributionSha256Result = false ]; then
246
+ echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2
247
+ echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2
248
+ exit 1
249
+ fi
250
+ fi
251
+
252
+ # unzip and move
253
+ if command -v unzip >/dev/null; then
254
+ unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip"
255
+ else
256
+ tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar"
257
+ fi
258
+
259
+ # Find the actual extracted directory name (handles snapshots where filename != directory name)
260
+ actualDistributionDir=""
261
+
262
+ # First try the expected directory name (for regular distributions)
263
+ if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then
264
+ if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then
265
+ actualDistributionDir="$distributionUrlNameMain"
266
+ fi
267
+ fi
268
+
269
+ # If not found, search for any directory with the Maven executable (for snapshots)
270
+ if [ -z "$actualDistributionDir" ]; then
271
+ # enable globbing to iterate over items
272
+ set +f
273
+ for dir in "$TMP_DOWNLOAD_DIR"/*; do
274
+ if [ -d "$dir" ]; then
275
+ if [ -f "$dir/bin/$MVN_CMD" ]; then
276
+ actualDistributionDir="$(basename "$dir")"
277
+ break
278
+ fi
279
+ fi
280
+ done
281
+ set -f
282
+ fi
283
+
284
+ if [ -z "$actualDistributionDir" ]; then
285
+ verbose "Contents of $TMP_DOWNLOAD_DIR:"
286
+ verbose "$(ls -la "$TMP_DOWNLOAD_DIR")"
287
+ die "Could not find Maven distribution directory in extracted archive"
288
+ fi
289
+
290
+ verbose "Found extracted Maven distribution directory: $actualDistributionDir"
291
+ printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url"
292
+ mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME"
293
+
294
+ clean || :
295
+ exec_maven "$@"