sas2parquet 0.1.2__tar.gz → 0.1.6__tar.gz

{sas2parquet-0.1.2 → sas2parquet-0.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sas2parquet
-Version: 0.1.2
+Version: 0.1.6
 Summary: SAS → Parquet Hybrid Converter & Validator
 License-File: LICENSE
 Author: Zaman Ziabakhshganji

{sas2parquet-0.1.2 → sas2parquet-0.1.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sas2parquet"
-version = "0.1.2"
+version = "0.1.6"
 description = "SAS → Parquet Hybrid Converter & Validator"
 authors = [
     {name = "Zaman Ziabakhshganji",email = "zaman.ganji@gmail.com"}

{sas2parquet-0.1.2 → sas2parquet-0.1.6}/src/sas2parquet/convert.py

@@ -317,7 +317,27 @@ def reconvert_file_ultimate(sas_path: Path, parquet_path: Path) -> bool:
             writer.close()
             print("   ✅ Conversion succeeded")
 
+            # ===== FULL PARQUET VALIDATION (WORKING) =====
+            print("   🔍 Full Parquet validation...")
+            try:
+                pf = pq.ParquetFile(parquet_path)
+                total_rows = 0
+                num_groups = pf.metadata.num_row_groups
+                batch_count = 0
+                
+                for batch in pf.iter_batches():
+                    total_rows += batch.num_rows
+                    batch_count += 1
+                
+                print(f"   ✅ Parquet fully validated: {total_rows:,} rows across {num_groups} groups ({batch_count} batches)")
+                pf.close()
+            except Exception as e:
+                print(f"   ❌ Parquet validation failed: {e}")
+                return False
+            # ===== END =====
+
             st, dt = compare_and_report_diffs(sas_path, parquet_path)
+
             print(f"   🔍 Validation: {st}")
             for d in dt:
                 print("     -", d.replace("\n", "\n       "))
@@ -366,8 +386,14 @@ def main():
             print(f"\n🗂 Processing: {rel}")
 
             # Mirror structure under parquetdata/ (which lives next to sasdata/)
-            pqf = (PARQUET_INPUT_DIR / rel).with_suffix('.parquet')
 
+            if rel.parent == Path("."):
+                pqf = (PARQUET_INPUT_DIR / rel.name).with_suffix(".parquet")
+            else:
+                parquet_dirs = [f"{p}_parquet" for p in rel.parent.parts]
+                pqf = (PARQUET_INPUT_DIR.joinpath(*parquet_dirs) / rel.name).with_suffix(".parquet")
+           
+            
             reconvert_file_ultimate(sas, pqf)
             print("-"*60)