sas2parquet 0.1.1__tar.gz → 0.1.5__tar.gz

{sas2parquet-0.1.1 → sas2parquet-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sas2parquet
-Version: 0.1.1
+Version: 0.1.5
 Summary: SAS → Parquet Hybrid Converter & Validator
 License-File: LICENSE
 Author: Zaman Ziabakhshganji
@@ -128,37 +128,6 @@ Each file undergoes 4-stage validation:
 3. Column order
 4. Value comparison
 
-## 💻 Development
-```bash
-git clone <repo>
-cd sas2parquet
-pip install -e .
-```
-
-
-## 🧪 Testing
-
-This project includes a comprehensive **pytest** test suite covering:
-
-- Schema inference and overrides
-- SAS ↔ Parquet validation logic
-- Error handling and retry behavior
-- Directory mirroring (`sasdata/ → parquetdata/`)
-- Logging and CLI execution paths
-
-### Run tests
-
-```bash
-pip install -e .[dev]
-pytest -q
-```
-
-## 📦 Dependencies
-- pandas>=2.0
-- polars>=0.20
-- pyarrow>=15.0
-- pyreadstat>=1.4
-
 ## 📄 License
 MIT License
 

{sas2parquet-0.1.1 → sas2parquet-0.1.5}/README.md

@@ -98,36 +98,5 @@ Each file undergoes 4-stage validation:
 3. Column order
 4. Value comparison
 
-## 💻 Development
-```bash
-git clone <repo>
-cd sas2parquet
-pip install -e .
-```
-
-
-## 🧪 Testing
-
-This project includes a comprehensive **pytest** test suite covering:
-
-- Schema inference and overrides
-- SAS ↔ Parquet validation logic
-- Error handling and retry behavior
-- Directory mirroring (`sasdata/ → parquetdata/`)
-- Logging and CLI execution paths
-
-### Run tests
-
-```bash
-pip install -e .[dev]
-pytest -q
-```
-
-## 📦 Dependencies
-- pandas>=2.0
-- polars>=0.20
-- pyarrow>=15.0
-- pyreadstat>=1.4
-
 ## 📄 License
 MIT License

{sas2parquet-0.1.1 → sas2parquet-0.1.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sas2parquet"
-version = "0.1.1"
+version = "0.1.5"
 description = "SAS → Parquet Hybrid Converter & Validator"
 authors = [
     {name = "Zaman Ziabakhshganji",email = "zaman.ganji@gmail.com"}

{sas2parquet-0.1.1 → sas2parquet-0.1.5}/src/sas2parquet/convert.py

@@ -317,7 +317,27 @@ def reconvert_file_ultimate(sas_path: Path, parquet_path: Path) -> bool:
             writer.close()
             print("   ✅ Conversion succeeded")
 
+            # ===== FULL PARQUET VALIDATION (WORKING) =====
+            print("   🔍 Full Parquet validation...")
+            try:
+                pf = pq.ParquetFile(parquet_path)
+                total_rows = 0
+                num_groups = pf.metadata.num_row_groups
+                batch_count = 0
+                
+                for batch in pf.iter_batches():
+                    total_rows += batch.num_rows
+                    batch_count += 1
+                
+                print(f"   ✅ Parquet fully validated: {total_rows:,} rows across {num_groups} groups ({batch_count} batches)")
+                pf.close()
+            except Exception as e:
+                print(f"   ❌ Parquet validation failed: {e}")
+                return False
+            # ===== END =====
+
             st, dt = compare_and_report_diffs(sas_path, parquet_path)
+
             print(f"   🔍 Validation: {st}")
             for d in dt:
                 print("     -", d.replace("\n", "\n       "))