sas2parquet 0.1.6__tar.gz → 0.1.9__tar.gz

{sas2parquet-0.1.6 → sas2parquet-0.1.9}/PKG-INFO

@@ -1,11 +1,15 @@
 Metadata-Version: 2.4
 Name: sas2parquet
-Version: 0.1.6
+Version: 0.1.9
 Summary: SAS → Parquet Hybrid Converter & Validator
+License: MIT
 License-File: LICENSE
+Keywords: sas,parquet,etl,data,pyarrow
 Author: Zaman Ziabakhshganji
 Author-email: zaman.ganji@gmail.com
-Requires-Python: >=3.11
+Requires-Python: >=3.11,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -15,17 +19,15 @@ Requires-Dist: narwhals (==2.13.0)
 Requires-Dist: numpy (==2.3.5)
 Requires-Dist: pandas (==2.3.3)
 Requires-Dist: polars (==1.36.1)
-Requires-Dist: polars-runtime-32 (==1.36.1)
-Requires-Dist: py4j (==0.10.9.9)
 Requires-Dist: pyarrow (==22.0.0)
 Requires-Dist: pyreadstat (==1.3.2)
-Requires-Dist: pyspark (==4.0.1)
-Requires-Dist: pytest (>=9.0.2,<10.0.0)
 Requires-Dist: python-dateutil (==2.9.0.post0)
 Requires-Dist: pytz (==2025.2)
 Requires-Dist: requests (>=2.32.5,<3.0.0)
 Requires-Dist: six (==1.17.0)
 Requires-Dist: tzdata (==2025.2)
+Project-URL: Homepage, https://github.com/<you>/<repo>
+Project-URL: Repository, https://github.com/<you>/<repo>
 Description-Content-Type: text/markdown
 
 # sas2parquet
@@ -75,13 +77,15 @@ sas2parquet --dir-mode
 
 ```text
 your-project/
-├── sasdata/
+├── sasdata/              # ← Put your .sas7bdat files here
 │   ├── file1.sas7bdat
-│   └── nested.sas7bdat
-├── parquetdata/
+│   └── subfolder/
+│       └── nested.sas7bdat
+├── parquetdata/          # ← AUTO-CREATED (mirrors sasdata/)
 │   ├── file1.parquet
-│   └── nested.parquet
-└── logging/
+│   └── subfolder/
+│       └── nested.parquet
+└── logging/             # ← AUTO-CREATED (detailed logs)
     └── conversion_20260205_1145.log
 ```
 

{sas2parquet-0.1.6 → sas2parquet-0.1.9}/README.md

@@ -45,13 +45,15 @@ sas2parquet --dir-mode
 
 ```text
 your-project/
-├── sasdata/
+├── sasdata/              # ← Put your .sas7bdat files here
 │   ├── file1.sas7bdat
-│   └── nested.sas7bdat
-├── parquetdata/
+│   └── subfolder/
+│       └── nested.sas7bdat
+├── parquetdata/          # ← AUTO-CREATED (mirrors sasdata/)
 │   ├── file1.parquet
-│   └── nested.parquet
-└── logging/
+│   └── subfolder/
+│       └── nested.parquet
+└── logging/             # ← AUTO-CREATED (detailed logs)
     └── conversion_20260205_1145.log
 ```
 

sas2parquet-0.1.9/pyproject.toml

@@ -0,0 +1,48 @@
+[tool.poetry]
+name = "sas2parquet"
+version = "0.1.9"
+description = "SAS → Parquet Hybrid Converter & Validator"
+readme = "README.md"
+authors = ["Zaman Ziabakhshganji <zaman.ganji@gmail.com>"]
+license = "MIT"
+packages = [{ include = "sas2parquet", from = "src" }]
+# Optional but nice for PyPI:
+repository = "https://github.com/<you>/<repo>"
+homepage = "https://github.com/<you>/<repo>"
+keywords = ["sas", "parquet", "etl", "data", "pyarrow"]
+classifiers = [
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
+]
+
+[tool.poetry.dependencies]
+python = ">=3.11,<4.0"
+requests = ">=2.32.5,<3.0.0"
+narwhals = "==2.13.0"
+numpy = "==2.3.5"
+pandas = "==2.3.3"
+polars = "==1.36.1"
+pyarrow = "==22.0.0"
+pyreadstat = "==1.3.2"
+python-dateutil = "==2.9.0.post0"
+pytz = "==2025.2"
+six = "==1.17.0"
+tzdata = "==2025.2"
+
+# 🚫 Strongly consider NOT shipping these as required deps:
+# pyspark, py4j, polars-runtime-32
+# They dramatically inflate installs and aren't required for your conversion script.
+# If you still want them, put them behind extras (see below).
+
+[tool.poetry.group.dev.dependencies]
+pytest = ">=9.0.2,<10.0.0"
+
+[tool.poetry.scripts]
+sas2parquet = "sas2parquet.cli:main"
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"

sas2parquet-0.1.9/src/sas2parquet/cli.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+"""CLI entrypoint for sas2parquet."""
+import argparse
+import sys
+from pathlib import Path
+import importlib.metadata
+
+try:
+    __version__ = importlib.metadata.version("sas2parquet")
+except importlib.metadata.PackageNotFoundError:
+    __version__ = "dev"
+
+from .convert import main as convert_dir, reconvert_file_ultimate
+
+
+def main():
+    parser = argparse.ArgumentParser(prog="sas2parquet", description="SAS to Parquet converter")
+    parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
+
+    parser.add_argument(
+        "path",
+        help="Path to a .sas7bdat file OR a directory containing SAS files (recursively)."
+    )
+    parser.add_argument(
+        "--out", "-o",
+        help="Output Parquet file (file mode) OR output directory (dir mode). "
+             "If omitted, dir mode uses sibling 'parquetdata/'.",
+        default=None
+    )
+    parser.add_argument(
+        "--log-dir",
+        help="Directory where logs are written (dir mode). If omitted, uses sibling 'logging/'.",
+        default=None
+    )
+
+    args = parser.parse_args()
+
+    p = Path(args.path).expanduser().resolve()
+    if not p.exists():
+        print(f"❌ Path not found: {p}")
+        sys.exit(2)
+
+    # Directory mode
+    if p.is_dir():
+        out_dir = Path(args.out).expanduser().resolve() if args.out else None
+        log_dir = Path(args.log_dir).expanduser().resolve() if args.log_dir else None
+        rc = convert_dir(p, parquet_output_dir=out_dir, log_dir=log_dir)
+        sys.exit(rc)
+
+    # File mode
+    if p.is_file():
+        if p.suffix.lower() != ".sas7bdat":
+            print(f"❌ Not a .sas7bdat file: {p.name}")
+            sys.exit(2)
+
+        if args.out:
+            out_file = Path(args.out).expanduser().resolve()
+        else:
+            out_file = p.with_suffix(".parquet")
+
+        success = reconvert_file_ultimate(p, out_file)
+        sys.exit(0 if success else 1)
+
+    print(f"❌ Unsupported path type: {p}")
+    sys.exit(2)
+
+
+if __name__ == "__main__":
+    main()

{sas2parquet-0.1.6 → sas2parquet-0.1.9}/src/sas2parquet/convert.py

@@ -12,19 +12,8 @@ import pyarrow as pa
 import pyarrow.parquet as pq
 import pyreadstat
 
-# --- Suppress pandas FutureWarnings ---
 warnings.simplefilter(action='ignore', category=FutureWarning)
 
-# --- Configuration ---
-# Put your .sas7bdat files inside SAS_INPUT_DIR (including subfolders).
-SAS_INPUT_DIR = Path("sasdata")
-
-# IMPORTANT:
-# parquetdata/ and logging/ are created NEXT TO sasdata/ (i.e., in the same parent directory).
-PARQUET_INPUT_DIR = SAS_INPUT_DIR.parent / "parquetdata"
-LOG_DIR = SAS_INPUT_DIR.parent / "logging"
-LOG_FILE_PATH = LOG_DIR / f"conversion_{datetime.now():%Y%m%d_%H%M%S}.log"
-
 KNOWN_DATETIME_COLUMNS = [
     'RPNA_DATE_UTC','RPNA_TIME_UTC','RPA_DATE_UTC','TIMESTAMP_UTC',
     'EVENT_START_DATE_UTC','EVENT_END_DATE_UTC',
@@ -44,13 +33,10 @@ MAX_CHUNK_SIZE = 10_000_000
 
 # --- Logger ---
 class Logger:
-    def __init__(self, path):
+    def __init__(self, path: Path):
         self.terminal = sys.stdout
-
-        # Ensure log folder exists
         path = Path(path)
         path.parent.mkdir(parents=True, exist_ok=True)
-
         self.logfile = open(path, 'w', encoding='utf-8')
 
     def write(self, msg):
@@ -111,7 +97,6 @@ def compare_and_report_diffs(sas_path: Path, parquet_path: Path):
         pq_it  = pq.ParquetFile(parquet_path).iter_batches(batch_size=CH)
         chunk_i = 0
 
-        # SAS→UNIX epoch offset µs
         offset_us = int((pd.Timestamp("1970-01-01") -
                          pd.Timestamp("1960-01-01")).total_seconds() * 1e6)
 
@@ -146,7 +131,6 @@ def compare_and_report_diffs(sas_path: Path, parquet_path: Path):
                 pcol = ppq.get_column(col)
                 ds, dp = scol.dtype, pcol.dtype
                 if ds in num_types and dp in num_types:
-                    # unify int<->float
                     if ds in float_types and dp in int_types:
                         pcol = pcol.cast(ds)
                     elif dp in float_types and ds in int_types:
@@ -162,7 +146,6 @@ def compare_and_report_diffs(sas_path: Path, parquet_path: Path):
                 sser = psas.get_column(col)
                 pser = ppq.get_column(col)
 
-                # epoch check
                 if sser.dtype == pl.Datetime("us") and pser.dtype == pl.Datetime("us"):
                     raw = sas_chunk[col]
                     if pd.api.types.is_datetime64_ns_dtype(raw):
@@ -180,7 +163,6 @@ def compare_and_report_diffs(sas_path: Path, parquet_path: Path):
                         ))
                         continue
 
-                # string compare (with date-only normalization)
                 s_str = sser.cast(pl.Utf8)
                 p_str = pser.cast(pl.Utf8)
                 mask  = (s_str != p_str) | (s_str.is_null() != p_str.is_null())
@@ -209,7 +191,6 @@ def compare_and_report_diffs(sas_path: Path, parquet_path: Path):
 def reconvert_file_ultimate(sas_path: Path, parquet_path: Path) -> bool:
     print(f"🛠️ Fixing {sas_path.name}...")
 
-    # 1) metadata & encoding
     _, meta0 = pyreadstat.read_sas7bdat(sas_path, metadataonly=True)
     enc0 = getattr(meta0, 'file_encoding', None)
     if enc0:
@@ -232,13 +213,12 @@ def reconvert_file_ultimate(sas_path: Path, parquet_path: Path) -> bool:
     cols = meta0.column_names
     read_types = getattr(meta0, 'readstat_variable_types', {}) or {}
 
-    # SAS formats if available
     fmt_map = {}
     if hasattr(meta0, 'formats'):
         for name, fmt in zip(meta0.column_names, meta0.formats):
             fmt_map[name] = fmt or ""
 
-    # infer content types from first few chunks
+    # infer content types
     content, inf, cnt = {}, {}, 0
     it = pd.read_sas(sas_path, chunksize=MIN_CHUNK_SIZE, encoding=encoding)
     for chunk in it:
@@ -267,17 +247,14 @@ def reconvert_file_ultimate(sas_path: Path, parquet_path: Path) -> bool:
         print(f"   Attempt {attempt}…")
         fields = []
         for c in cols:
-            # 1) SAS-declared numeric → float64
             if read_types.get(c) == 'double':
                 at = pa.float64()
             else:
                 cu = c.upper()
-                # 2) forced-string
                 if cu in {x.upper() for x in COLUMNS_TO_FORCE_AS_STRING}:
                     at = pa.string()
                 else:
                     fmt = fmt_map.get(c, "").upper()
-                    # 3) datetime/date/time
                     if (cu in {x.upper() for x in KNOWN_DATETIME_COLUMNS}
                         or any(x in fmt for x in ('DATE', 'TIME', 'DATETIME'))):
                         if 'DATE' in fmt and 'DATETIME' not in fmt:
@@ -286,14 +263,11 @@ def reconvert_file_ultimate(sas_path: Path, parquet_path: Path) -> bool:
                             at = pa.time64('ms')
                         else:
                             at = pa.timestamp('ms')
-                    # 4) fallback
                     else:
                         at = pa.string()
 
-            # apply any dynamic override
             if c in overrides:
                 at = overrides[c]
-
             fields.append(pa.field(c, at))
 
         schema = pa.schema(fields)
@@ -317,27 +291,22 @@ def reconvert_file_ultimate(sas_path: Path, parquet_path: Path) -> bool:
             writer.close()
             print("   ✅ Conversion succeeded")
 
-            # ===== FULL PARQUET VALIDATION (WORKING) =====
             print("   🔍 Full Parquet validation...")
             try:
                 pf = pq.ParquetFile(parquet_path)
                 total_rows = 0
                 num_groups = pf.metadata.num_row_groups
                 batch_count = 0
-                
                 for batch in pf.iter_batches():
                     total_rows += batch.num_rows
                     batch_count += 1
-                
                 print(f"   ✅ Parquet fully validated: {total_rows:,} rows across {num_groups} groups ({batch_count} batches)")
                 pf.close()
             except Exception as e:
                 print(f"   ❌ Parquet validation failed: {e}")
                 return False
-            # ===== END =====
 
             st, dt = compare_and_report_diffs(sas_path, parquet_path)
-
             print(f"   🔍 Validation: {st}")
             for d in dt:
                 print("     -", d.replace("\n", "\n       "))
@@ -369,39 +338,78 @@ def reconvert_file_ultimate(sas_path: Path, parquet_path: Path) -> bool:
     return False
 
 
-# --- Main loop ---
-def main():
+def default_parquet_dir_for(sas_input_dir: Path) -> Path:
+    # sibling parquetdata/ next to sas_input_dir
+    return sas_input_dir.parent / "parquetdata"
+
+
+def default_log_dir_for(sas_input_dir: Path) -> Path:
+    # sibling logging/ next to sas_input_dir
+    return sas_input_dir.parent / "logging"
+
+
+def parquet_path_for_sas(sas_file: Path, sas_input_dir: Path, parquet_output_dir: Path) -> Path:
+    rel = sas_file.relative_to(sas_input_dir)
+
+    if rel.parent == Path("."):
+        return (parquet_output_dir / rel.name).with_suffix(".parquet")
+
+    parquet_dirs = [f"{p}_parquet" for p in rel.parent.parts]
+    return (parquet_output_dir.joinpath(*parquet_dirs) / rel.name).with_suffix(".parquet")
+
+
+# --- Main loop (directory mode) ---
+def main(
+    sas_input_dir: Path,
+    parquet_output_dir: Path | None = None,
+    log_dir: Path | None = None,
+) -> int:
+    sas_input_dir = Path(sas_input_dir).expanduser().resolve()
+    if not sas_input_dir.exists() or not sas_input_dir.is_dir():
+        print(f"❌ Input directory not found or not a directory: {sas_input_dir}")
+        return 2
+
+    parquet_output_dir = (Path(parquet_output_dir).expanduser().resolve()
+                          if parquet_output_dir else default_parquet_dir_for(sas_input_dir))
+    log_dir = (Path(log_dir).expanduser().resolve()
+               if log_dir else default_log_dir_for(sas_input_dir))
+
+    log_file_path = log_dir / f"conversion_{datetime.now():%Y%m%d_%H%M%S}.log"
+
     orig = sys.stdout
-    sys.stdout = Logger(LOG_FILE_PATH)
+    sys.stdout = Logger(log_file_path)
     try:
         print("🚀 SAS → Parquet Hybrid Fix & Validate (full folder)\n")
-        files = list(SAS_INPUT_DIR.rglob("*.sas7bdat"))
+        print(f"Input:   {sas_input_dir}")
+        print(f"Output:  {parquet_output_dir}")
+        print(f"Logs:    {log_file_path}\n")
+
+        files = list(sas_input_dir.rglob("*.sas7bdat"))
         if not files:
             print("❌ No SAS files found. Exiting.")
-            return
+            return 1
 
         print(f"Found {len(files)} files.\n" + "="*60)
+        ok = 0
+        bad = 0
+
         for sas in files:
-            rel = sas.relative_to(SAS_INPUT_DIR)
+            rel = sas.relative_to(sas_input_dir)
             print(f"\n🗂 Processing: {rel}")
 
-            # Mirror structure under parquetdata/ (which lives next to sasdata/)
+            pqf = parquet_path_for_sas(sas, sas_input_dir, parquet_output_dir)
+            success = reconvert_file_ultimate(sas, pqf)
 
-            if rel.parent == Path("."):
-                pqf = (PARQUET_INPUT_DIR / rel.name).with_suffix(".parquet")
+            if success:
+                ok += 1
             else:
-                parquet_dirs = [f"{p}_parquet" for p in rel.parent.parts]
-                pqf = (PARQUET_INPUT_DIR.joinpath(*parquet_dirs) / rel.name).with_suffix(".parquet")
-           
-            
-            reconvert_file_ultimate(sas, pqf)
+                bad += 1
+
             print("-"*60)
 
-        print("\n✅ All done. See log at:", LOG_FILE_PATH)
+        print(f"\n✅ Done. Success={ok}, Failed={bad}. See log at: {log_file_path}")
+        return 0 if bad == 0 else 1
+
     finally:
         sys.stdout.close()
-        sys.stdout = orig
-
-
-if __name__ == "__main__":
-    main()
+        sys.stdout = orig

sas2parquet-0.1.6/pyproject.toml

@@ -1,37 +0,0 @@
-[project]
-name = "sas2parquet"
-version = "0.1.6"
-description = "SAS → Parquet Hybrid Converter & Validator"
-authors = [
-    {name = "Zaman Ziabakhshganji",email = "zaman.ganji@gmail.com"}
-]
-readme = "README.md"
-requires-python = ">=3.11"
-dependencies = [
-    "pytest (>=9.0.2,<10.0.0)", 
-    "requests (>=2.32.5,<3.0.0)", 
-    "narwhals (==2.13.0)", 
-    "numpy (==2.3.5)", 
-    "pandas (==2.3.3)", 
-    "polars (==1.36.1)", 
-    "polars-runtime-32 (==1.36.1)", 
-    "py4j (==0.10.9.9)", 
-    "pyarrow (==22.0.0)", 
-    "pyreadstat (==1.3.2)", 
-    "pyspark (==4.0.1)", 
-    "python-dateutil (==2.9.0.post0)", 
-    "pytz (==2025.2)", 
-    "six (==1.17.0)", 
-    "tzdata (==2025.2)",
-]
-
-[tool.poetry]
-packages = [{include = "sas2parquet", from = "src"}]
-
-[tool.poetry.scripts]
-sas2parquet = "sas2parquet.cli:main"
-
-[build-system]
-requires = ["poetry-core>=2.0.0,<3.0.0"]
-build-backend = "poetry.core.masonry.api"
-

sas2parquet-0.1.6/src/sas2parquet/cli.py

@@ -1,24 +0,0 @@
-#!/usr/bin/env python
-"""CLI entrypoint for sas2parquet."""
-import argparse
-import sys
-from pathlib import Path
-from .convert import main as _convert_main  # Import your existing main()
-
-def main():
-    parser = argparse.ArgumentParser(description="SAS to Parquet converter")
-    parser.add_argument("sas_file", nargs="?", help="Single SAS file to convert")
-    parser.add_argument("parquet_file", nargs="?", help="Output Parquet file")
-    parser.add_argument("--dir-mode", action="store_true", 
-                       help="Process entire SAS_INPUT_DIR (ignores file args)")
-    
-    args = parser.parse_args()
-    
-    # Patch sys.argv for your convert.main() if single file mode
-    if args.sas_file and not args.dir_mode:
-        sys.argv = [sys.argv[0], str(Path(args.sas_file)), str(Path(args.parquet_file))]
-    
-    _convert_main()
-
-if __name__ == "__main__":
-    main()