ping-dataexport 0.1.0__py3-none-any.whl

ping_dataexport/writer/base.py

@@ -0,0 +1,37 @@
+"""RowWriter interface — one implementation per output format."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, List, Optional, Sequence
+
+from ..db.connector import ColumnInfo
+
+
+class RowWriter(ABC):
+    """open() -> write_row()* -> close(). abort() removes the partial file on failure."""
+
+    def __init__(self) -> None:
+        self._path: Optional[Path] = None
+
+    @abstractmethod
+    def open(self, path: Path, columns: List[ColumnInfo]) -> None:
+        """Create/truncate the target file for the given result columns."""
+
+    @abstractmethod
+    def write_row(self, row: Sequence[Any], row_id: int) -> None:
+        """Write one row. row_id is the engine's running counter (audit column)."""
+
+    @abstractmethod
+    def close(self) -> None:
+        """Flush and close; must be idempotent."""
+
+    def abort(self) -> None:
+        """Close and delete the partial output of a failed partition."""
+        try:
+            self.close()
+        finally:
+            if self._path is not None and self._path.exists():
+                self._path.unlink()
+                self._path = None

ping_dataexport/writer/delimited.py

@@ -0,0 +1,108 @@
+"""DelimitedRowWriter — CSV/TXT with gzip, strip rules, audit columns.
+
+Hand-rolled (NOT the csv module) so the quoting rules are exact and ours:
+- csv: minimal quoting — a field is quoted only when it contains the separator,
+  the quote char, or a newline; embedded quotes are doubled.
+- txt: tab separator forced, quoting disabled.
+- NULL -> empty string; UTF-8; '\n' line endings; create/truncate.
+- Value rendering preserves type fidelity: Decimal via str() (no float round-trip),
+  dates yyyy-MM-dd, timestamps yyyy-MM-dd HH:mm:ss[.ffffff], True/False booleans.
+- Audit columns (RowID / DataDate) are appended unconditionally.
+"""
+
+from __future__ import annotations
+
+import datetime
+import decimal
+import gzip
+import io
+from pathlib import Path
+from typing import Any, Callable, List, Optional, Sequence
+
+from ..db.connector import STRING, ColumnInfo
+from ..export.options import ExportOptions
+from ..export.util import build_text_cleaner
+from .base import RowWriter
+
+
+def render_value(value: Any, clean_text: Callable[[str], str]) -> str:
+    """One DB value -> its delimited-text form. Strip rules apply to strings only."""
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return clean_text(value)
+    if isinstance(value, bool):
+        return "True" if value else "False"
+    if isinstance(value, decimal.Decimal):
+        return str(value)
+    if isinstance(value, datetime.datetime):
+        base = value.strftime("%Y-%m-%d %H:%M:%S")
+        return f"{base}.{value.microsecond:06d}" if value.microsecond else base
+    if isinstance(value, datetime.date):
+        return value.isoformat()
+    if isinstance(value, datetime.time):
+        return value.isoformat()
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        return bytes(value).hex()
+    return str(value)
+
+
+class DelimitedRowWriter(RowWriter):
+    def __init__(self, options: ExportOptions, data_date: str):
+        super().__init__()
+        if options.format == "txt":
+            self.sep = "\t"
+            self.quote: Optional[str] = None  # txt: no quoting
+        else:
+            self.sep = options.sep
+            self.quote = options.quote
+        self.gzip = options.gzip
+        self.row_id_column = options.row_id_column
+        self.data_date_column = options.data_date_column
+        self.data_date = data_date
+        self._clean = build_text_cleaner(options.stripnewline, options.stripmetachar)
+        self._fh: Optional[io.TextIOBase] = None
+        self._is_string_col: List[bool] = []
+
+    def open(self, path: Path, columns: List[ColumnInfo]) -> None:
+        self._path = path
+        if self.gzip:
+            self._fh = gzip.open(path, "wt", encoding="utf-8", newline="\n")
+        else:
+            self._fh = open(path, "w", encoding="utf-8", newline="\n")
+        self._is_string_col = [c.kind == STRING for c in columns]
+        header = [c.name for c in columns]
+        if self.row_id_column:
+            header.append(self.row_id_column)
+        if self.data_date_column:
+            header.append(self.data_date_column)
+        self._fh.write(self.sep.join(self._quote_field(h) for h in header) + "\n")
+
+    def write_row(self, row: Sequence[Any], row_id: int) -> None:
+        fields = [
+            self._quote_field(render_value(v, self._clean if is_str else _identity))
+            for v, is_str in zip(row, self._is_string_col)
+        ]
+        if self.row_id_column:
+            fields.append(str(row_id))
+        if self.data_date_column:
+            fields.append(self._quote_field(self.data_date))
+        self._fh.write(self.sep.join(fields) + "\n")
+
+    def close(self) -> None:
+        if self._fh is not None:
+            try:
+                self._fh.close()
+            finally:
+                self._fh = None
+
+    def _quote_field(self, text: str) -> str:
+        if self.quote is None:
+            return text
+        if self.quote in text or self.sep in text or "\n" in text or "\r" in text:
+            return self.quote + text.replace(self.quote, self.quote * 2) + self.quote
+        return text
+
+
+def _identity(value: str) -> str:
+    return value

ping_dataexport/writer/parquet.py

@@ -0,0 +1,84 @@
+"""ParquetRowWriter — PyArrow ParquetWriter lifecycle; ArrowRowMapper does the mapping.
+
+- Snappy compression; --gzip is rejected at validation (parquet compresses itself).
+- Rows buffer per-column and flush as a typed RecordBatch every fetch_size rows,
+  so memory stays bounded regardless of result size.
+- Audit columns: RowID -> int64, DataDate -> string; a clash with a query column
+  name (case-insensitive) skips that audit column.
+- Split mode reuses the same schema across part files; row_id keeps counting
+  (the engine owns the counter).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, List, Optional, Sequence
+
+from ..db.connector import BIGINT, STRING, ColumnInfo
+from ..export.options import ExportOptions
+from ..export.util import build_text_cleaner
+from .arrow_map import ArrowRowMapper
+from .base import RowWriter
+
+
+class ParquetRowWriter(RowWriter):
+    def __init__(self, options: ExportOptions, data_date: str):
+        super().__init__()
+        self.fetch_size = options.fetch_size
+        self.data_date = data_date
+        self._clean = build_text_cleaner(options.stripnewline, options.stripmetachar)
+        self._row_id_column = options.row_id_column
+        self._data_date_column = options.data_date_column
+        self._writer: Optional[Any] = None
+        self._mapper: Optional[ArrowRowMapper] = None
+        self._buffers: List[List[Any]] = []
+        self._buffered = 0
+        self._audit_row_id = False
+        self._audit_data_date = False
+
+    def open(self, path: Path, columns: List[ColumnInfo]) -> None:
+        import pyarrow.parquet as pq
+
+        self._path = path
+        query_names = {c.name.lower() for c in columns}
+        effective = list(columns)
+        self._audit_row_id = bool(
+            self._row_id_column and self._row_id_column.lower() not in query_names
+        )
+        if self._audit_row_id:
+            effective.append(ColumnInfo(self._row_id_column, BIGINT))
+        self._audit_data_date = bool(
+            self._data_date_column and self._data_date_column.lower() not in query_names
+        )
+        if self._audit_data_date:
+            effective.append(ColumnInfo(self._data_date_column, STRING))
+
+        self._mapper = ArrowRowMapper(effective, self._clean)
+        self._writer = pq.ParquetWriter(str(path), self._mapper.schema, compression="snappy")
+        self._buffers = self._mapper.new_buffers()
+        self._buffered = 0
+
+    def write_row(self, row: Sequence[Any], row_id: int) -> None:
+        values = list(row)
+        if self._audit_row_id:
+            values.append(row_id)
+        if self._audit_data_date:
+            values.append(self.data_date)
+        self._mapper.append_row(self._buffers, values)
+        self._buffered += 1
+        if self._buffered >= self.fetch_size:
+            self._flush()
+
+    def close(self) -> None:
+        if self._writer is not None:
+            try:
+                if self._buffered:
+                    self._flush()
+            finally:
+                self._writer.close()
+                self._writer = None
+
+    def _flush(self) -> None:
+        self._writer.write_batch(self._mapper.to_record_batch(self._buffers))
+        self._buffers = self._mapper.new_buffers()
+        self._buffered = 0

ping_dataexport-0.1.0.dist-info/METADATA

@@ -0,0 +1,228 @@
+Metadata-Version: 2.4
+Name: ping-dataexport
+Version: 0.1.0
+Summary: Universal database data export to CSV/TXT/Parquet — streaming, typed, no pandas
+Author: vorapol
+License-Expression: Apache-2.0
+Keywords: export,database,csv,parquet,etl,data-platform,airflow
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Database
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: pyarrow>=18
+Provides-Extra: odbc
+Requires-Dist: pyodbc>=5; extra == "odbc"
+Provides-Extra: oracle
+Requires-Dist: oracledb>=2; extra == "oracle"
+Provides-Extra: drivers
+Requires-Dist: pyodbc>=5; extra == "drivers"
+Requires-Dist: oracledb>=2; extra == "drivers"
+Provides-Extra: dev
+Requires-Dist: pytest>=8; extra == "dev"
+Dynamic: license-file
+
+# ping-dataexport
+
+Universal database data export: connect to a database, run a SQL query, stream
+the result to **CSV / TXT / Parquet**. Built for data-platform work — exporting
+legacy databases into files a data lake or warehouse can ingest.
+
+- **Type fidelity first** — rows stream cursor → writer with native driver
+  types. `DECIMAL(p,s)` lands in Parquet as `decimal128(p,s)`, ints with NULLs
+  stay ints, dates stay dates. No pandas, ever.
+- **Databases** — SQLite (stdlib, zero setup), SQL Server (MS ODBC / FreeTDS),
+  Oracle (native thin mode or ODBC), any ODBC source.
+- **Two front doors** — a CLI and a library API (`ping_dataexport.api`) built
+  for Airflow: exceptions instead of exit codes, results as objects.
+
+## Install
+
+```bash
+pip install ping-dataexport             # base (pyarrow) — sqlite works out of the box
+pip install ping-dataexport[odbc]       # + pyodbc     (SQL Server, FreeTDS, ODBC)
+pip install ping-dataexport[oracle]     # + oracledb   (Oracle native)
+pip install ping-dataexport[drivers]    # + both
+```
+
+## 1. Create a config.ini
+
+One `[Main]` section for defaults, one section per database source:
+
+```ini
+[Main]
+ExportFolder=./output        ; relative output/log paths resolve under this
+FetchSize=10000
+ForceMakeDir=Yes
+RowIDColumn=row_id           ; optional audit column names
+DataDateColumn=data_date
+
+[SQLITE01]
+Type=sqlite
+Database=./demo.sqlite
+
+[DB01]
+Type=mssql-odbc
+Host=host
+Port=1433
+Database=DB
+Username=user
+Password=secret
+
+[ORA01]
+Type=oracle
+Host=orahost
+Port=1521
+Service=ORCLPDB
+Username=scott
+Password=tiger
+```
+
+`Type` values: `sqlite`, `mssql-odbc`, `freetds`, `oracle`, `oracle-odbc`,
+`odbc`. ODBC sources also accept a raw `ConnectionString=` instead of
+Host/Port/Database. A full sample ships as `config.sample.ini` in the source
+distribution.
+
+Instead of `Password=` you can use `PasswordBase64=` with the base64-encoded
+password (obfuscation, not encryption). If both are present, `Password` wins.
+
+```ini
+PasswordBase64=c2VjcmV0    ; base64("secret")
+```
+
+## 2. Run an export (CLI)
+
+`ping-dataexport` and `python -m ping_dataexport` are equivalent. The config
+file defaults to `./config.ini` (`-cf` to point elsewhere).
+
+```bash
+# preview to the console (no -o/-od): prints the first rows
+ping-dataexport -s SQLITE01 -q "SELECT * FROM sales"
+
+# single file — format inferred from the extension (.csv/.txt/.parquet)
+ping-dataexport -s SQLITE01 -q "SELECT * FROM sales" -o sales.parquet
+ping-dataexport -s SQLITE01 -q "SELECT * FROM sales" -o sales.csv -gz   # gzip
+
+# validate + preview only, write nothing
+ping-dataexport -s SQLITE01 -q "SELECT * FROM sales" -o sales.csv -t
+```
+
+Common options:
+
+| Option | Meaning |
+|--------|---------|
+| `-s` / `-q` | source section name / SQL query |
+| `-o` | single output file (format from extension, or `-f`) |
+| `-od` / `-bn` | output directory + base filename (required for `-m` modes) |
+| `-f` | `csv`, `txt`, `parquet` |
+| `-gz` | gzip the output (CSV/TXT only) |
+| `-sp` / `-quo` | field separator / quote character |
+| `-stnl space\|blank`, `-stmc` | strip newlines / metacharacters inside values |
+| `-fs` | fetch size (rows per batch) |
+| `-rm Yes` | clear the output directory before exporting |
+| `-cf` / `-lf` | config file path / log file path |
+| `-t` | test mode: validate + preview, no files |
+
+## 3. Export modes (partitioned outputs)
+
+Modes split one query into multiple files. All need `-od` + `-bn`; partition
+values are always bound SQL parameters.
+
+```bash
+# one file per distinct value of a column
+ping-dataexport -s DB01 -q "SELECT * FROM sales" -m groupby -col region -od by_region -bn sales
+
+# one file per day in a date range
+ping-dataexport -s DB01 -q "SELECT * FROM sales" -m date -col txn_date \
+    -df 2026-01-01 -dt 2026-01-31 -od daily -bn sales
+
+# N days back from an anchor date / N months of month-files
+ping-dataexport -s DB01 -q "..." -m relativedate -col txn_date -df 2026-01-10 -dr 7 -od rel -bn r
+ping-dataexport -s DB01 -q "..." -m monthbydate  -col txn_date -df 2026-03-15 -mr 3 -od monthly -bn m
+
+# rotate a new file every N rows
+ping-dataexport -s DB01 -q "SELECT * FROM sales ORDER BY id" -m splitrow -row 100000 -od split -bn part
+
+# incremental: exports only rows newer than the last run (state file in -od)
+ping-dataexport -s DB01 -q "SELECT * FROM sales" -m watermark -col txn_date -od incr -bn sales
+```
+
+## 4. Job files (job.ini)
+
+Put recurring exports in a job file — keys map 1:1 to the CLI options:
+
+```ini
+[DailySales]
+Source=DB01
+Query=SELECT id, txn_date, amount FROM sales
+    WHERE region = 'TH'
+Mode=date
+Column=txn_date
+DateFrom=2026-01-01
+DateTo=2026-01-31
+OutputDir=out/daily_sales
+BaseName=sales
+Format=csv
+
+[FullDump]
+Source=DB01
+Query=SELECT * FROM sales
+Output=out/full_dump.parquet
+```
+
+```bash
+ping-dataexport -j DailySales                 # one section (job.ini by default, -jf elsewhere)
+ping-dataexport -j DailySales -od other_dir   # CLI options override the section
+ping-dataexport -pl 4                         # ALL sections in parallel, 4 workers
+```
+
+A full sample ships as `job.sample.ini` in the source distribution.
+
+## 5. Use as a library (Python / Airflow)
+
+```python
+from ping_dataexport.api import run_export, run_job, run_jobfile
+
+# ad-hoc — any CLI option works as a keyword argument
+result = run_export("DB01", "SELECT * FROM sales",
+                    configfile="config.ini", output="sales.parquet")
+print(result.status, result.rows, result.files)
+
+# one job.ini section / every section in parallel
+run_job("DailySales", jobfile="job.ini", configfile="config.ini")
+results = run_jobfile(jobfile="job.ini", configfile="config.ini", workers=4)
+```
+
+Failures raise exceptions (`ConfigError`, `JobValidationError`, `ExportError`)
+— never `sys.exit` — so an Airflow task fails cleanly. `run_jobfile` is the
+exception: one job's failure never stops the others; check each
+`ExportResult.status`.
+
+The package also works with **no pip install at all**: copy the
+`ping_dataexport/` folder next to your code and import it — relative imports
+only, no metadata lookups.
+
+## Output and logs
+
+- Relative output paths resolve under `ExportFolder`; `ForceMakeDir=Yes`
+  creates missing directories.
+- Every run logs structured lines to stdout and to a log file:
+  `-lf path` > `LogFile=` in config.ini > default
+  `{ExportFolder}/logs/log_{jobname}.txt`.
+- CLI exit codes: `0` success, `1` failure (in `-pl` parallel mode: `1` if any
+  job failed).
+
+## ODBC prerequisites (ODBC sources only)
+
+The Python drivers install via pip, but ODBC drivers are OS-level installs:
+Microsoft ODBC Driver 17/18 for SQL Server, FreeTDS (`apt install tdsodbc`), or
+your Oracle ODBC driver. `Type=sqlite` and `Type=oracle` (thin mode) need none.
+
+## License
+
+Apache-2.0.

ping_dataexport-0.1.0.dist-info/RECORD

@@ -0,0 +1,42 @@
+ping_dataexport/__init__.py,sha256=X7yvlh0O2PpianqHs6AoWVs-dqk96uXkAsakLj6oWBQ,452
+ping_dataexport/__main__.py,sha256=yV5QogtDfzTy4gfX6eDyypddJqOpuBhVHSBm6w1CQ10,1530
+ping_dataexport/api.py,sha256=0VAtlTJ1ht3cNSHvGkfoDlVjiEgTEJo1Vk4bEU0ktEA,5914
+ping_dataexport/applogger.py,sha256=ELa-TkDJWPlQu21IvpvMz1PwWxqlKiK7oCpqyiVyRKA,3750
+ping_dataexport/cli.py,sha256=Rg-e39zdUeyqRzZt2KQEAokFpMg9IHdQmd0ZGfXe4Lg,4468
+ping_dataexport/config.py,sha256=Ui6OgVnanV7IsY-uBZWOY7oiM-5BkU6hx6s4iJHaeK8,9180
+ping_dataexport/errors.py,sha256=gX-UNqcq374Znq9K-8LRmBvUtCGxhz1xo2tspMGImXQ,583
+ping_dataexport/paths.py,sha256=G6whXV4RnRCnIEypRwpvSBnbnsNVkDU7H-Kn_I7L1iI,2684
+ping_dataexport/db/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ping_dataexport/db/connector.py,sha256=wHHryAk-k4QtUjrYUAyXnb9AoC6VgWE6o1OY2cOcwBw,3239
+ping_dataexport/db/oracle_connector.py,sha256=RAAr-ABjXnDLzXikyTVjwlvWEGPjFtMAK1DvpMPAczU,3079
+ping_dataexport/db/pyodbc_connector.py,sha256=lXdP8TareRIyWKzMLnY1kmSfMDuQrlUV_-2UbhmYgCw,3355
+ping_dataexport/db/sqlite_connector.py,sha256=CVmnbWMD0epcVnQHagwN8Z1_MwYHRF8L5fIAcyesLnA,2354
+ping_dataexport/db/type_map.py,sha256=nJ7xmyYhQG9nRbtp7jTwXc69tAFcLTB9eQfWsZlPutY,4286
+ping_dataexport/export/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ping_dataexport/export/engine.py,sha256=YFIH6QJDpzuHhPoKxmcMOzRQwF2SP61RTGjU7KsqZnQ,6144
+ping_dataexport/export/options.py,sha256=qyMAt8Chk1z3DZRmbisNFLUqi1y1SEOFIARb2RNoRCg,1500
+ping_dataexport/export/runner.py,sha256=ruwJQy0SOX_x0C2loI6ZwqTHaniay1io9MnJoN_qQUM,1750
+ping_dataexport/export/util.py,sha256=US5Xx3xkQsUjDH4tpZtYFq7X0Ofau0sHsRdpPU02Ncw,2461
+ping_dataexport/job/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ping_dataexport/job/executor.py,sha256=7AWNk8thyeCtch9Uiny0vDTTs-6qKt_Ut8zBt6bvZ80,4161
+ping_dataexport/job/shared_context.py,sha256=q3oBVLj0XmFbd-uQNsCWQLjDTy1VLxSOewUK6jQkR3A,719
+ping_dataexport/job/spec.py,sha256=Zrnw0YOVqv9XNuucCFaxzF47cTMQBVKAFd-dHa8R-q8,8995
+ping_dataexport/job/validator.py,sha256=bPLwzzD0SkNygW6rdriitQbranahaeSi2FcASjYA_ZA,3669
+ping_dataexport/planner/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ping_dataexport/planner/base.py,sha256=qM9bPPms-E_I-vtjr15azbtvEJDimrv-HR9AekOxQNc,2334
+ping_dataexport/planner/datelist.py,sha256=djYxFTiRtIcWDt90gBQYydiycSKUvHOd-P0SFLO_Tvw,1271
+ping_dataexport/planner/groupby.py,sha256=Opo9r_Rh9xx8d_OP0J10cco_40qq7TVArojkRmpvw9U,2569
+ping_dataexport/planner/monthbydate.py,sha256=M6GkWZc0bpMeVs107pBDmv6uyNMOQDx_-Li539D78Wc,4331
+ping_dataexport/planner/single.py,sha256=kVb755hnjHHld16O7tUSkduzrubf9w2Quaf9nS5aHm8,622
+ping_dataexport/planner/watermark.py,sha256=Rc1Z_aQJBcYeD57Mq-3Li27by_Nq1mlqTZCS0gX9NeU,7764
+ping_dataexport/writer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+ping_dataexport/writer/arrow_map.py,sha256=KYeABbs_Dm34qT19CqNmAWTzwrjWQ8e8VZMfPTyhvjw,6148
+ping_dataexport/writer/base.py,sha256=Pc16NVbXf9nnpHax52jim8wcWMRAZnLXvYgNG8Glf7M,1170
+ping_dataexport/writer/delimited.py,sha256=zr4sX_4sDakLuaUZ-I43VnmhrAoZEzrZSqOGjpugSAg,4123
+ping_dataexport/writer/parquet.py,sha256=Neu2OW2XXYFcCUEMVgnIUIlANkp4rSzcqfwr_biGuDM,3274
+ping_dataexport-0.1.0.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+ping_dataexport-0.1.0.dist-info/METADATA,sha256=0CkxcMRl-ND5V299OFsgtimnvz-woMQsYW8xwkBUSWA,8016
+ping_dataexport-0.1.0.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+ping_dataexport-0.1.0.dist-info/entry_points.txt,sha256=GLTn1c1dKXLLib0N7MOzLRnRTgvwZOwigRqxCPphenw,66
+ping_dataexport-0.1.0.dist-info/top_level.txt,sha256=_-lMZI38na42QmuQQF-WscRvuXvYyD6QzpXR8BClBJg,16
+ping_dataexport-0.1.0.dist-info/RECORD,,

ping_dataexport-0.1.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (84.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ping_dataexport-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+ping-dataexport = ping_dataexport.__main__:main