fixedwidthparser 2.0.0rc1__py3-none-manylinux_2_34_x86_64.whl

fixedwidth/__init__.py

@@ -0,0 +1,69 @@
+"""Read and write fixed-width files through the FixedWidthParser native library.
+
+    import fixedwidth as fw
+
+    layout = fw.Layout([
+        fw.Column("codneg", 12, 12, fw.String),
+        fw.Column("preabe", 56, 13, fw.Float64, scale=2),
+    ])
+    df = fw.read_pandas("COTAHIST_A2023.TXT", layout, fw.ReadOptions(header_lines=1, skip_invalid=True))
+"""
+
+from fixedwidth.arrow_stream import ArrowStream
+from fixedwidth.layout import Layout
+from fixedwidth.reader import read, read_arrow, read_arrow_stream, read_batches, read_pandas, read_polars
+from fixedwidth.table import Table
+from fixedwidth.types import (
+    Alignment,
+    Column,
+    ColumnType,
+    FixedWidthError,
+    LayoutOptions,
+    Overflow,
+    ParseError,
+    ReaderOptions,
+    ReadOptions,
+    TrimMode,
+    WriteOptions,
+)
+from fixedwidth.writer import write, write_bytes
+
+String = ColumnType.STRING
+Int64 = ColumnType.INT64
+Float64 = ColumnType.FLOAT64
+Decimal = ColumnType.DECIMAL
+Date = ColumnType.DATE
+Time = ColumnType.TIME
+Timestamp = ColumnType.TIMESTAMP
+
+__all__ = [
+    "Alignment",
+    "ArrowStream",
+    "Column",
+    "ColumnType",
+    "Date",
+    "Decimal",
+    "FixedWidthError",
+    "Float64",
+    "Int64",
+    "Layout",
+    "LayoutOptions",
+    "Overflow",
+    "ParseError",
+    "ReadOptions",
+    "ReaderOptions",
+    "String",
+    "Table",
+    "Time",
+    "Timestamp",
+    "TrimMode",
+    "WriteOptions",
+    "read",
+    "read_arrow",
+    "read_arrow_stream",
+    "read_batches",
+    "read_pandas",
+    "read_polars",
+    "write",
+    "write_bytes",
+]

fixedwidth/_native.py

@@ -0,0 +1,248 @@
+"""ctypes binding to the FixedWidthParser NativeAOT shared library.
+
+Everything here mirrors src/FixedWidthParser.Native/include/fixedwidth.h. If you change one, change both.
+"""
+
+from __future__ import annotations
+
+import ctypes
+import os
+import platform
+from functools import lru_cache
+from pathlib import Path
+
+FW_OK = 0
+FW_EOF = -1
+FW_BUFFER_TOO_SMALL = -2
+FW_INVALID_HANDLE = -3
+FW_INVALID_ARGUMENT = -4
+FW_ERROR = -5
+FW_PARSE_ERROR = -6
+
+FW_ABI_VERSION = 1
+
+FW_OPT_DEFAULT = 0
+FW_OPT_FALSE = 1
+FW_OPT_TRUE = 2
+
+_u8p = ctypes.POINTER(ctypes.c_uint8)
+
+
+class ColumnSpec(ctypes.Structure):
+    _fields_ = [
+        ("struct_size", ctypes.c_int32),
+        ("name", _u8p),
+        ("name_len", ctypes.c_int32),
+        ("start", ctypes.c_int32),
+        ("length", ctypes.c_int32),
+        ("type", ctypes.c_int32),
+        ("nullable", ctypes.c_int32),
+        ("scale", ctypes.c_int32),
+        ("format", _u8p),
+        ("format_len", ctypes.c_int32),
+        ("trim_char", ctypes.c_int32),
+        ("trim_mode", ctypes.c_int32),
+        ("alignment", ctypes.c_int32),
+        ("padding", ctypes.c_int32),
+        ("overflow", ctypes.c_int32),
+    ]
+
+
+class LayoutOptions(ctypes.Structure):
+    _fields_ = [("struct_size", ctypes.c_int32), ("line_length", ctypes.c_int32), ("filler", ctypes.c_int32)]
+
+
+class ReadOptions(ctypes.Structure):
+    _fields_ = [
+        ("struct_size", ctypes.c_int32),
+        ("header_lines", ctypes.c_int64),
+        ("skip_invalid", ctypes.c_int32),
+        ("degree_of_parallelism", ctypes.c_int32),
+        ("encoding", ctypes.c_int32),
+        ("max_line_length", ctypes.c_int32),
+        ("skip_byte_order_mark", ctypes.c_int32),
+    ]
+
+
+class ReaderOptions(ctypes.Structure):
+    _fields_ = [("struct_size", ctypes.c_int32), ("read", ReadOptions), ("batch_rows", ctypes.c_int64)]
+
+
+class Column(ctypes.Structure):
+    _fields_ = [
+        ("type", ctypes.c_int32),
+        ("scale", ctypes.c_int32),
+        ("length", ctypes.c_int64),
+        ("values", ctypes.c_void_p),
+        ("validity", ctypes.c_void_p),
+        ("data", ctypes.c_void_p),
+        ("data_len", ctypes.c_int64),
+    ]
+
+
+class Table(ctypes.Structure):
+    _fields_ = [("column_count", ctypes.c_int32), ("row_count", ctypes.c_int64), ("columns", ctypes.POINTER(Column))]
+
+
+class ReadResult(ctypes.Structure):
+    _fields_ = [("struct_size", ctypes.c_int32), ("lines_read", ctypes.c_int64), ("lines_skipped", ctypes.c_int64)]
+
+
+class ParseErrorInfo(ctypes.Structure):
+    _fields_ = [("struct_size", ctypes.c_int32), ("line_number", ctypes.c_int64), ("column", ctypes.c_int32)]
+
+
+class WriteOptions(ctypes.Structure):
+    _fields_ = [
+        ("struct_size", ctypes.c_int32),
+        ("encoding", ctypes.c_int32),
+        ("newline", _u8p),
+        ("newline_len", ctypes.c_int32),
+        ("degree_of_parallelism", ctypes.c_int32),
+        ("batch_rows", ctypes.c_int64),
+    ]
+
+
+class Buffer(ctypes.Structure):
+    _fields_ = [("data", ctypes.c_void_p), ("length", ctypes.c_int64)]
+
+
+class ArrowSchema(ctypes.Structure):
+    pass
+
+
+class ArrowArray(ctypes.Structure):
+    pass
+
+
+_ArrowSchemaRelease = ctypes.CFUNCTYPE(None, ctypes.POINTER(ArrowSchema))
+_ArrowArrayRelease = ctypes.CFUNCTYPE(None, ctypes.POINTER(ArrowArray))
+
+ArrowSchema._fields_ = [
+    ("format", ctypes.c_char_p),
+    ("name", ctypes.c_char_p),
+    ("metadata", ctypes.c_char_p),
+    ("flags", ctypes.c_int64),
+    ("n_children", ctypes.c_int64),
+    ("children", ctypes.POINTER(ctypes.POINTER(ArrowSchema))),
+    ("dictionary", ctypes.POINTER(ArrowSchema)),
+    ("release", _ArrowSchemaRelease),
+    ("private_data", ctypes.c_void_p),
+]
+ArrowArray._fields_ = [
+    ("length", ctypes.c_int64),
+    ("null_count", ctypes.c_int64),
+    ("offset", ctypes.c_int64),
+    ("n_buffers", ctypes.c_int64),
+    ("n_children", ctypes.c_int64),
+    ("buffers", ctypes.POINTER(ctypes.c_void_p)),
+    ("children", ctypes.POINTER(ctypes.POINTER(ArrowArray))),
+    ("dictionary", ctypes.POINTER(ArrowArray)),
+    ("release", _ArrowArrayRelease),
+    ("private_data", ctypes.c_void_p),
+]
+
+
+class ArrowArrayStream(ctypes.Structure):
+    pass
+
+
+_ArrowStreamGetSchema = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(ArrowArrayStream), ctypes.POINTER(ArrowSchema))
+_ArrowStreamGetNext = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(ArrowArrayStream), ctypes.POINTER(ArrowArray))
+_ArrowStreamGetLastError = ctypes.CFUNCTYPE(ctypes.c_char_p, ctypes.POINTER(ArrowArrayStream))
+_ArrowStreamRelease = ctypes.CFUNCTYPE(None, ctypes.POINTER(ArrowArrayStream))
+
+ArrowArrayStream._fields_ = [
+    ("get_schema", _ArrowStreamGetSchema),
+    ("get_next", _ArrowStreamGetNext),
+    ("get_last_error", _ArrowStreamGetLastError),
+    ("release", _ArrowStreamRelease),
+    ("private_data", ctypes.c_void_p),
+]
+
+_P = ctypes.POINTER
+_SIGNATURES: dict[str, tuple[object, list[object]]] = {
+    "fw_abi_version": (ctypes.c_int32, []),
+    "fw_last_error": (ctypes.c_int32, [_u8p, ctypes.c_int32, _P(ctypes.c_int32)]),
+    "fw_last_parse_error": (ctypes.c_int32, [_P(ParseErrorInfo)]),
+    "fw_layout_create": (ctypes.c_int32, [_P(ColumnSpec), ctypes.c_int32, _P(LayoutOptions), _P(ctypes.c_void_p)]),
+    "fw_layout_free": (None, [ctypes.c_void_p]),
+    "fw_read_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReadOptions), _P(Table), _P(ReadResult)]),
+    "fw_read_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReadOptions), _P(Table), _P(ReadResult)]),
+    "fw_free_table": (None, [_P(Table)]),
+    "fw_reader_open_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]),
+    "fw_reader_open_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]),
+    "fw_reader_next": (ctypes.c_int32, [ctypes.c_void_p, _P(Table)]),
+    "fw_reader_close": (None, [ctypes.c_void_p]),
+    "fw_read_arrow_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReadOptions), _P(ArrowArray), _P(ArrowSchema), _P(ReadResult)]),
+    "fw_read_arrow_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReadOptions), _P(ArrowArray), _P(ArrowSchema), _P(ReadResult)]),
+    "fw_read_arrow_stream_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]),
+    "fw_read_arrow_stream_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]),
+    "fw_write_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, _P(Table), ctypes.c_void_p, _P(WriteOptions)]),
+    "fw_write_memory": (ctypes.c_int32, [_P(Table), ctypes.c_void_p, _P(WriteOptions), _P(Buffer)]),
+    "fw_free_buffer": (None, [_P(Buffer)]),
+}
+
+
+def library_filename() -> str:
+    system = platform.system()
+    if system == "Windows":
+        return "FixedWidthParser.Native.dll"
+    if system == "Darwin":
+        return "FixedWidthParser.Native.dylib"
+    return "FixedWidthParser.Native.so"
+
+
+def _candidate_paths() -> list[Path]:
+    paths = []
+    override = os.environ.get("FIXEDWIDTH_NATIVE_LIB")
+    if override:
+        paths.append(Path(override))
+    paths.append(Path(__file__).parent / "_lib" / library_filename())
+    return paths
+
+
+@lru_cache(maxsize=1)
+def load_library() -> ctypes.CDLL:
+    """Loads the bundled library (or FIXEDWIDTH_NATIVE_LIB) once and checks its ABI version."""
+    tried = []
+    for path in _candidate_paths():
+        if not path.exists():
+            tried.append(str(path))
+            continue
+        lib = ctypes.CDLL(str(path))
+        for name, (restype, argtypes) in _SIGNATURES.items():
+            function = getattr(lib, name)
+            function.restype = restype
+            function.argtypes = argtypes
+        version = lib.fw_abi_version()
+        if version != FW_ABI_VERSION:
+            raise OSError(f"{path} has ABI version {version}; this package needs {FW_ABI_VERSION}")
+        return lib
+    raise OSError(
+        "FixedWidthParser.Native was not found (looked in " + ", ".join(tried) + "). "
+        "Build it with `python python/scripts/build_native.py` or set FIXEDWIDTH_NATIVE_LIB."
+    )
+
+
+def last_error() -> str:
+    lib = load_library()
+    buffer = (ctypes.c_uint8 * 4096)()
+    length = ctypes.c_int32()
+    status = lib.fw_last_error(buffer, len(buffer), ctypes.byref(length))
+    if status == FW_BUFFER_TOO_SMALL:
+        buffer = (ctypes.c_uint8 * length.value)()
+        lib.fw_last_error(buffer, len(buffer), ctypes.byref(length))
+    return bytes(buffer[: length.value]).decode("utf-8", "replace")
+
+
+def last_parse_error() -> tuple[int, int]:
+    info = ParseErrorInfo(struct_size=ctypes.sizeof(ParseErrorInfo))
+    load_library().fw_last_parse_error(ctypes.byref(info))
+    return int(info.line_number), int(info.column)
+
+
+def utf8(text: str) -> tuple[ctypes.Array, int]:
+    """A UTF-8 buffer and its length, for a pointer + length argument."""
+    encoded = text.encode("utf-8")
+    return (ctypes.c_uint8 * max(1, len(encoded))).from_buffer_copy(encoded.ljust(1, b"\0")), len(encoded)

fixedwidth/arrow_stream.py

@@ -0,0 +1,127 @@
+"""An Arrow C stream exposed through the Arrow PyCapsule interface, with no pyarrow dependency."""
+
+from __future__ import annotations
+
+import ctypes
+
+from fixedwidth import _native
+from fixedwidth.types import FixedWidthError
+
+_CAPSULE_NAME = b"arrow_array_stream"
+
+_capsule_new = ctypes.pythonapi.PyCapsule_New
+_capsule_new.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p]
+_capsule_new.restype = ctypes.py_object
+
+_capsule_pointer = ctypes.pythonapi.PyCapsule_GetPointer
+_capsule_pointer.argtypes = [ctypes.c_void_p, ctypes.c_char_p]
+_capsule_pointer.restype = ctypes.c_void_p
+
+_ReadFn = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p)
+_ErrorFn = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)
+_ReleaseFn = ctypes.CFUNCTYPE(None, ctypes.c_void_p)
+
+
+class _Target:
+    """The native stream behind a proxy, re-typed so ctypes releases the GIL in its callbacks, and the
+    caller's bytes a memory read reads lazily."""
+
+    def __init__(self, stream: _native.ArrowArrayStream, keep: object) -> None:
+        self.stream = stream
+        self.keep = keep
+        self.address = ctypes.addressof(stream)
+        self.get_schema = _ReadFn(_address_of(stream.get_schema))
+        self.get_next = _ReadFn(_address_of(stream.get_next))
+        self.get_last_error = _ErrorFn(_address_of(stream.get_last_error))
+
+
+def _address_of(function: object) -> int:
+    return ctypes.cast(function, ctypes.c_void_p).value
+
+
+_streams: dict[int, _Target] = {}
+_proxies: dict[int, _native.ArrowArrayStream] = {}
+
+
+def _target(stream: int) -> _Target:
+    return _streams[_native.ArrowArrayStream.from_address(stream).private_data]
+
+
+@_ReadFn
+def _get_schema(stream: int, out: int) -> int:
+    target = _target(stream)
+    return target.get_schema(target.address, out)
+
+
+@_ReadFn
+def _get_next(stream: int, out: int) -> int:
+    target = _target(stream)
+    return target.get_next(target.address, out)
+
+
+@_ErrorFn
+def _get_last_error(stream: int) -> int | None:
+    target = _target(stream)
+    return target.get_last_error(target.address)
+
+
+def _close(proxy: _native.ArrowArrayStream) -> None:
+    _release(_streams.pop(proxy.private_data).stream)
+    proxy.release = _native._ArrowStreamRelease()
+
+
+@_ReleaseFn
+def _release_proxy(stream: int) -> None:
+    _close(_native.ArrowArrayStream.from_address(stream))
+
+
+def _release(stream: _native.ArrowArrayStream) -> None:
+    if stream.release:
+        stream.release(ctypes.byref(stream))
+
+
+def _proxy(stream: _native.ArrowArrayStream, keep: object) -> _native.ArrowArrayStream:
+    target = _Target(stream, keep)
+    _streams[id(target)] = target
+    proxy = _native.ArrowArrayStream()
+    proxy.get_schema = ctypes.cast(_get_schema, _native._ArrowStreamGetSchema)
+    proxy.get_next = ctypes.cast(_get_next, _native._ArrowStreamGetNext)
+    proxy.get_last_error = ctypes.cast(_get_last_error, _native._ArrowStreamGetLastError)
+    proxy.release = ctypes.cast(_release_proxy, _native._ArrowStreamRelease)
+    proxy.private_data = id(target)
+    return proxy
+
+
+def _destroy_capsule(capsule: int) -> None:
+    proxy = _proxies.pop(_capsule_pointer(capsule, _CAPSULE_NAME))
+    if proxy.release:
+        _close(proxy)
+
+
+_destroy_capsule_callback = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(_destroy_capsule)
+
+
+class ArrowStream:
+    """A single-use stream of Arrow record batches implementing `__arrow_c_stream__`.
+
+    Hand it to any consumer of the Arrow PyCapsule interface — `polars.DataFrame(stream)`,
+    `pyarrow.RecordBatchReader.from_stream(stream)`. Once a consumer takes it, the stream is spent;
+    dropping it unconsumed releases the native read.
+    """
+
+    def __init__(self, stream: _native.ArrowArrayStream, keep: object = None) -> None:
+        self._stream: _native.ArrowArrayStream | None = stream
+        self._keep = keep
+
+    def __arrow_c_stream__(self, requested_schema: object = None) -> object:
+        if self._stream is None:
+            raise FixedWidthError("this Arrow stream was already consumed")
+        stream, self._stream = self._stream, None
+        proxy = _proxy(stream, self._keep)
+        address = ctypes.addressof(proxy)
+        _proxies[address] = proxy
+        return _capsule_new(address, _CAPSULE_NAME, _destroy_capsule_callback)
+
+    def __del__(self) -> None:
+        if getattr(self, "_stream", None) is not None:
+            _release(self._stream)

fixedwidth/layout.py

@@ -0,0 +1,94 @@
+"""Layout: the columns of a line, validated once by the native library and shared by every call."""
+
+from __future__ import annotations
+
+import ctypes
+from collections.abc import Sequence
+
+from fixedwidth import _native
+from fixedwidth.types import Column, ColumnType, FixedWidthError, LayoutOptions, ParseError
+
+
+def check(status: int) -> None:
+    """Raises the error a native status reports."""
+    if status == _native.FW_OK:
+        return
+    message = _native.last_error() or f"native status {status}"
+    if status == _native.FW_PARSE_ERROR:
+        line_number, column = _native.last_parse_error()
+        raise ParseError(message, line_number, column)
+    raise FixedWidthError(message)
+
+
+def ascii_option(value: str | None, what: str) -> int:
+    if value is None:
+        return 0
+    if len(value) != 1 or ord(value) > 127:
+        raise ValueError(f"{what} must be one ASCII character, got {value!r}")
+    return ord(value)
+
+
+class Layout:
+    """How the columns of a line map to names and types; pass it to every read and write.
+
+    The native layout is immutable and may be shared across threads.
+    """
+
+    def __init__(self, columns: Sequence[Column], options: LayoutOptions | None = None) -> None:
+        self._lib = _native.load_library()
+        self._handle: int | None = None
+        self._columns = list(columns)
+        specs = (_native.ColumnSpec * max(1, len(self._columns)))()
+        keep = [self._fill(specs[i], column) for i, column in enumerate(self._columns)]
+        raw = None
+        if options is not None:
+            raw = _native.LayoutOptions(
+                struct_size=ctypes.sizeof(_native.LayoutOptions),
+                line_length=options.line_length,
+                filler=ascii_option(options.filler, "filler"),
+            )
+        handle = ctypes.c_void_p()
+        check(self._lib.fw_layout_create(specs, len(self._columns), ctypes.byref(raw) if raw else None, ctypes.byref(handle)))
+        self._handle = handle.value
+        del keep
+
+    @staticmethod
+    def _fill(spec: _native.ColumnSpec, column: Column) -> tuple[object, ...]:
+        name, name_len = _native.utf8(column.name)
+        fmt, fmt_len = _native.utf8(column.format) if column.format is not None else (None, 0)
+        spec.struct_size = ctypes.sizeof(_native.ColumnSpec)
+        spec.name = ctypes.cast(name, ctypes.POINTER(ctypes.c_uint8))
+        spec.name_len = name_len
+        spec.start = column.start
+        spec.length = column.length
+        spec.type = int(ColumnType(column.type))
+        spec.nullable = int(column.nullable)
+        spec.scale = column.scale
+        spec.format = ctypes.cast(fmt, ctypes.POINTER(ctypes.c_uint8)) if fmt is not None else None
+        spec.format_len = fmt_len
+        spec.trim_char = ascii_option(column.trim_char, "trim_char")
+        spec.trim_mode = int(column.trim_mode or 0)
+        spec.alignment = int(column.alignment or 0)
+        spec.padding = ascii_option(column.padding, "padding")
+        spec.overflow = int(column.overflow or 0)
+        return name, fmt
+
+    @property
+    def columns(self) -> list[Column]:
+        return list(self._columns)
+
+    @property
+    def names(self) -> list[str]:
+        return [column.name for column in self._columns]
+
+    @property
+    def handle(self) -> int:
+        return self._handle or 0
+
+    def __del__(self) -> None:
+        if getattr(self, "_handle", None):
+            self._lib.fw_layout_free(self._handle)
+            self._handle = None
+
+    def __repr__(self) -> str:
+        return f"Layout({len(self._columns)} columns: {', '.join(self.names)})"

fixedwidth/reader.py

@@ -0,0 +1,157 @@
+"""Reading: whole sources into a Table, Arrow, pandas or polars, and batched reads."""
+
+from __future__ import annotations
+
+import ctypes
+import os
+from collections.abc import Iterator
+from typing import Any, Union
+
+from fixedwidth import _native
+from fixedwidth.arrow_stream import ArrowStream
+from fixedwidth.layout import Layout, check
+from fixedwidth.table import Table, _Owner
+from fixedwidth.types import ReaderOptions, ReadOptions, code_page
+
+Source = Union[str, "os.PathLike[str]", bytes, bytearray, memoryview]
+
+
+class _Input:
+    """A file path or a buffer, as the pointer + length arguments the native calls take."""
+
+    def __init__(self, source: Source) -> None:
+        self.keep: object = None
+        if isinstance(source, (str, os.PathLike)):
+            self.path, self.path_len = _native.utf8(os.fspath(source))
+            return
+        self.path = None
+        view = memoryview(source)
+        if view.readonly or not view.contiguous:
+            data = bytes(view)
+            self.keep = data
+            self.address = ctypes.cast(ctypes.c_char_p(data), ctypes.c_void_p).value
+        else:
+            buffer = (ctypes.c_char * view.nbytes).from_buffer(view)
+            self.keep = (view, buffer)
+            self.address = ctypes.addressof(buffer)
+        self.length = view.nbytes
+
+    @property
+    def is_file(self) -> bool:
+        return self.path is not None
+
+
+def _read_options(options: ReadOptions | None) -> _native.ReadOptions | None:
+    if options is None:
+        return None
+    bom = options.skip_byte_order_mark
+    return _native.ReadOptions(
+        struct_size=ctypes.sizeof(_native.ReadOptions),
+        header_lines=options.header_lines,
+        skip_invalid=int(options.skip_invalid),
+        degree_of_parallelism=options.degree_of_parallelism,
+        encoding=code_page(options.encoding),
+        max_line_length=options.max_line_length,
+        skip_byte_order_mark=_native.FW_OPT_DEFAULT if bom is None else (_native.FW_OPT_TRUE if bom else _native.FW_OPT_FALSE),
+    )
+
+
+def _reader_options(options: ReaderOptions | None) -> _native.ReaderOptions | None:
+    if options is None:
+        return None
+    raw = _native.ReaderOptions(struct_size=ctypes.sizeof(_native.ReaderOptions), batch_rows=options.batch_rows)
+    read = _read_options(options.read)
+    if read is not None:
+        raw.read = read
+    return raw
+
+
+def _ref(value: object) -> object:
+    return ctypes.byref(value) if value is not None else None
+
+
+def read(source: Source, layout: Layout, options: ReadOptions | None = None) -> Table:
+    """Reads a whole file or buffer into a Table (see `ReadOptions` for header lines, skipping and threads)."""
+    lib = _native.load_library()
+    data = _Input(source)
+    raw = _read_options(options)
+    table = _native.Table()
+    result = _native.ReadResult(struct_size=ctypes.sizeof(_native.ReadResult))
+    if data.is_file:
+        status = lib.fw_read_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(table), ctypes.byref(result))
+    else:
+        status = lib.fw_read_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(table), ctypes.byref(result))
+    check(status)
+    return Table(_Owner(table), layout.names, int(result.lines_read), int(result.lines_skipped))
+
+
+def read_arrow(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any:
+    """Reads a whole file or buffer into a `pyarrow.Table`, handed over without a copy. Requires pyarrow."""
+    import pyarrow
+
+    lib = _native.load_library()
+    data = _Input(source)
+    raw = _read_options(options)
+    array = _native.ArrowArray()
+    schema = _native.ArrowSchema()
+    if data.is_file:
+        status = lib.fw_read_arrow_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(array), ctypes.byref(schema), None)
+    else:
+        status = lib.fw_read_arrow_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(array), ctypes.byref(schema), None)
+    check(status)
+    batch = pyarrow.RecordBatch._import_from_c(ctypes.addressof(array), ctypes.addressof(schema))
+    return pyarrow.Table.from_batches([batch])
+
+
+def read_pandas(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any:
+    """`read_arrow()` as a `pandas.DataFrame`; each Arrow buffer is freed as pandas takes it. Requires pyarrow and pandas."""
+    return read_arrow(source, layout, options).to_pandas(self_destruct=True, split_blocks=True)
+
+
+def read_polars(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any:
+    """`read_arrow()` as a `polars.DataFrame`, zero-copy. Without pyarrow, reads sequentially through an Arrow stream."""
+    import polars
+
+    try:
+        return polars.from_arrow(read_arrow(source, layout, options))
+    except ImportError:
+        return polars.DataFrame(read_arrow_stream(source, layout, ReaderOptions(read=options or ReadOptions())))
+
+
+def read_batches(source: Source, layout: Layout, options: ReaderOptions | None = None) -> Iterator[Table]:
+    """Reads sequentially, yielding a Table of at most `options.batch_rows` rows at a time."""
+    lib = _native.load_library()
+    data = _Input(source)
+    raw = _reader_options(options)
+    reader = ctypes.c_void_p()
+    if data.is_file:
+        status = lib.fw_reader_open_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(reader))
+    else:
+        status = lib.fw_reader_open_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(reader))
+    check(status)
+    try:
+        while True:
+            table = _native.Table()
+            status = lib.fw_reader_next(reader, ctypes.byref(table))
+            if status == _native.FW_EOF:
+                return
+            check(status)
+            yield Table(_Owner(table), layout.names)
+    finally:
+        lib.fw_reader_close(reader)
+        del data
+
+
+def read_arrow_stream(source: Source, layout: Layout, options: ReaderOptions | None = None) -> ArrowStream:
+    """The batches of `read_batches()` as a single-use Arrow PyCapsule stream, for `polars.DataFrame(stream)`
+    or `pyarrow.RecordBatchReader.from_stream(stream)`. Needs no third-party package to create."""
+    lib = _native.load_library()
+    data = _Input(source)
+    raw = _reader_options(options)
+    stream = _native.ArrowArrayStream()
+    if data.is_file:
+        status = lib.fw_read_arrow_stream_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(stream))
+    else:
+        status = lib.fw_read_arrow_stream_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(stream))
+    check(status)
+    return ArrowStream(stream, keep=data)

fixedwidth/table.py

@@ -0,0 +1,135 @@
+"""Table: a columnar read whose numeric columns are NumPy views over the native buffers."""
+
+from __future__ import annotations
+
+import ctypes
+from collections.abc import Callable
+from decimal import Decimal
+from typing import Any
+
+from fixedwidth import _native
+from fixedwidth.types import ColumnType
+
+_ELEMENTS = {
+    ColumnType.INT64: (ctypes.c_int64, "int64"),
+    ColumnType.FLOAT64: (ctypes.c_double, "float64"),
+    ColumnType.DECIMAL: (ctypes.c_int64, "int64"),
+    ColumnType.DATE: (ctypes.c_int32, "int32"),
+    ColumnType.TIME: (ctypes.c_int64, "int64"),
+    ColumnType.TIMESTAMP: (ctypes.c_int64, "int64"),
+}
+
+
+class _Owner:
+    """Owns one native fw_table; frees it once nothing (the Table or any view) references it."""
+
+    def __init__(self, table: _native.Table) -> None:
+        self.table = table
+        self._on_free: Callable[[bool], object] | None = None
+
+    def view(self, address: int, element: type, count: int) -> ctypes.Array:
+        buffer = (element * count).from_address(address)
+        buffer._owner = self  # type: ignore[attr-defined]  # a view keeps the native memory alive
+        return buffer
+
+    def __del__(self) -> None:
+        _native.load_library().fw_free_table(ctypes.byref(self.table))
+        if self._on_free is not None:
+            self._on_free(True)
+
+
+class Table:
+    """One read: `table[name]` is a NumPy view for numeric and date/time columns (as stored: int64,
+    float64, DECIMAL's unscaled int64, DATE's int32 days, TIME/TIMESTAMP's int64 microseconds) and a
+    list of `str` for strings. Null rows are marked by `validity(name)`. Views stay valid after the
+    Table is gone; the native memory is freed when the last one is."""
+
+    def __init__(self, owner: _Owner, names: list[str], lines_read: int = 0, lines_skipped: int = 0) -> None:
+        self._owner = owner
+        self._names = names
+        self.lines_read = lines_read
+        self.lines_skipped = lines_skipped
+
+    @property
+    def num_rows(self) -> int:
+        return int(self._owner.table.row_count)
+
+    @property
+    def column_names(self) -> list[str]:
+        return list(self._names)
+
+    def __len__(self) -> int:
+        return self.num_rows
+
+    def _column(self, name: str) -> _native.Column:
+        try:
+            return self._owner.table.columns[self._names.index(name)]
+        except ValueError:
+            raise KeyError(name) from None
+
+    def __getitem__(self, name: str) -> Any:
+        column = self._column(name)
+        kind = ColumnType(column.type)
+        if kind is ColumnType.STRING:
+            return _strings(column)
+        import numpy
+
+        element, dtype = _ELEMENTS[kind]
+        if column.length == 0:
+            return numpy.empty(0, dtype=dtype)
+        return numpy.frombuffer(self._owner.view(column.values, element, column.length), dtype=dtype)
+
+    def validity(self, name: str) -> Any:
+        """A NumPy bool array, True where the row holds a value, or None when the column has no nulls."""
+        column = self._column(name)
+        if not column.validity:
+            return None
+        import numpy
+
+        bits = numpy.frombuffer(ctypes.string_at(column.validity, (column.length + 7) // 8), dtype=numpy.uint8)
+        return numpy.unpackbits(bits, bitorder="little")[: column.length].astype(bool)
+
+    def to_arrow(self) -> Any:
+        """The same columns as a `pyarrow.Table`, sharing the native buffers. Requires pyarrow."""
+        import pyarrow
+
+        arrays = [self._arrow_column(name, pyarrow) for name in self._names]
+        return pyarrow.Table.from_arrays(arrays, names=self._names)
+
+    def _arrow_column(self, name: str, pa: Any) -> Any:
+        column = self._column(name)
+        kind = ColumnType(column.type)
+        n = int(column.length)
+        validity = pa.foreign_buffer(column.validity, (n + 7) // 8, base=self._owner) if column.validity else None
+        if kind is ColumnType.DECIMAL:
+            # ponytail: decimals go through Python objects; fw.read_arrow() hands them over natively.
+            mask = self.validity(name)
+            values = self[name]
+            decimals = [None if mask is not None and not mask[i] else Decimal(int(v)).scaleb(-column.scale) for i, v in enumerate(values)]
+            return pa.array(decimals, type=pa.decimal128(18, column.scale))
+        if kind is ColumnType.STRING:
+            offsets = pa.foreign_buffer(column.values, (n + 1) * 4, base=self._owner)
+            data = pa.foreign_buffer(column.data, column.data_len, base=self._owner) if column.data else pa.py_buffer(b"")
+            return pa.Array.from_buffers(pa.string(), n, [validity, offsets, data])
+        arrow_type = {
+            ColumnType.INT64: pa.int64(),
+            ColumnType.FLOAT64: pa.float64(),
+            ColumnType.DATE: pa.date32(),
+            ColumnType.TIME: pa.time64("us"),
+            ColumnType.TIMESTAMP: pa.timestamp("us"),
+        }[kind]
+        width = 4 if kind is ColumnType.DATE else 8
+        values = pa.foreign_buffer(column.values, n * width, base=self._owner) if n else pa.py_buffer(b"")
+        return pa.Array.from_buffers(arrow_type, n, [validity, values])
+
+    def __repr__(self) -> str:
+        return f"Table({self.num_rows} rows: {', '.join(self._names)})"
+
+
+def _strings(column: _native.Column) -> list[str]:
+    n = int(column.length)
+    if n == 0:
+        return []
+    offsets = (ctypes.c_int32 * (n + 1)).from_address(column.values)
+    data = ctypes.string_at(column.data, column.data_len) if column.data else b""
+    return [data[offsets[i] : offsets[i + 1]].decode("utf-8") for i in range(n)]

fixedwidth/types.py

@@ -0,0 +1,136 @@
+"""The public value types: column types, column descriptions, options and errors."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import KW_ONLY, dataclass, field
+from enum import IntEnum
+
+
+class ColumnType(IntEnum):
+    """FW_T_*: how a column is parsed and what its values are."""
+
+    STRING = 0
+    INT64 = 1
+    FLOAT64 = 2
+    DECIMAL = 3
+    DATE = 4
+    TIME = 5
+    TIMESTAMP = 6
+
+
+class TrimMode(IntEnum):
+    BOTH = 0
+    TRAILING = 1
+    LEADING = 2
+
+
+class Alignment(IntEnum):
+    DEFAULT = 0
+    LEFT = 1
+    RIGHT = 2
+
+
+class Overflow(IntEnum):
+    THROW = 0
+    TRUNCATE = 1
+
+
+@dataclass(frozen=True)
+class Column:
+    """One column of a layout: its name, zero-based byte `start`, byte `length` and `type`.
+
+    - `nullable`: a blank column (only `trim_char` and spaces) reads as null instead of failing.
+    - `scale`: implied decimals. FLOAT64 divides digits-only text by 10**scale (B3's `N(11)V99`);
+      DECIMAL keeps the unscaled integer.
+    - `format`: the exact date/time format (.NET syntax, e.g. `yyyyMMdd`), or a number format on write.
+    - `trim_char`, `trim_mode`: what is stripped before parsing; a null is written as `trim_char`.
+    - `alignment`, `padding`, `overflow`: how a value is written. `padding='0'` keeps a sign leftmost.
+    """
+
+    name: str
+    start: int
+    length: int
+    type: ColumnType
+    _: KW_ONLY
+    nullable: bool = False
+    scale: int = 0
+    format: str | None = None
+    trim_char: str | None = None
+    trim_mode: TrimMode | None = None
+    alignment: Alignment | None = None
+    padding: str | None = None
+    overflow: Overflow | None = None
+
+
+@dataclass(frozen=True)
+class LayoutOptions:
+    """`line_length`: 0 ends lines at the last column; longer pads them with `filler` on write."""
+
+    line_length: int = 0
+    filler: str | None = None
+
+
+@dataclass(frozen=True)
+class ReadOptions:
+    """Mirrors fw_read_options. `encoding` is a code page or a codec name (`"latin-1"`, `"cp1252"`)."""
+
+    header_lines: int = 0
+    skip_invalid: bool = False
+    degree_of_parallelism: int = 0
+    encoding: int | str | None = None
+    max_line_length: int = 0
+    skip_byte_order_mark: bool | None = None
+
+
+@dataclass(frozen=True)
+class ReaderOptions:
+    """Mirrors fw_reader_options: a sequential read in tables of `batch_rows` rows (0 = 65,536)."""
+
+    read: ReadOptions = field(default_factory=ReadOptions)
+    batch_rows: int = 0
+
+
+@dataclass(frozen=True)
+class WriteOptions:
+    """Mirrors fw_write_options."""
+
+    encoding: int | str | None = None
+    newline: str = "\n"
+    degree_of_parallelism: int = 0
+    batch_rows: int = 0
+
+
+class FixedWidthError(Exception):
+    """Raised when the native library reports a failure."""
+
+
+class ParseError(FixedWidthError):
+    """A line failed to parse. `line_number` is 1-based; `column` is the layout index, -1 for a short line."""
+
+    def __init__(self, message: str, line_number: int, column: int) -> None:
+        super().__init__(message)
+        self.line_number = line_number
+        self.column = column
+
+
+_CODE_PAGES = {
+    "utf-8": 0, "utf8": 0, "ascii": 20127, "us-ascii": 20127,
+    "latin-1": 28591, "latin1": 28591, "iso-8859-1": 28591, "iso8859-1": 28591,
+    "windows-1252": 1252, "cp1252": 1252, "iso-8859-15": 28605, "latin-9": 28605,
+}
+
+
+def code_page(encoding: int | str | None) -> int:
+    """The code page the native options take for `encoding`."""
+    if encoding is None:
+        return 0
+    if isinstance(encoding, int):
+        return encoding
+    name = encoding.strip().lower().replace("_", "-")
+    if name in _CODE_PAGES:
+        return _CODE_PAGES[name]
+    match = re.fullmatch(r"(?:cp|windows-)(\d+)", name)
+    if match:
+        return int(match.group(1))
+    raise ValueError(f"unknown encoding {encoding!r}; pass a code page number")

fixedwidth/writer.py

@@ -0,0 +1,183 @@
+"""Writing: a pandas, polars or pyarrow table through a layout, to a file or to bytes. Requires pyarrow."""
+
+from __future__ import annotations
+
+import ctypes
+import os
+from typing import Any
+
+from fixedwidth import _native
+from fixedwidth.layout import Layout, check
+from fixedwidth.types import (
+    Column,
+    ColumnType,
+    FixedWidthError,
+    WriteOptions,
+    code_page,
+)
+
+
+def write(data: Any, path: str | os.PathLike[str], layout: Layout, options: WriteOptions | None = None) -> None:
+    """Writes every row of `data` (pandas, polars, pyarrow, or anything with `__arrow_c_stream__`) to `path`.
+
+    Columns are matched to the layout by name. A null writes as its column filled with `trim_char`."""
+    lib = _native.load_library()
+    table, keep = _native_table(data, layout)
+    raw, newline = _write_options(options)
+    name, name_len = _native.utf8(os.fspath(path))
+    check(lib.fw_write_file(name, name_len, ctypes.byref(table), layout.handle, ctypes.byref(raw) if raw else None))
+    del keep, newline
+
+
+def write_bytes(data: Any, layout: Layout, options: WriteOptions | None = None) -> bytes:
+    """Same as `write()`, returning the bytes."""
+    lib = _native.load_library()
+    table, keep = _native_table(data, layout)
+    raw, newline = _write_options(options)
+    buffer = _native.Buffer()
+    check(lib.fw_write_memory(ctypes.byref(table), layout.handle, ctypes.byref(raw) if raw else None, ctypes.byref(buffer)))
+    try:
+        return ctypes.string_at(buffer.data, buffer.length) if buffer.length else b""
+    finally:
+        lib.fw_free_buffer(ctypes.byref(buffer))
+        del keep, newline
+
+
+def _write_options(options: WriteOptions | None) -> tuple[_native.WriteOptions | None, object]:
+    if options is None:
+        return None, None
+    newline, newline_len = _native.utf8(options.newline)
+    raw = _native.WriteOptions(
+        struct_size=ctypes.sizeof(_native.WriteOptions),
+        encoding=code_page(options.encoding),
+        newline=ctypes.cast(newline, ctypes.POINTER(ctypes.c_uint8)),
+        newline_len=newline_len,
+        degree_of_parallelism=options.degree_of_parallelism,
+        batch_rows=options.batch_rows,
+    )
+    return raw, newline
+
+
+def _arrow_table(data: Any) -> Any:
+    import pyarrow
+
+    if isinstance(data, pyarrow.RecordBatch):
+        return pyarrow.Table.from_batches([data])
+    if isinstance(data, pyarrow.Table):
+        return data
+    if type(data).__module__.startswith("pandas"):
+        return pyarrow.Table.from_pandas(data, preserve_index=False)
+    if hasattr(data, "__arrow_c_stream__"):
+        return pyarrow.table(data)
+    raise TypeError(f"cannot write {type(data).__name__}: pass a pandas, polars or pyarrow table")
+
+
+def _native_table(data: Any, layout: Layout) -> tuple[_native.Table, list[object]]:
+    import pyarrow
+
+    arrow = _arrow_table(data).combine_chunks()
+    columns = (_native.Column * max(1, len(layout.columns)))()
+    keep: list[object] = [columns]
+    for index, spec in enumerate(layout.columns):
+        if spec.name not in arrow.column_names:
+            raise FixedWidthError(f"column '{spec.name}' of the layout is not in the data")
+        chunked = arrow.column(spec.name)
+        array = chunked.chunk(0) if chunked.num_chunks else pyarrow.array([], type=chunked.type)
+        _fill(columns[index], spec, array, keep)
+    table = _native.Table(column_count=len(layout.columns), row_count=arrow.num_rows, columns=columns)
+    return table, keep
+
+
+def _fill(target: _native.Column, spec: Column, array: Any, keep: list[object]) -> None:
+    import numpy
+    import pyarrow
+
+    kind = ColumnType(spec.type)
+    target.type = int(kind)
+    target.scale = spec.scale if kind is ColumnType.DECIMAL else 0
+    target.length = len(array)
+    if array.null_count:
+        valid = numpy.packbits(~array.is_null().to_numpy(zero_copy_only=False), bitorder="little")
+        keep.append(valid)
+        target.validity = valid.ctypes.data
+    if kind is ColumnType.STRING:
+        _fill_strings(target, array.cast(pyarrow.string()).fill_null(""), keep)
+        return
+    values = numpy.ascontiguousarray(_values(kind, spec, array, pyarrow, numpy))
+    keep.append(values)
+    target.values = values.ctypes.data if len(values) else None
+
+
+def _values(kind: ColumnType, spec: Column, array: Any, pa: Any, np: Any) -> Any:
+    if kind is ColumnType.INT64:
+        return array.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False)
+    if kind is ColumnType.FLOAT64:
+        return array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False)
+    if kind is ColumnType.DECIMAL:
+        return _decimals(spec, array, pa, np)
+    if kind is ColumnType.DATE:
+        return array.cast(pa.date32()).cast(pa.int32()).fill_null(0).to_numpy(zero_copy_only=False)
+    if kind is ColumnType.TIME:
+        return array.cast(pa.time64("us")).cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False)
+    if pa.types.is_timestamp(array.type) and array.type.tz is not None:
+        array = array.cast(pa.timestamp(array.type.unit))
+    return array.cast(pa.timestamp("us")).cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False)
+
+
+def _decimals(spec: Column, array: Any, pa: Any, np: Any) -> Any:
+    """The unscaled integers a DECIMAL column stores, each of which must fit an int64.
+
+    Decimals and integers scale exactly, and a decimal with more digits than the scale fails. Floats, binary
+    approximations to begin with, round half to even to the scale; NaN and infinity fail."""
+    if pa.types.is_decimal(array.type):
+        return np.array([_unscaled(v, spec) for v in array.to_pylist()], dtype=np.int64)
+    factor = 10**spec.scale
+    if pa.types.is_integer(array.type):
+        import pyarrow.compute
+
+        bounds = pyarrow.compute.min_max(array).as_py()
+        for value in (bounds["min"], bounds["max"]):
+            if value is not None:
+                _check_int64(value * factor, value, spec)
+        return array.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) * factor
+    floats = array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False)
+    with np.errstate(over="ignore", invalid="ignore"):
+        scaled = np.round(floats * factor)
+    # Written so NaN fails too; 2**63 is exact in a float64, the int64 range is not.
+    outside = ~((scaled >= -(2.0**63)) & (scaled < 2.0**63))
+    if outside.any():
+        value = floats[int(np.argmax(outside))]
+        raise FixedWidthError(f"value {value} of column '{spec.name}' is not finite or does not fit an int64 at scale {spec.scale}")
+    return scaled.astype(np.int64)
+
+
+def _unscaled(value: Any, spec: Column) -> int:
+    """A decimal as the integer the column stores; more decimals than its scale would be lost, so they fail."""
+    if value is None:
+        return 0
+    scaled = value.scaleb(spec.scale)
+    if scaled != scaled.to_integral_value():
+        raise FixedWidthError(f"value {value} of column '{spec.name}' has more decimals than its scale {spec.scale}")
+    return _check_int64(int(scaled), value, spec)
+
+
+def _check_int64(unscaled: int, value: Any, spec: Column) -> int:
+    if not -(2**63) <= unscaled < 2**63:
+        raise FixedWidthError(f"value {value} of column '{spec.name}' does not fit an int64 at scale {spec.scale}")
+    return unscaled
+
+
+def _fill_strings(target: _native.Column, array: Any, keep: list[object]) -> None:
+    import numpy
+
+    buffers = array.buffers()
+    n = len(array)
+    offsets = numpy.frombuffer(buffers[1], dtype=numpy.int32)[array.offset : array.offset + n + 1] if n else numpy.zeros(1, numpy.int32)
+    base = int(offsets[0])
+    rebased = numpy.ascontiguousarray(offsets - base, dtype=numpy.int32)
+    keep.append(rebased)
+    keep.append(array)
+    target.values = rebased.ctypes.data
+    data_len = int(rebased[-1])
+    target.data_len = data_len
+    target.data = buffers[2].address + base if data_len and buffers[2] is not None else None

fixedwidthparser-2.0.0rc1.dist-info/METADATA

@@ -0,0 +1,111 @@
+Metadata-Version: 2.5
+Name: fixedwidthparser
+Version: 2.0.0-rc.1
+Summary: Read and write fixed-width files through the FixedWidthParser NativeAOT library
+Project-URL: Homepage, https://github.com/GabrielMarquezMatte/FixedWidthParser
+Author: Gabriel Matte
+License: MIT
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Provides-Extra: dev
+Requires-Dist: pytest>=7.0; extra == 'dev'
+Provides-Extra: numpy
+Requires-Dist: numpy>=1.21; extra == 'numpy'
+Provides-Extra: pandas
+Requires-Dist: pandas>=2.0; extra == 'pandas'
+Requires-Dist: pyarrow>=14.0; extra == 'pandas'
+Provides-Extra: polars
+Requires-Dist: polars>=1.0; extra == 'polars'
+Provides-Extra: pyarrow
+Requires-Dist: pyarrow>=14.0; extra == 'pyarrow'
+Description-Content-Type: text/markdown
+
+# fixedwidthparser
+
+Read and write fixed-width files from Python through FixedWidthParser's NativeAOT library: no .NET runtime,
+parallel reads, results as NumPy views, Arrow, pandas or polars.
+
+```bash
+pip install "fixedwidthparser[numpy]"    # fw.read: NumPy views
+pip install "fixedwidthparser[pandas]"    # or [pyarrow], [polars]
+```
+
+Wheels ship for Windows x64, macOS (arm64, x64) and Linux (glibc and musl, x64 and arm64). The package installs
+as `fixedwidthparser` and imports as `fixedwidth`. pyarrow has no musllinux wheels, so on Alpine and other
+musl systems Arrow, pandas, polars and writing need a pyarrow built for them.
+
+```python
+import fixedwidth as fw
+
+# B3's COTAHIST quote history: 245-byte lines, a header (00) and a trailer (99) around the quotes (01).
+layout = fw.Layout([
+    fw.Column("tipreg", 0, 2, fw.String),
+    fw.Column("data_pregao", 2, 8, fw.Date, format="yyyyMMdd"),
+    fw.Column("codneg", 12, 12, fw.String),
+    fw.Column("preabe", 56, 13, fw.Float64, scale=2),   # N(11)V99: two implied decimals
+    fw.Column("quatot", 152, 18, fw.Int64),
+])
+options = fw.ReadOptions(header_lines=1, skip_invalid=True)   # the trailer fails to parse and is skipped
+
+df = fw.read_pandas("COTAHIST_A2023.TXT", layout, options)
+```
+
+## Reading
+
+| Function | Returns |
+|---|---|
+| `fw.read(source, layout, options)` | `fw.Table`: NumPy views over the native columns, no copy |
+| `fw.read_arrow(source, layout, options)` | `pyarrow.Table` (needs pyarrow) |
+| `fw.read_pandas(source, layout, options)` | `pandas.DataFrame` (needs pyarrow and pandas) |
+| `fw.read_polars(source, layout, options)` | `polars.DataFrame` |
+| `fw.read_batches(source, layout, fw.ReaderOptions(batch_rows=...))` | an iterator of `fw.Table` |
+| `fw.read_arrow_stream(source, layout, fw.ReaderOptions(...))` | an Arrow PyCapsule stream for `polars.DataFrame(...)` or `pyarrow.RecordBatchReader.from_stream(...)` |
+
+`source` is a path, or `bytes`/`bytearray`/`memoryview` to read from memory. Every optional setting lives in
+`fw.ReadOptions`: `header_lines`, `skip_invalid`, `degree_of_parallelism` (0 = one thread per processor,
+1 = sequential), `encoding` (a code page or `"latin-1"`, `"cp1252"`, ...), `max_line_length` and
+`skip_byte_order_mark`. A parallel read gives the same rows, in the same order, as a sequential one.
+
+A line that fails to parse raises `fw.ParseError` with `line_number` and `column`, unless `skip_invalid` is set.
+
+`fw.Table` maps a column name to a NumPy array (a list for strings), and has `num_rows`, `column_names`,
+`lines_read`, `lines_skipped`, `validity(name)` (a bool array, `True` where the row holds a value, or `None`
+when the column has no nulls) and `to_arrow()`, which shares the native buffers. Without pyarrow,
+`read_polars` reads sequentially through the Arrow stream.
+
+## Columns
+
+`fw.Column(name, start, length, type, *, nullable, scale, format, trim_char, trim_mode, alignment, padding, overflow)`.
+Positions are bytes. Types: `fw.String`, `fw.Int64`, `fw.Float64`, `fw.Decimal` (int64 unscaled, Arrow
+`decimal128(18, scale)`), `fw.Date`, `fw.Time`, `fw.Timestamp`. `scale` is the number of implied decimals:
+`0000000123456` with `scale=2` reads as `1234.56`. `format` uses .NET date syntax (`yyyyMMdd`, `HHmmss`).
+`trim_mode`, `alignment` and `overflow` take `fw.TrimMode`, `fw.Alignment` and `fw.Overflow`.
+
+`fw.Layout(columns, fw.LayoutOptions(line_length=..., filler=...))` sets how written lines are laid out:
+`line_length` past the last column pads each line, and `filler` (default a space) fills the gaps between
+columns.
+
+## Writing
+
+```python
+fw.write(df, "out.txt", layout, fw.WriteOptions(degree_of_parallelism=0))
+data = fw.write_bytes(df, layout)
+```
+
+Takes pandas, polars, pyarrow or anything with `__arrow_c_stream__` (needs pyarrow), matching columns by name.
+`padding="0"` writes zero-padded numbers with the sign leftmost; a null writes as its column filled with
+`trim_char`. Parallel writes are byte-identical to serial ones. `fw.WriteOptions` also takes `encoding`,
+`newline` (default `"\n"`) and `batch_rows`.
+
+A `fw.Decimal` column stores its value times `10**scale` as an int64. Decimals and integers are scaled
+exactly, and a decimal with more digits than `scale` raises. Floats round half to even to `scale`.
+A value that is NaN, infinite or outside int64 once scaled raises before anything is written.
+
+## Building from source
+
+    python python/scripts/build_native.py      # requires the .NET 10 SDK; publishes the library into src/fixedwidth/_lib
+    pip install -e "python[dev,pandas,polars]"
+    pytest python/tests
+
+To point at a library built elsewhere, set `FIXEDWIDTH_NATIVE_LIB` to its full path.

fixedwidthparser-2.0.0rc1.dist-info/RECORD

@@ -0,0 +1,14 @@
+fixedwidth/__init__.py,sha256=Yo5En8tVtsIsJ3Vhjbev8VVfiar-0yV836M5XZZfVT0,1545
+fixedwidth/_native.py,sha256=IsamIha2UfosWu8JudFpZ7dD6sJQv50F7ISj8nP0Ba0,9061
+fixedwidth/arrow_stream.py,sha256=Ojc-ib_QiDwnR1a5fjsEQwnIySTG34GaYgd8hEilZkY,4363
+fixedwidth/layout.py,sha256=inffqA57gvwvgmai-V04TzFbtz5ubaAYRR1frh8XPLs,3589
+fixedwidth/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fixedwidth/reader.py,sha256=Q72J1HSWBM5caXb8Lv0iKrqOkyzKoSKXUs03finGIqo,6650
+fixedwidth/table.py,sha256=OSZpckLf0ASSMfAENsDftELQZS9wmcuNJFCo6jKvmS0,5577
+fixedwidth/types.py,sha256=ngVbA_dbQ5w-LYt34xDhXKEdisBQhNkcPGvvRR8551c,3805
+fixedwidth/writer.py,sha256=-J1eGoM8-5YB_4hHRns-vrKSbsnsdvpNBkijFAGVkBw,7920
+fixedwidth/_lib/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+fixedwidth/_lib/FixedWidthParser.Native.so,sha256=tTtcswDbo5YmGk03v2BlHbO-9MdDoQF1Dg9rpRv2oEg,2867944
+fixedwidthparser-2.0.0rc1.dist-info/METADATA,sha256=tSU9wrY_K82WZ6_p8rlhgGwaMD6_9bgI2iyH9laf36E,5493
+fixedwidthparser-2.0.0rc1.dist-info/WHEEL,sha256=2kooVln1mqHN7dntfiTqe3M4vnPcJFkeYpfgXu26Plw,106
+fixedwidthparser-2.0.0rc1.dist-info/RECORD,,

fixedwidthparser-2.0.0rc1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: false
+Tag: py3-none-manylinux_2_34_x86_64