parafetch 0.2.0__tar.gz → 0.2.1__tar.gz

{parafetch-0.2.0 → parafetch-0.2.1}/Cargo.lock

@@ -1201,7 +1201,7 @@ dependencies = [
 
 [[package]]
 name = "parafetch"
-version = "0.2.0"
+version = "0.2.1"
 dependencies = [
  "arrow",
  "base64 0.22.1",

{parafetch-0.2.0 → parafetch-0.2.1}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "parafetch"
-version = "0.2.0"
+version = "0.2.1"
 edition = "2021"
 rust-version = "1.86"
 description = "Rust-powered parallel data access for Python: fast SQLite -> Arrow, parallel map, batched REST / JSON-RPC / XML-RPC."

{parafetch-0.2.0 → parafetch-0.2.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: parafetch
-Version: 0.2.0
+Version: 0.2.1
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.12
@@ -41,10 +41,12 @@ Rust with Python's GIL released. Your Python code only sees ordinary lists, dict
 
 - [Installation](#installation)
 - [Quick start](#quick-start)
+- [How it works (diagrams)](#how-it-works)
 - [1. Reading SQLite fast — `read_sqlite`](#1-reading-sqlite-fast)
 - [2. Parallelising one-at-a-time functions — `parallel_map`](#2-parallelising-one-at-a-time-functions)
 - [3. Fetching from size-limited APIs — REST, JSON-RPC, XML-RPC](#3-fetching-from-size-limited-apis)
 - [Authentication — Basic, Bearer, Windows Negotiate (Kerberos/NTLM)](#authentication)
+- [Proxies](#proxies)
 - [Error handling](#error-handling)
 - [How parafetch deals with the GIL](#how-parafetch-deals-with-the-gil)
 - [API reference](#api-reference)
@@ -114,12 +116,159 @@ trades = api.fetch_batched(trade_ids, batch_size=50, path="/trades", id_param="i
 
 ---
 
+## How it works
+
+Every feature is split into a thin **Python** layer and a **Rust** engine. Python handles your
+arguments and the final objects you get back. Rust does the heavy lifting in between, with the
+GIL **released**, so it runs on many CPU cores at once.
+
+In the diagrams below:
+
+```text
+[PY]  runs in Python   (holds the GIL: one thread at a time)
+[RS]  runs in Rust     (GIL released: truly parallel, on all cores)
+═══   the moment control crosses between Python and Rust
+```
+
+### The big picture
+
+```text
+                          your Python code
+                                 │
+        ┌────────────────────────┼─────────────────────────────┐
+        ▼                        ▼                             ▼
+ [PY] read_sqlite()       [PY] parallel_map()        [PY] HttpClient / JsonRpcClient
+      read_sqlite_pandas()     parallel_dict()            XmlRpcClient
+      read_sqlite_arrow()      batched_map()              fetch_batched() …
+      check arguments          split into items           split IDs into batches,
+                                                          build request bodies
+        │                        │                             │
+════════╪════════ PYTHON ──▶ RUST (GIL released) ═════════════╪════════════════
+        ▼                        ▼                             ▼
+ [RS] SQLite reader       [RS] thread scheduler      [RS] tokio HTTP engine
+      • N threads, one         • N OS threads             • connection pool, TLS
+        connection each        • retries + back-off       • concurrency + rate limit
+      • decode rows            • rate limit               • retries, Retry-After
+      • build Arrow            • collect results          • proxies, Basic/Negotiate auth
+        columns                  in input order           • parse JSON / XML-RPC
+        │                        │  ▲                          │
+        │                        ▼  │ (per item)               │
+        │                 [PY] YOUR function runs              │
+        │                      (the only Python step;          │
+        │                       GIL held just for this)        │
+════════╪════════ RUST ──▶ PYTHON ════════════════════════════╪════════════════
+        ▼                        ▼                             ▼
+ [PY] polars / pandas /   [PY] list of results       [PY] flatten batches into one
+      pyarrow (zero copy)      or ParallelError           list / dict, or BatchError
+```
+
+### Where Rust takes over
+
+| step | `read_sqlite` | `parallel_map` (thread mode) | `fetch_batched` / `call_batched` |
+|---|---|---|---|
+| prepare | [PY] validate arguments | [PY] collect items | [PY] split IDs, build request bodies |
+| plan | [RS] find rowid range, split it | [RS] start N worker threads | [RS] queue requests |
+| heavy work | [RS] N threads read SQLite → Arrow | [PY] **your function** (GIL held only while it runs Python code) | [RS] send, retry, authenticate, proxy |
+| waiting | none (CPU-bound) | [RS] back-off sleeps, rate limiting | [RS] network waits, back-off, rate limiting |
+| decode | [RS] typed Arrow buffers | none | [RS] parse JSON / XML-RPC |
+| hand back | [PY] polars/pandas/pyarrow (zero copy) | [PY] results in input order | [PY] build dicts/lists, flatten |
+
+`parallel_map` is the only feature where Python code (yours) runs in the middle. That's why it
+speeds up functions that wait on I/O (the GIL is free while they wait) but not pure-Python number
+crunching. For that, use `mode="process"`.
+
+### `read_sqlite`: parallel range scan into Arrow
+
+```text
+[PY] read_sqlite("market.db", "prices")                    # partitions = CPU count
+═══════════════════════════ PYTHON ──▶ RUST ═════════════════════════════════
+[RS] SELECT MIN(rowid), MAX(rowid)  ─▶  1 … 2,000,000  → split into ranges
+                                              │
+     ┌────────────────┬───────────────────────┼───────────────┬──────────────┐
+     ▼                ▼                       ▼               ▼
+[RS] thread 1    [RS] thread 2           [RS] thread 3   [RS] thread 4
+     own read-only    own read-only           …               …
+     connection       connection
+     rowid 1–500k     500k–1M                 1M–1.5M         1.5M–2M
+     │                │                       │               │
+     ▼                ▼                       ▼               ▼
+[RS] values written straight into typed Arrow buffers (int64, float64, string …)
+     no Python object is ever created per row or per cell
+     └────────────────┴───────────┬───────────┴───────────────┘
+                                  ▼  joined in rowid order
+═══════════════════════════ RUST ──▶ PYTHON ═════════════════════════════════
+[PY]               polars.DataFrame   (zero copy: the Arrow memory is shared)
+```
+
+### `parallel_map`: why threads help I/O-bound functions
+
+```text
+[PY] parallel_map(get_price, isins, workers=3)
+═══════════════════════════ PYTHON ──▶ RUST ═════════════════════════════════
+[RS] 3 OS threads pull items from a shared queue
+
+            time ───────────────────────────────────────────────────▶
+thread 1    ▓░░░░░░░ waiting for DB ░░░░░░░▓  ▓░░░░░░░ waiting ░░░░░▓
+thread 2     ▓░░░░░░░ waiting for DB ░░░░░░░▓  ▓░░░░░░░ waiting ░░░░░▓
+thread 3      ▓░░░░░░░ waiting for DB ░░░░░░░▓  ▓░░░░░░░ waiting ░░░░▓
+
+▓ = [PY] your function's Python code: holds the GIL, but only briefly
+░ = waiting on network / DB: GIL released, so the other threads run
+    → 3 workers ≈ 3× faster, 32 workers ≈ 32× (up to what the server allows)
+    [RS] between items: retry back-off sleeps and rate-limit waits, GIL released
+═══════════════════════════ RUST ──▶ PYTHON ═════════════════════════════════
+[PY] results in input order (or ParallelError with the partial results)
+
+CPU-bound Python is all ▓ and threads can't overlap it → use mode="process"
+(one GIL per process) or mode="interpreter" on Python 3.14+.
+```
+
+### `fetch_batched` / `call_batched`: size-limited APIs
+
+```text
+[PY] api.fetch_batched(10_000 ids, batch_size=50)
+[PY] split into 200 batches → build 200 request bodies ("$ids" filled in)
+═══════════════════════════ PYTHON ──▶ RUST ═════════════════════════════════
+[RS] ┌── at most `concurrency` in flight ── at most `rate_limit` per second ──┐
+[RS] │  POST /trades {"ids": [1 … 50]}       ───▶ 200 OK                      │
+[RS] │  POST /trades {"ids": [51 … 100]}     ───▶ 503 ─ back-off ─ retry ─▶ 200
+[RS] │  POST /trades {"ids": [101 … 150]}    ───▶ 429 ─ waits Retry-After ─▶ 200
+[RS] │  …                                                                    │
+[RS] └───────────────────────────────────────────────────────────────────────┘
+[RS] parse JSON / XML-RPC responses in parallel
+═══════════════════════════ RUST ──▶ PYTHON ═════════════════════════════════
+[PY] result_key="data.trades" → flatten in input order → [trade, trade, …]
+[PY] batches that still fail → BatchError with .results and .failed_ids
+```
+
+### What happens to each HTTP request (all inside Rust)
+
+```text
+[RS] request ─▶ proxy selection (same rules as requests)
+     │          ├─ host in no_proxy?                   → connect directly
+     │          └─ first match of  scheme://host → scheme → all://host → all
+     │             (proxy URL may carry user:password → Proxy-Authorization)
+     ├────────▶ authentication
+     │          ├─ auth=("user","pwd") / bearer_token  → header on every request
+     │          └─ auth=HTTPNegotiateAuth()            → Windows SSPI handshake:
+     │
+     │             GET /api                                   ─▶ 401  WWW-Authenticate: Negotiate
+     │             GET /api  Authorization: Negotiate <token> ─▶ 401  + challenge   (NTLM only)
+     │             GET /api  Authorization: Negotiate <reply> ─▶ 200  OK
+     │             └─ one kept-alive connection; later requests on it skip the handshake
+     │
+     ├────────▶ retry on network errors / 408 / 425 / 429 / 5xx  (exponential back-off + jitter)
+     └────────▶ parse body (JSON / XML-RPC / text / bytes) → outcome handed back to [PY]
+```
+
+---
+
 ## 1. Reading SQLite fast
 
 ```python
 pf.read_sqlite(path, table=None, *, query=None, columns=None, where=None, params=None,
                partition_on=None, partitions=None, schema_overrides=None,
-               batch_size=131072, immutable=False) -> polars.DataFrame
+               batch_size=131072, immutable=False, busy_timeout=5.0) -> polars.DataFrame
 ```
 
 `read_sqlite_pandas(...)` (pandas DataFrame) and `read_sqlite_arrow(...)` (pyarrow Table) take
@@ -215,6 +364,26 @@ pf.read_sqlite("market.db", "prices", schema_overrides={"isin": "str", "qty": "f
 Dates are stored as text in SQLite and come back as strings. Convert them after reading, e.g.
 `df.with_columns(pl.col("asof").str.to_date())` in polars, or `pd.to_datetime(pdf["asof"])` in pandas.
 
+### Locking and databases that are being written to
+
+parafetch **never writes and never takes a write lock**: every connection is opened read-only, and
+the file is left byte-for-byte unchanged. Locks are released as soon as the read ends, fails or is
+interrupted. What a running read means for *other* processes depends on the journal mode:
+
+| journal mode | while `read_sqlite` runs |
+|---|---|
+| **WAL** (`PRAGMA journal_mode=WAL`) | readers and writers never block each other (recommended for live databases) |
+| rollback journal (SQLite's default) | the read holds a shared lock, so a writer can prepare changes but **waits to commit** until the read finishes |
+| `immutable=True` | no locks at all. Fastest, but only for files nothing writes to (snapshots, copies). |
+
+- `busy_timeout=5.0` (default): if a writer is committing just as the read starts, parafetch waits up
+  to this many seconds instead of failing with `database is locked`.
+- Parallel partitions use one connection each. If another process commits *during* the read,
+  different partitions can see the data before and after that commit. For an exact point-in-time
+  snapshot of a live database, use `partitions=1` or read a copy.
+- SQLite's locking is unreliable on network drives (SMB/NFS). Prefer local disk for databases that are
+  written to while being read.
+
 ### Limitations
 
 - Tables declared `WITHOUT ROWID` are read on a single thread unless you pass `partition_on`.
@@ -373,7 +542,7 @@ api.request_many([                                                # arbitrary mi
 ])
 ```
 
-Each `request_many` entry takes `method`, `path` (or a full URL), `params`, `json`, `data`,
+Each `request_many` entry takes `method`, `path` or `url`, `params`, `json`, `data`,
 `headers` and `parse` (`"json"`, `"text"`, `"bytes"` or `"auto"`).
 
 ### JSON-RPC — `JsonRpcClient`
@@ -429,7 +598,10 @@ prices = xr.call_many("pricing.get", isins, multicall_size=100)  # system.multic
 | `timeout`, `connect_timeout` | `60`, `10` | seconds (`None` disables the timeout) |
 | `verify` | `True` | set `False` to skip TLS certificate checks (not recommended) |
 | `ca_cert` | `None` | path to a PEM bundle with extra trusted CAs |
-| `proxy` | `None` | e.g. `"http://proxy.corp:8080"`. `HTTP(S)_PROXY` env vars are used automatically. |
+| `proxy` | `None` | one proxy for everything, e.g. `"http://user:pwd@proxy.corp:8080"` |
+| `proxies` | `None` | `requests`-style dict, e.g. `{"http": ..., "https": ..., "http://host": ...}`. See [Proxies](#proxies). |
+| `no_proxy` | `None` | hosts to reach directly: `".corp.local,10.0.0.0/8"` or a list |
+| `trust_env` | `True` | use `HTTP(S)_PROXY` / `NO_PROXY` env vars and the Windows proxy settings, like `requests` |
 | `user_agent` | `parafetch/<version>` | |
 
 Create a client once and reuse it. It keeps connections open between calls.
@@ -461,6 +633,7 @@ Every client takes the same `auth=` argument as the `requests` library:
 | `requests_negotiate_sspi.HttpNegotiateAuth()` | `auth=pf.HttpNegotiateAuth()` |
 | `requests_kerberos.HTTPKerberosAuth()` | `auth=pf.HTTPNegotiateAuth(package="Kerberos")` |
 | `requests_ntlm.HttpNtlmAuth(user, pwd)` | `auth=pf.HTTPNegotiateAuth(username=user, password=pwd, package="NTLM")` |
+| `proxies={...}`, `trust_env=False` | the same `proxies=` / `trust_env=` arguments ([Proxies](#proxies)) |
 
 ### Windows integrated auth (HTTP Negotiate: Kerberos / NTLM)
 
@@ -508,6 +681,44 @@ Getting `HTTP 401` with your own (logged-in) account? See the [FAQ](#faq--troubl
 
 ---
 
+## Proxies
+
+Proxy settings work exactly like in `requests`, including credentials in the URL:
+
+```python
+# one proxy for everything (user/password in the URL, like requests)
+api = pf.HttpClient(base, proxy="http://jdoe:s3cret@proxy.corp:8080")
+
+# requests-style dict
+api = pf.HttpClient(base, proxies={
+    "http":  "http://jdoe:s3cret@proxy.corp:8080",
+    "https": "http://jdoe:s3cret@proxy.corp:8080",
+    "https://internal.corp.local": "",         # empty / None = connect directly
+    "all://legacy.example.com": "http://other-proxy:3128",
+})
+
+# hosts that skip the proxy
+api = pf.HttpClient(base, proxy="http://proxy.corp:8080", no_proxy=".corp.local,10.0.0.0/8,localhost")
+```
+
+- **Lookup order per request** is the same as `requests`: `scheme://host`, then `scheme`, then
+  `all://host`, then `all`. The first key present wins.
+- **Credentials:** `user:password@` in the proxy URL is sent as `Proxy-Authorization: Basic`, both for
+  plain HTTP and for HTTPS (`CONNECT`) tunnels. Percent-encode special characters in the password, as
+  you would for `requests`: `@` → `%40`, `:` → `%3A`, `#` → `%23`, `%` → `%25`.
+  Example: `urllib.parse.quote(password, safe="")`.
+- **`no_proxy`** accepts host names (`intranet`), domain suffixes (`.corp.local` or `*.corp.local`),
+  IP addresses, CIDR ranges (`10.0.0.0/8`), `host:port`, `*` (everything) and the Windows `<local>` token.
+- **Environment and system settings** (`trust_env=True`, the default): `HTTP_PROXY`, `HTTPS_PROXY`,
+  `ALL_PROXY` and `NO_PROXY` are used, or on Windows the system proxy from *Internet Options*,
+  including its bypass list. Explicit `proxies` take precedence key by key. `trust_env=False` ignores
+  all of them.
+- All three clients (`HttpClient`, `JsonRpcClient`, `XmlRpcClient`) take these options.
+
+Not supported yet: proxies that require Windows sign-in (Negotiate/NTLM, `407 Proxy-Authenticate: Negotiate`).
+
+---
+
 ## Error handling
 
 parafetch never throws away work that succeeded.
@@ -569,7 +780,7 @@ Rule of thumb: **waiting on I/O → `thread`** (the default), **async code → `
 
 | function | returns |
 |---|---|
-| `read_sqlite(path, table=None, *, query=None, columns=None, where=None, params=None, partition_on=None, partitions=None, schema_overrides=None, batch_size=131072, immutable=False)` | `polars.DataFrame` |
+| `read_sqlite(path, table=None, *, query=None, columns=None, where=None, params=None, partition_on=None, partitions=None, schema_overrides=None, batch_size=131072, immutable=False, busy_timeout=5.0)` | `polars.DataFrame` |
 | `read_sqlite_pandas(path, table=None, **kwargs)` | `pandas.DataFrame` (needs pandas) |
 | `read_sqlite_arrow(path, table=None, **kwargs)` | `pyarrow.Table` |
 
@@ -661,8 +872,9 @@ Lower `concurrency` or set `rate_limit`. 429 responses are retried automatically
 That column contains both binary and numeric data. Use `schema_overrides={"col": "bytes"}` (or `"str"`).
 
 **Can I read the database while another process writes to it?**
-Yes, with the default settings. Readers use normal SQLite locking (WAL mode is ideal). Don't use
-`immutable=True` in that case.
+Yes. parafetch only reads and never takes a write lock. In WAL mode nothing blocks; in the default
+rollback mode a writer waits to commit until the read finishes. Don't use `immutable=True` in that
+case. See [Locking](#locking-and-databases-that-are-being-written-to).
 
 ---
 

{parafetch-0.2.0 → parafetch-0.2.1}/README.md

@@ -19,10 +19,12 @@ Rust with Python's GIL released. Your Python code only sees ordinary lists, dict
 
 - [Installation](#installation)
 - [Quick start](#quick-start)
+- [How it works (diagrams)](#how-it-works)
 - [1. Reading SQLite fast — `read_sqlite`](#1-reading-sqlite-fast)
 - [2. Parallelising one-at-a-time functions — `parallel_map`](#2-parallelising-one-at-a-time-functions)
 - [3. Fetching from size-limited APIs — REST, JSON-RPC, XML-RPC](#3-fetching-from-size-limited-apis)
 - [Authentication — Basic, Bearer, Windows Negotiate (Kerberos/NTLM)](#authentication)
+- [Proxies](#proxies)
 - [Error handling](#error-handling)
 - [How parafetch deals with the GIL](#how-parafetch-deals-with-the-gil)
 - [API reference](#api-reference)
@@ -92,12 +94,159 @@ trades = api.fetch_batched(trade_ids, batch_size=50, path="/trades", id_param="i
 
 ---
 
+## How it works
+
+Every feature is split into a thin **Python** layer and a **Rust** engine. Python handles your
+arguments and the final objects you get back. Rust does the heavy lifting in between, with the
+GIL **released**, so it runs on many CPU cores at once.
+
+In the diagrams below:
+
+```text
+[PY]  runs in Python   (holds the GIL: one thread at a time)
+[RS]  runs in Rust     (GIL released: truly parallel, on all cores)
+═══   the moment control crosses between Python and Rust
+```
+
+### The big picture
+
+```text
+                          your Python code
+                                 │
+        ┌────────────────────────┼─────────────────────────────┐
+        ▼                        ▼                             ▼
+ [PY] read_sqlite()       [PY] parallel_map()        [PY] HttpClient / JsonRpcClient
+      read_sqlite_pandas()     parallel_dict()            XmlRpcClient
+      read_sqlite_arrow()      batched_map()              fetch_batched() …
+      check arguments          split into items           split IDs into batches,
+                                                          build request bodies
+        │                        │                             │
+════════╪════════ PYTHON ──▶ RUST (GIL released) ═════════════╪════════════════
+        ▼                        ▼                             ▼
+ [RS] SQLite reader       [RS] thread scheduler      [RS] tokio HTTP engine
+      • N threads, one         • N OS threads             • connection pool, TLS
+        connection each        • retries + back-off       • concurrency + rate limit
+      • decode rows            • rate limit               • retries, Retry-After
+      • build Arrow            • collect results          • proxies, Basic/Negotiate auth
+        columns                  in input order           • parse JSON / XML-RPC
+        │                        │  ▲                          │
+        │                        ▼  │ (per item)               │
+        │                 [PY] YOUR function runs              │
+        │                      (the only Python step;          │
+        │                       GIL held just for this)        │
+════════╪════════ RUST ──▶ PYTHON ════════════════════════════╪════════════════
+        ▼                        ▼                             ▼
+ [PY] polars / pandas /   [PY] list of results       [PY] flatten batches into one
+      pyarrow (zero copy)      or ParallelError           list / dict, or BatchError
+```
+
+### Where Rust takes over
+
+| step | `read_sqlite` | `parallel_map` (thread mode) | `fetch_batched` / `call_batched` |
+|---|---|---|---|
+| prepare | [PY] validate arguments | [PY] collect items | [PY] split IDs, build request bodies |
+| plan | [RS] find rowid range, split it | [RS] start N worker threads | [RS] queue requests |
+| heavy work | [RS] N threads read SQLite → Arrow | [PY] **your function** (GIL held only while it runs Python code) | [RS] send, retry, authenticate, proxy |
+| waiting | none (CPU-bound) | [RS] back-off sleeps, rate limiting | [RS] network waits, back-off, rate limiting |
+| decode | [RS] typed Arrow buffers | none | [RS] parse JSON / XML-RPC |
+| hand back | [PY] polars/pandas/pyarrow (zero copy) | [PY] results in input order | [PY] build dicts/lists, flatten |
+
+`parallel_map` is the only feature where Python code (yours) runs in the middle. That's why it
+speeds up functions that wait on I/O (the GIL is free while they wait) but not pure-Python number
+crunching. For that, use `mode="process"`.
+
+### `read_sqlite`: parallel range scan into Arrow
+
+```text
+[PY] read_sqlite("market.db", "prices")                    # partitions = CPU count
+═══════════════════════════ PYTHON ──▶ RUST ═════════════════════════════════
+[RS] SELECT MIN(rowid), MAX(rowid)  ─▶  1 … 2,000,000  → split into ranges
+                                              │
+     ┌────────────────┬───────────────────────┼───────────────┬──────────────┐
+     ▼                ▼                       ▼               ▼
+[RS] thread 1    [RS] thread 2           [RS] thread 3   [RS] thread 4
+     own read-only    own read-only           …               …
+     connection       connection
+     rowid 1–500k     500k–1M                 1M–1.5M         1.5M–2M
+     │                │                       │               │
+     ▼                ▼                       ▼               ▼
+[RS] values written straight into typed Arrow buffers (int64, float64, string …)
+     no Python object is ever created per row or per cell
+     └────────────────┴───────────┬───────────┴───────────────┘
+                                  ▼  joined in rowid order
+═══════════════════════════ RUST ──▶ PYTHON ═════════════════════════════════
+[PY]               polars.DataFrame   (zero copy: the Arrow memory is shared)
+```
+
+### `parallel_map`: why threads help I/O-bound functions
+
+```text
+[PY] parallel_map(get_price, isins, workers=3)
+═══════════════════════════ PYTHON ──▶ RUST ═════════════════════════════════
+[RS] 3 OS threads pull items from a shared queue
+
+            time ───────────────────────────────────────────────────▶
+thread 1    ▓░░░░░░░ waiting for DB ░░░░░░░▓  ▓░░░░░░░ waiting ░░░░░▓
+thread 2     ▓░░░░░░░ waiting for DB ░░░░░░░▓  ▓░░░░░░░ waiting ░░░░░▓
+thread 3      ▓░░░░░░░ waiting for DB ░░░░░░░▓  ▓░░░░░░░ waiting ░░░░▓
+
+▓ = [PY] your function's Python code: holds the GIL, but only briefly
+░ = waiting on network / DB: GIL released, so the other threads run
+    → 3 workers ≈ 3× faster, 32 workers ≈ 32× (up to what the server allows)
+    [RS] between items: retry back-off sleeps and rate-limit waits, GIL released
+═══════════════════════════ RUST ──▶ PYTHON ═════════════════════════════════
+[PY] results in input order (or ParallelError with the partial results)
+
+CPU-bound Python is all ▓ and threads can't overlap it → use mode="process"
+(one GIL per process) or mode="interpreter" on Python 3.14+.
+```
+
+### `fetch_batched` / `call_batched`: size-limited APIs
+
+```text
+[PY] api.fetch_batched(10_000 ids, batch_size=50)
+[PY] split into 200 batches → build 200 request bodies ("$ids" filled in)
+═══════════════════════════ PYTHON ──▶ RUST ═════════════════════════════════
+[RS] ┌── at most `concurrency` in flight ── at most `rate_limit` per second ──┐
+[RS] │  POST /trades {"ids": [1 … 50]}       ───▶ 200 OK                      │
+[RS] │  POST /trades {"ids": [51 … 100]}     ───▶ 503 ─ back-off ─ retry ─▶ 200
+[RS] │  POST /trades {"ids": [101 … 150]}    ───▶ 429 ─ waits Retry-After ─▶ 200
+[RS] │  …                                                                    │
+[RS] └───────────────────────────────────────────────────────────────────────┘
+[RS] parse JSON / XML-RPC responses in parallel
+═══════════════════════════ RUST ──▶ PYTHON ═════════════════════════════════
+[PY] result_key="data.trades" → flatten in input order → [trade, trade, …]
+[PY] batches that still fail → BatchError with .results and .failed_ids
+```
+
+### What happens to each HTTP request (all inside Rust)
+
+```text
+[RS] request ─▶ proxy selection (same rules as requests)
+     │          ├─ host in no_proxy?                   → connect directly
+     │          └─ first match of  scheme://host → scheme → all://host → all
+     │             (proxy URL may carry user:password → Proxy-Authorization)
+     ├────────▶ authentication
+     │          ├─ auth=("user","pwd") / bearer_token  → header on every request
+     │          └─ auth=HTTPNegotiateAuth()            → Windows SSPI handshake:
+     │
+     │             GET /api                                   ─▶ 401  WWW-Authenticate: Negotiate
+     │             GET /api  Authorization: Negotiate <token> ─▶ 401  + challenge   (NTLM only)
+     │             GET /api  Authorization: Negotiate <reply> ─▶ 200  OK
+     │             └─ one kept-alive connection; later requests on it skip the handshake
+     │
+     ├────────▶ retry on network errors / 408 / 425 / 429 / 5xx  (exponential back-off + jitter)
+     └────────▶ parse body (JSON / XML-RPC / text / bytes) → outcome handed back to [PY]
+```
+
+---
+
 ## 1. Reading SQLite fast
 
 ```python
 pf.read_sqlite(path, table=None, *, query=None, columns=None, where=None, params=None,
                partition_on=None, partitions=None, schema_overrides=None,
-               batch_size=131072, immutable=False) -> polars.DataFrame
+               batch_size=131072, immutable=False, busy_timeout=5.0) -> polars.DataFrame
 ```
 
 `read_sqlite_pandas(...)` (pandas DataFrame) and `read_sqlite_arrow(...)` (pyarrow Table) take
@@ -193,6 +342,26 @@ pf.read_sqlite("market.db", "prices", schema_overrides={"isin": "str", "qty": "f
 Dates are stored as text in SQLite and come back as strings. Convert them after reading, e.g.
 `df.with_columns(pl.col("asof").str.to_date())` in polars, or `pd.to_datetime(pdf["asof"])` in pandas.
 
+### Locking and databases that are being written to
+
+parafetch **never writes and never takes a write lock**: every connection is opened read-only, and
+the file is left byte-for-byte unchanged. Locks are released as soon as the read ends, fails or is
+interrupted. What a running read means for *other* processes depends on the journal mode:
+
+| journal mode | while `read_sqlite` runs |
+|---|---|
+| **WAL** (`PRAGMA journal_mode=WAL`) | readers and writers never block each other (recommended for live databases) |
+| rollback journal (SQLite's default) | the read holds a shared lock, so a writer can prepare changes but **waits to commit** until the read finishes |
+| `immutable=True` | no locks at all. Fastest, but only for files nothing writes to (snapshots, copies). |
+
+- `busy_timeout=5.0` (default): if a writer is committing just as the read starts, parafetch waits up
+  to this many seconds instead of failing with `database is locked`.
+- Parallel partitions use one connection each. If another process commits *during* the read,
+  different partitions can see the data before and after that commit. For an exact point-in-time
+  snapshot of a live database, use `partitions=1` or read a copy.
+- SQLite's locking is unreliable on network drives (SMB/NFS). Prefer local disk for databases that are
+  written to while being read.
+
 ### Limitations
 
 - Tables declared `WITHOUT ROWID` are read on a single thread unless you pass `partition_on`.
@@ -351,7 +520,7 @@ api.request_many([                                                # arbitrary mi
 ])
 ```
 
-Each `request_many` entry takes `method`, `path` (or a full URL), `params`, `json`, `data`,
+Each `request_many` entry takes `method`, `path` or `url`, `params`, `json`, `data`,
 `headers` and `parse` (`"json"`, `"text"`, `"bytes"` or `"auto"`).
 
 ### JSON-RPC — `JsonRpcClient`
@@ -407,7 +576,10 @@ prices = xr.call_many("pricing.get", isins, multicall_size=100)  # system.multic
 | `timeout`, `connect_timeout` | `60`, `10` | seconds (`None` disables the timeout) |
 | `verify` | `True` | set `False` to skip TLS certificate checks (not recommended) |
 | `ca_cert` | `None` | path to a PEM bundle with extra trusted CAs |
-| `proxy` | `None` | e.g. `"http://proxy.corp:8080"`. `HTTP(S)_PROXY` env vars are used automatically. |
+| `proxy` | `None` | one proxy for everything, e.g. `"http://user:pwd@proxy.corp:8080"` |
+| `proxies` | `None` | `requests`-style dict, e.g. `{"http": ..., "https": ..., "http://host": ...}`. See [Proxies](#proxies). |
+| `no_proxy` | `None` | hosts to reach directly: `".corp.local,10.0.0.0/8"` or a list |
+| `trust_env` | `True` | use `HTTP(S)_PROXY` / `NO_PROXY` env vars and the Windows proxy settings, like `requests` |
 | `user_agent` | `parafetch/<version>` | |
 
 Create a client once and reuse it. It keeps connections open between calls.
@@ -439,6 +611,7 @@ Every client takes the same `auth=` argument as the `requests` library:
 | `requests_negotiate_sspi.HttpNegotiateAuth()` | `auth=pf.HttpNegotiateAuth()` |
 | `requests_kerberos.HTTPKerberosAuth()` | `auth=pf.HTTPNegotiateAuth(package="Kerberos")` |
 | `requests_ntlm.HttpNtlmAuth(user, pwd)` | `auth=pf.HTTPNegotiateAuth(username=user, password=pwd, package="NTLM")` |
+| `proxies={...}`, `trust_env=False` | the same `proxies=` / `trust_env=` arguments ([Proxies](#proxies)) |
 
 ### Windows integrated auth (HTTP Negotiate: Kerberos / NTLM)
 
@@ -486,6 +659,44 @@ Getting `HTTP 401` with your own (logged-in) account? See the [FAQ](#faq--troubl
 
 ---
 
+## Proxies
+
+Proxy settings work exactly like in `requests`, including credentials in the URL:
+
+```python
+# one proxy for everything (user/password in the URL, like requests)
+api = pf.HttpClient(base, proxy="http://jdoe:s3cret@proxy.corp:8080")
+
+# requests-style dict
+api = pf.HttpClient(base, proxies={
+    "http":  "http://jdoe:s3cret@proxy.corp:8080",
+    "https": "http://jdoe:s3cret@proxy.corp:8080",
+    "https://internal.corp.local": "",         # empty / None = connect directly
+    "all://legacy.example.com": "http://other-proxy:3128",
+})
+
+# hosts that skip the proxy
+api = pf.HttpClient(base, proxy="http://proxy.corp:8080", no_proxy=".corp.local,10.0.0.0/8,localhost")
+```
+
+- **Lookup order per request** is the same as `requests`: `scheme://host`, then `scheme`, then
+  `all://host`, then `all`. The first key present wins.
+- **Credentials:** `user:password@` in the proxy URL is sent as `Proxy-Authorization: Basic`, both for
+  plain HTTP and for HTTPS (`CONNECT`) tunnels. Percent-encode special characters in the password, as
+  you would for `requests`: `@` → `%40`, `:` → `%3A`, `#` → `%23`, `%` → `%25`.
+  Example: `urllib.parse.quote(password, safe="")`.
+- **`no_proxy`** accepts host names (`intranet`), domain suffixes (`.corp.local` or `*.corp.local`),
+  IP addresses, CIDR ranges (`10.0.0.0/8`), `host:port`, `*` (everything) and the Windows `<local>` token.
+- **Environment and system settings** (`trust_env=True`, the default): `HTTP_PROXY`, `HTTPS_PROXY`,
+  `ALL_PROXY` and `NO_PROXY` are used, or on Windows the system proxy from *Internet Options*,
+  including its bypass list. Explicit `proxies` take precedence key by key. `trust_env=False` ignores
+  all of them.
+- All three clients (`HttpClient`, `JsonRpcClient`, `XmlRpcClient`) take these options.
+
+Not supported yet: proxies that require Windows sign-in (Negotiate/NTLM, `407 Proxy-Authenticate: Negotiate`).
+
+---
+
 ## Error handling
 
 parafetch never throws away work that succeeded.
@@ -547,7 +758,7 @@ Rule of thumb: **waiting on I/O → `thread`** (the default), **async code → `
 
 | function | returns |
 |---|---|
-| `read_sqlite(path, table=None, *, query=None, columns=None, where=None, params=None, partition_on=None, partitions=None, schema_overrides=None, batch_size=131072, immutable=False)` | `polars.DataFrame` |
+| `read_sqlite(path, table=None, *, query=None, columns=None, where=None, params=None, partition_on=None, partitions=None, schema_overrides=None, batch_size=131072, immutable=False, busy_timeout=5.0)` | `polars.DataFrame` |
 | `read_sqlite_pandas(path, table=None, **kwargs)` | `pandas.DataFrame` (needs pandas) |
 | `read_sqlite_arrow(path, table=None, **kwargs)` | `pyarrow.Table` |
 
@@ -639,8 +850,9 @@ Lower `concurrency` or set `rate_limit`. 429 responses are retried automatically
 That column contains both binary and numeric data. Use `schema_overrides={"col": "bytes"}` (or `"str"`).
 
 **Can I read the database while another process writes to it?**
-Yes, with the default settings. Readers use normal SQLite locking (WAL mode is ideal). Don't use
-`immutable=True` in that case.
+Yes. parafetch only reads and never takes a write lock. In WAL mode nothing blocks; in the default
+rollback mode a writer waits to commit until the read finishes. Don't use `immutable=True` in that
+case. See [Locking](#locking-and-databases-that-are-being-written-to).
 
 ---
 

{parafetch-0.2.0 → parafetch-0.2.1}/python/parafetch/_core.pyi

@@ -18,6 +18,7 @@ def read_sqlite(
     schema_overrides: dict[str, str] | None = None,
     batch_size: int = 131072,
     immutable: bool = False,
+    busy_timeout: float = 5.0,
 ) -> tuple[pa.Schema, list[pa.RecordBatch]]: ...
 def parallel_map(
     func: Callable[..., Any],
@@ -49,7 +50,9 @@ class HttpTransport:
         rate_limit: float | None = None,
         verify: bool = True,
         ca_cert: str | None = None,
-        proxy: str | None = None,
+        proxies: list[tuple[str, str | None]] | None = None,
+        no_proxy: str | None = None,
+        trust_env: bool = True,
         basic_auth: tuple[str, str | None] | None = None,
         user_agent: str | None = None,
         negotiate: tuple[str, str, str | None, str | None, str | None, str | None, bool, bool] | None = None,