pi-cloudflare 0.1.0

package/skills/cloudflare/references/r2/patterns.md

@@ -0,0 +1,193 @@
+# R2 Patterns & Best Practices
+
+## Streaming Large Files
+
+```typescript
+const object = await env.MY_BUCKET.get(key);
+if (!object) return new Response('Not found', { status: 404 });
+
+const headers = new Headers();
+object.writeHttpMetadata(headers);
+headers.set('etag', object.httpEtag);
+
+return new Response(object.body, { headers });
+```
+
+## Conditional GET (304 Not Modified)
+
+```typescript
+const ifNoneMatch = request.headers.get('if-none-match');
+const object = await env.MY_BUCKET.get(key, {
+  onlyIf: { etagDoesNotMatch: ifNoneMatch?.replace(/"/g, '') || '' }
+});
+
+if (!object) return new Response('Not found', { status: 404 });
+if (!object.body) return new Response(null, { status: 304, headers: { 'etag': object.httpEtag } });
+
+return new Response(object.body, { headers: { 'etag': object.httpEtag } });
+```
+
+## Upload with Validation
+
+```typescript
+const key = url.pathname.slice(1);
+if (!key || key.includes('..')) return new Response('Invalid key', { status: 400 });
+
+const object = await env.MY_BUCKET.put(key, request.body, {
+  httpMetadata: { contentType: request.headers.get('content-type') || 'application/octet-stream' },
+  customMetadata: { uploadedAt: new Date().toISOString(), ip: request.headers.get('cf-connecting-ip') || 'unknown' }
+});
+
+return Response.json({ key: object.key, size: object.size, etag: object.httpEtag });
+```
+
+## Multipart with Progress
+
+```typescript
+const PART_SIZE = 5 * 1024 * 1024; // 5MB
+const partCount = Math.ceil(file.size / PART_SIZE);
+const multipart = await env.MY_BUCKET.createMultipartUpload(key, { httpMetadata: { contentType: file.type } });
+
+const uploadedParts: R2UploadedPart[] = [];
+try {
+  for (let i = 0; i < partCount; i++) {
+    const start = i * PART_SIZE;
+    const part = await multipart.uploadPart(i + 1, file.slice(start, start + PART_SIZE));
+    uploadedParts.push(part);
+    onProgress?.(Math.round(((i + 1) / partCount) * 100));
+  }
+  return await multipart.complete(uploadedParts);
+} catch (error) {
+  await multipart.abort();
+  throw error;
+}
+```
+
+## Batch Delete
+
+```typescript
+async function deletePrefix(prefix: string, env: Env) {
+  let cursor: string | undefined;
+  let truncated = true;
+
+  while (truncated) {
+    const listed = await env.MY_BUCKET.list({ prefix, limit: 1000, cursor });
+    if (listed.objects.length > 0) {
+      await env.MY_BUCKET.delete(listed.objects.map(o => o.key));
+    }
+    truncated = listed.truncated;
+    cursor = listed.cursor;
+  }
+}
+```
+
+## Checksum Validation & Storage Transitions
+
+```typescript
+// Upload with checksum
+const hash = await crypto.subtle.digest('SHA-256', data);
+await env.MY_BUCKET.put(key, data, { sha256: hash });
+
+// Transition storage class (requires S3 SDK)
+import { S3Client, CopyObjectCommand } from '@aws-sdk/client-s3';
+await s3.send(new CopyObjectCommand({
+  Bucket: 'my-bucket', Key: key,
+  CopySource: `/my-bucket/${key}`,
+  StorageClass: 'STANDARD_IA'
+}));
+```
+
+## Client-Side Uploads (Presigned URLs)
+
+```typescript
+import { S3Client } from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import { PutObjectCommand } from '@aws-sdk/client-s3';
+
+// Worker: Generate presigned upload URL
+const s3 = new S3Client({
+  region: 'auto',
+  endpoint: `https://${env.ACCOUNT_ID}.r2.cloudflarestorage.com`,
+  credentials: { accessKeyId: env.R2_ACCESS_KEY_ID, secretAccessKey: env.R2_SECRET_ACCESS_KEY }
+});
+
+const url = await getSignedUrl(s3, new PutObjectCommand({ Bucket: 'my-bucket', Key: key }), { expiresIn: 3600 });
+return Response.json({ uploadUrl: url });
+
+// Client: Upload directly
+const { uploadUrl } = await fetch('/api/upload-url').then(r => r.json());
+await fetch(uploadUrl, { method: 'PUT', body: file });
+```
+
+## Caching with Cache API
+
+```typescript
+export default {
+  async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise<Response> {
+    const cache = caches.default;
+    const url = new URL(request.url);
+    const cacheKey = new Request(url.toString(), request);
+
+    // Check cache first
+    let response = await cache.match(cacheKey);
+    if (response) return response;
+
+    // Fetch from R2
+    const key = url.pathname.slice(1);
+    const object = await env.MY_BUCKET.get(key);
+    if (!object) return new Response('Not found', { status: 404 });
+
+    const headers = new Headers();
+    object.writeHttpMetadata(headers);
+    headers.set('etag', object.httpEtag);
+    headers.set('cache-control', 'public, max-age=31536000, immutable');
+
+    response = new Response(object.body, { headers });
+
+    // Cache for subsequent requests
+    ctx.waitUntil(cache.put(cacheKey, response.clone()));
+
+    return response;
+  }
+};
+```
+
+## Public Bucket with Custom Domain
+
+```typescript
+export default {
+  async fetch(request: Request, env: Env): Promise<Response> {
+    // CORS preflight
+    if (request.method === 'OPTIONS') {
+      return new Response(null, {
+        headers: {
+          'access-control-allow-origin': '*',
+          'access-control-allow-methods': 'GET, HEAD',
+          'access-control-max-age': '86400'
+        }
+      });
+    }
+
+    const key = new URL(request.url).pathname.slice(1);
+    if (!key) return Response.redirect('/index.html', 302);
+
+    const object = await env.MY_BUCKET.get(key);
+    if (!object) return new Response('Not found', { status: 404 });
+
+    const headers = new Headers();
+    object.writeHttpMetadata(headers);
+    headers.set('etag', object.httpEtag);
+    headers.set('access-control-allow-origin', '*');
+    headers.set('cache-control', 'public, max-age=31536000, immutable');
+
+    return new Response(object.body, { headers });
+  }
+};
+```
+
+## r2.dev Public URLs
+
+Enable r2.dev in dashboard for simple public access: `https://pub-${hashId}.r2.dev/${key}`  
+Or add custom domain via dashboard: `https://files.example.com/${key}`
+
+**Limitations:** No auth, bucket-level CORS, no cache override.

package/skills/cloudflare/references/r2-data-catalog/README.md

@@ -0,0 +1,75 @@
+# Cloudflare R2 Data Catalog
+
+Managed Apache Iceberg REST catalog built into R2 buckets. No catalog servers to run.
+
+## Documentation
+
+This reference is a fast-start with verified connection details and code. For limits, maintenance settings, engine config examples, and pricing, **retrieve the live docs** — use the Cloudflare MCP `docs` tool if available, otherwise `webfetch` the URL. Docs are source of truth over this file.
+
+| Topic | URL |
+|-------|-----|
+| Overview / get started | `https://developers.cloudflare.com/r2/data-catalog/get-started/` |
+| Manage catalogs (enable, tokens) | `https://developers.cloudflare.com/r2/data-catalog/manage-catalogs/` |
+| Engine config examples | `https://developers.cloudflare.com/r2/data-catalog/config-examples/` (`pyiceberg/`, `spark-python/`, `spark-scala/`, `duckdb/`, `snowflake/`, `trino/`, `starrocks/`) |
+| Table maintenance (compaction, snapshots) | `https://developers.cloudflare.com/r2/data-catalog/table-maintenance/` |
+| Deleting data | `https://developers.cloudflare.com/r2/data-catalog/deleting-data/` |
+| Metrics (GraphQL) | `https://developers.cloudflare.com/r2/data-catalog/observability/metrics/` |
+| Pricing | `https://developers.cloudflare.com/r2/data-catalog/platform/pricing/` |
+| Iceberg spec | `https://iceberg.apache.org/spec/` |
+
+## Connection Values
+
+Use the exact **Catalog URI** and **Warehouse** printed by `npx wrangler r2 bucket catalog enable <bucket>` (also shown in the dashboard). They follow these formats:
+
+| Value | Format | Example |
+|-------|--------|---------|
+| Catalog URI | `https://catalog.cloudflarestorage.com/{ACCOUNT_ID}/{BUCKET}` | `https://catalog.cloudflarestorage.com/4482a1.../live-data` |
+| Warehouse | `{ACCOUNT_ID}_{BUCKET}` (hyphens preserved) | `4482a1..._live-data` |
+| Token | R2 API token (Admin R&W on Storage + R&W on Data Catalog) | `cfut_...` |
+
+The Iceberg `/config` route needs `?warehouse={WAREHOUSE}`.
+
+## Architecture
+
+```
+Engines (PyIceberg, PySpark, Trino, Snowflake, DuckDB, R2 SQL)
+   │  Iceberg REST API (Bearer token)
+   ▼
+R2 Data Catalog  ── namespace/table metadata, snapshots, txn coordination
+   │  vended S3 credentials
+   ▼
+R2 Bucket  ── Parquet data files + Iceberg metadata
+```
+
+- **Warehouse** — top-level catalog grouping (`{ACCOUNT_ID}_{BUCKET}`)
+- **Namespace** — schema/database; nested namespaces supported
+- **Table** — Iceberg table (schema, partition spec, snapshots)
+- **Vended credentials** — temp S3 creds the catalog hands engines (`X-Iceberg-Access-Delegation: vended-credentials`)
+
+## When to Use
+
+**Use for:** log/analytics data lakes, BI pipelines, time-series/event data, multi-cloud or multi-engine analytics needing ACID + schema evolution on object storage.
+
+**Don't use for:** OLTP (use D1/a database), sub-second point lookups, tiny datasets (<1 GB), or unstructured blobs (store directly in R2).
+
+## Two APIs — Don't Confuse Them
+
+| API | Base | Use for |
+|-----|------|---------|
+| **Iceberg REST catalog** | `https://catalog.cloudflarestorage.com/{ACCOUNT_ID}/{BUCKET}` | Table reads/writes via PyIceberg, PySpark, Trino, etc. |
+| **Control-plane REST API** | `https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/r2-catalog/{BUCKET}` | Enable/disable, maintenance config, list namespaces/tables, get-table |
+
+**Status:** Open beta. Available to all R2 subscribers; verify pricing/billing status in docs.
+
+## Reading Order
+
+1. [configuration.md](configuration.md) — enable catalog, tokens, maintenance, client connection
+2. [api.md](api.md) — control-plane REST (incl. get-table), PyIceberg client, maintenance
+3. [patterns.md](patterns.md) — PyIceberg + PySpark templates, partitioning, external engines
+4. [gotchas.md](gotchas.md) — auth errors, maintenance behavior, troubleshooting
+
+## See Also
+
+- [pipelines](../pipelines/) — stream events into Iceberg tables
+- [r2-sql](../r2-sql/) — serverless SQL over these tables
+- [r2](../r2/) — underlying object storage

package/skills/cloudflare/references/r2-data-catalog/api.md

@@ -0,0 +1,122 @@
+# R2 Data Catalog API Reference
+
+Two APIs: the **control-plane REST API** (Cloudflare-specific) and the **Iceberg REST catalog API** (standard, used via PyIceberg/PySpark). For PyIceberg method details pull `https://py.iceberg.apache.org/`; for engine configs see `https://developers.cloudflare.com/r2/data-catalog/config-examples/`.
+
+## Control-Plane REST API
+
+Base: `https://api.cloudflare.com/client/v4/accounts/{ACCOUNT_ID}/r2-catalog/{BUCKET}`
+Auth: `Authorization: Bearer $API_TOKEN`
+
+| Operation | Method | Path |
+|-----------|--------|------|
+| Get catalog details | GET | `/r2-catalog/{bucket}` |
+| Enable / disable | POST | `/r2-catalog/{bucket}/enable` · `/disable` |
+| Store compaction credential | POST | `/r2-catalog/{bucket}/credential` |
+| List namespaces | GET | `/namespaces` |
+| List tables | GET | `/namespaces/{ns}/tables` |
+| **Get table metadata** | GET | `/namespaces/{ns}/tables/{table}` |
+| Get/update maintenance config | GET/POST | `/maintenance-configs` and `/namespaces/{ns}/tables/{table}/maintenance-configs` |
+
+List endpoints accept `?return_uuids=true`, `?return_details=true`, `?parent={ns}`, and pagination. **Nested namespaces use `%1F` (Unit Separator)**, not `/` or `.`: `/namespaces/parent%1Fchild/tables`.
+
+```bash
+# Catalog details (status, maintenance_config, credential_status)
+curl -s "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/r2-catalog/$BUCKET" \
+  -H "Authorization: Bearer $API_TOKEN"
+
+# Store token for compaction (pure-API setups)
+curl -s -X POST "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/r2-catalog/$BUCKET/credential" \
+  -H "Authorization: Bearer $API_TOKEN" -H "Content-Type: application/json" \
+  -d '{"token": "'$API_TOKEN'"}'
+
+# Update maintenance config (all fields optional; table-level overrides catalog-level)
+curl -s -X POST "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/r2-catalog/$BUCKET/maintenance-configs" \
+  -H "Authorization: Bearer $API_TOKEN" -H "Content-Type: application/json" \
+  -d '{"compaction": {"state": "enabled", "target_size_mb": "256"},
+       "snapshot_expiration": {"state": "enabled", "min_snapshots_to_keep": 10, "max_snapshot_age": "7d"}}'
+```
+
+### Get Table (metadata introspection)
+
+`GET /namespaces/{ns}/tables/{table}` returns schema, partition spec, sort order, and snapshot info — like Iceberg "load table" but on the control plane, with snapshots pruned to the most recent 10. (Newer than the published API docs.)
+
+```bash
+curl -s "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/r2-catalog/$BUCKET/namespaces/live/tables/earthquakes" \
+  -H "Authorization: Bearer $API_TOKEN"
+```
+
+```json
+{"result": {
+  "identifier": {"namespace": ["live"], "name": "earthquakes"},
+  "table_uuid": "019edccf-3ac8-73e3-...",
+  "metadata_location": "s3://live-data/__r2_data_catalog/.../metadata/01225-....metadata.json",
+  "total_snapshots": 1225,
+  "returned_snapshots": 10,
+  "metadata": { /* standard Iceberg TableMetadata: schemas, partition-specs, sort-orders,
+                   properties, current-snapshot-id, snapshots (≤10), snapshot-log, refs */ }
+}, "success": true}
+```
+
+| Field | Description |
+|-------|-------------|
+| `identifier` | `{namespace: [...], name}` |
+| `table_uuid` | Iceberg table UUID |
+| `metadata_location` | R2 path to current metadata file |
+| `total_snapshots` | Total before pruning |
+| `returned_snapshots` | Count in `metadata.snapshots` (max 10) |
+| `metadata` | Standard [Iceberg TableMetadata](https://iceberg.apache.org/spec/#table-metadata-fields), arrays pruned to 10 |
+
+### Error Format
+
+```json
+{"success": false, "errors": [{"code": 10000, "message": "Authentication error"}]}
+```
+
+Standard HTTP codes (401 auth, 403 perms, 404 not enabled/found, 409 conflict).
+
+## Iceberg REST Catalog API (via PyIceberg)
+
+Standard [Iceberg REST Catalog](https://github.com/apache/iceberg/blob/main/open-api/rest-catalog-open-api.yaml). Base: `https://catalog.cloudflarestorage.com/{ACCOUNT_ID}/{BUCKET}`. The `/config` route needs `?warehouse={WAREHOUSE}`.
+
+```python
+from pyiceberg.catalog.rest import RestCatalog
+catalog = RestCatalog(name="r2", warehouse=WAREHOUSE, uri=CATALOG_URI, token=TOKEN)
+```
+
+Common operations (see PyIceberg docs for full signatures):
+
+```python
+catalog.create_namespace_if_not_exists("logs")
+catalog.list_tables("logs")
+table = catalog.create_table(("logs", "events"), schema=schema)   # pyiceberg.schema.Schema
+table = catalog.load_table(("logs", "events"))
+table.append(pyarrow_table)          # also .overwrite(...)
+table.scan(row_filter="id > 100").to_pandas()
+```
+
+Schema evolution (add nullable columns; widen types only):
+```python
+with table.update_schema() as u:
+    u.add_column("user_id", LongType(), doc="User ID")
+    u.rename_column("msg", "message")
+```
+
+Time-travel:
+```python
+table.scan(snapshot_id=table.snapshots()[-2].snapshot_id)
+table.scan(as_of_timestamp=ms_epoch)
+```
+
+## Manual Maintenance (PySpark)
+
+Prefer automatic maintenance (control-plane API/wrangler). For manual control or very large tables, use Spark procedures (`rewrite_data_files`, `rewrite_manifests`, `expire_snapshots`, `remove_orphan_files`). See `https://developers.cloudflare.com/r2/data-catalog/table-maintenance/`.
+
+```python
+spark.sql("CALL r2dc.system.rewrite_data_files(table => 'ns.tbl')")
+# Orphan removal REQUIRES S3 credentials (vended creds fail with NoAuthWithAWSException)
+spark.sql("CALL r2dc.system.remove_orphan_files(table => 'ns.tbl', older_than => TIMESTAMP '2026-02-28 00:00:00')")
+```
+
+## See Also
+
+- [configuration.md](configuration.md) · [patterns.md](patterns.md) · [gotchas.md](gotchas.md)

package/skills/cloudflare/references/r2-data-catalog/configuration.md

@@ -0,0 +1,98 @@
+# R2 Data Catalog Configuration
+
+Enable the catalog, create tokens, turn on automatic maintenance, connect clients. For exhaustive token/permission options and maintenance settings, pull `https://developers.cloudflare.com/r2/data-catalog/manage-catalogs/` and `.../table-maintenance/`.
+
+## Step 1: Create Bucket + Enable Catalog
+
+```bash
+npx wrangler r2 bucket create my-bucket
+npx wrangler r2 bucket catalog enable my-bucket
+```
+
+`enable` outputs the two values used everywhere:
+
+```
+Warehouse:   4482a1cd43bf5197657ae1d8636c414a_my-bucket   # {ACCOUNT_ID}_{BUCKET}
+Catalog URI: https://catalog.cloudflarestorage.com/4482a1cd43bf5197657ae1d8636c414a/my-bucket
+```
+
+Enabling creates `__r2_data_catalog/` metadata in the bucket; existing objects are untouched.
+
+## Step 2: Create an API Token
+
+Dashboard → **R2** → **Manage R2 API tokens** → **Create API token**.
+
+**Simplest:** one token with **R2 Storage Admin Read & Write** + **R2 Data Catalog Read & Write**, scoped to your bucket(s). Add **R2 SQL Read** if you also query. This token works for the Iceberg REST API, control-plane API, R2 SQL, and GraphQL Analytics. Token creation also yields S3 Access Key ID / Secret (needed only for Spark orphan-file removal).
+
+> Open-beta limitation: R2 Storage **Admin Write is required even for read-only data access**. See the manage-catalogs doc for the current permission matrix.
+
+## Step 3: Enable Automatic Maintenance (Recommended)
+
+R2 Data Catalog runs compaction and snapshot expiration for you.
+
+```bash
+# Compaction — merges small files (target size MB; default 128)
+npx wrangler r2 bucket catalog compaction enable my-bucket \
+  --target-size 128 --token $API_TOKEN
+
+# Snapshot expiration — removes old snapshots AND their unreferenced data files
+npx wrangler r2 bucket catalog snapshot-expiration enable my-bucket \
+  --token $API_TOKEN --older-than-days 7 --retain-last 10
+```
+
+Compaction needs a **stored credential** to access files. `compaction enable` (and the dashboard wizard) stores it automatically; pure-API setups must call `/credential` (see [api.md](api.md)).
+
+> Compaction triggers **hourly** with **no hard throughput cap** (the former 2 GB/hour limit was lifted). Snapshot expiration deletes unreferenced data files automatically (since April 2026) — manual orphan cleanup is rarely needed. For target-size guidance per workload, see the table-maintenance doc.
+
+## Step 4: Verify
+
+```bash
+npx wrangler r2 bucket catalog status my-bucket
+# or control-plane API:
+curl -s "https://api.cloudflare.com/client/v4/accounts/$ACCOUNT_ID/r2-catalog/$BUCKET" \
+  -H "Authorization: Bearer $API_TOKEN"
+```
+
+Expect `"status": "active"`, `compaction.state: "enabled"`, `credential_status: "present"`.
+
+## Client Connection
+
+### PyIceberg
+
+```python
+import os
+from pyiceberg.catalog.rest import RestCatalog
+
+catalog = RestCatalog(
+    name="r2_catalog",
+    warehouse=os.environ["R2_WAREHOUSE"],   # {ACCOUNT_ID}_{BUCKET}
+    uri=os.environ["R2_CATALOG_URI"],       # https://catalog.cloudflarestorage.com/{ACCOUNT_ID}/{BUCKET}
+    token=os.environ["R2_TOKEN"],
+)
+print(catalog.list_namespaces())            # connection test
+```
+
+### PySpark / DuckDB / Trino / Snowflake
+
+Full, current engine configs live at `https://developers.cloudflare.com/r2/data-catalog/config-examples/`. A verified PySpark session template is in [patterns.md](patterns.md#pyspark-session) (needs Iceberg 1.6.1 and `X-Iceberg-Access-Delegation: vended-credentials`).
+
+## Environment Variables Pattern
+
+```bash
+# .env (never commit)
+R2_CATALOG_URI=https://catalog.cloudflarestorage.com/<ACCOUNT_ID>/<BUCKET>
+R2_WAREHOUSE=<ACCOUNT_ID>_<BUCKET>
+R2_TOKEN=<api-token>
+```
+
+## Disable Catalog
+
+```bash
+npx wrangler r2 bucket catalog disable my-bucket
+```
+
+Preserves data and metadata; tables become inaccessible via the catalog until re-enabled.
+
+## See Also
+
+- [api.md](api.md) — control-plane + PyIceberg API · [gotchas.md](gotchas.md) — auth & maintenance troubleshooting

package/skills/cloudflare/references/r2-data-catalog/gotchas.md

@@ -0,0 +1,55 @@
+# R2 Data Catalog Gotchas
+
+Common failure modes and operational behavior. For limits, recommendations, and supported settings, pull `https://developers.cloudflare.com/r2/data-catalog/` and `.../table-maintenance/`.
+
+## Connection / Auth
+
+- **Catalog URI / warehouse mismatch (most common).** Copy both values exactly from `wrangler r2 bucket catalog enable` (Catalog URI `https://catalog.cloudflarestorage.com/{ACCOUNT_ID}/{BUCKET}`, warehouse `{ACCOUNT_ID}_{BUCKET}`). Mismatched values fail to connect.
+- **401 Unauthorized** — token lacks Data Catalog R&W. Test with `catalog.list_namespaces()`.
+- **403 on data files** — token lacks R2 Storage. Open beta requires **Admin Read & Write on R2 Storage even for read-only** data access.
+- **`/config` "Warehouse name missing in query param"** — the Iceberg `/v1/config` route needs `?warehouse={ACCOUNT_ID}_{BUCKET}`. PyIceberg/PySpark add it automatically when you set `warehouse=`.
+
+## Maintenance Behavior (updated)
+
+- **No throughput cap on compaction.** The former 2 GB/hour/table limit is **lifted** — compaction triggers hourly and processes the backlog with no hard cap. Large small-file backlogs still take multiple hourly cycles.
+- **Snapshot expiration deletes data files** (since April 2026), not just metadata. Manual `remove_orphan_files` is rarely needed.
+- **Compaction requires a stored credential.** `wrangler ... compaction enable` and the dashboard wizard store it automatically; pure-API setups must POST `/credential`.
+- Compaction is **Parquet-only**.
+
+## Tables & Schema
+
+- `TableAlreadyExistsError` / `NamespaceAlreadyExistsError` → use `create_*_if_not_exists` / load existing.
+- `422 Validation` on schema update → only add nullable columns and widen types (int→long, float→double).
+- `TypeError: Cannot cast` on append → PyArrow type ≠ Iceberg schema; cast to int64 (Iceberg default); check `table.schema()`.
+
+## Concurrency
+
+- `CommitFailedException` → optimistic-locking conflict; retry with backoff (see [patterns.md](patterns.md#concurrent-writes-with-retry-pyiceberg)).
+- Stale metadata after external writes → reload: `table = catalog.load_table(("ns","tbl"))`.
+
+## PySpark / Iceberg
+
+| Issue | Fix |
+|-------|-----|
+| Catalog auth fails | Add header `X-Iceberg-Access-Delegation: vended-credentials` |
+| `NoAuthWithAWSException` on orphan removal | Supply S3 access/secret keys (vended creds don't work here) |
+| Version mismatch | Use Iceberg `1.6.1` |
+| Slow first run (~30–60s) | JAR download; cached after |
+| Remote signing errors | Set `s3.remote-signing-enabled=false` |
+
+## Nested Namespaces
+
+Control-plane URL separator for nested namespaces is **`%1F`** (Unit Separator), not `/` or `.`: `/namespaces/parent%1Fchild/tables`.
+
+## Debug Checklist
+
+1. `npx wrangler r2 bucket catalog status <bucket>` — enabled?
+2. Token has R2 Storage (Admin R&W) + R2 Data Catalog (R&W)?
+3. `catalog.list_namespaces()` succeeds?
+4. Catalog URI = `catalog.cloudflarestorage.com/{ACCOUNT_ID}/{BUCKET}`, warehouse = `{ACCOUNT_ID}_{BUCKET}`?
+5. Namespace created before `create_table`?
+6. Compaction enabled + `credential_status: present`?
+
+## See Also
+
+- [configuration.md](configuration.md) · [api.md](api.md) · [patterns.md](patterns.md)

package/skills/cloudflare/references/r2-data-catalog/patterns.md

@@ -0,0 +1,122 @@
+# R2 Data Catalog Patterns
+
+Code templates with PyIceberg (lightweight, no JVM) and PySpark (full Iceberg ecosystem). For per-engine config (DuckDB, Trino, Snowflake, StarRocks) and partitioning/maintenance best practices, pull `https://developers.cloudflare.com/r2/data-catalog/config-examples/` and `.../table-maintenance/`.
+
+| Need | Tool |
+|------|------|
+| Catalog ops, append/scan, small-medium loads | PyIceberg |
+| Batch ETL, INSERT INTO SELECT, DELETE/MERGE, write-back, >1 TB maintenance | PySpark |
+| Pure SQL analytics (no writes) | [R2 SQL](../r2-sql/) |
+
+## PyIceberg: Connect, Create, Load
+
+```python
+import os, pyarrow as pa
+from pyiceberg.catalog.rest import RestCatalog
+
+catalog = RestCatalog(
+    name="r2",
+    warehouse=os.environ["R2_WAREHOUSE"],   # {ACCOUNT_ID}_{BUCKET}
+    uri=os.environ["R2_CATALOG_URI"],       # https://catalog.cloudflarestorage.com/{ACCOUNT_ID}/{BUCKET}
+    token=os.environ["R2_TOKEN"],
+)
+catalog.create_namespace_if_not_exists("analytics")
+
+schema = pa.schema([("id", pa.int64()), ("name", pa.string()), ("amount", pa.float64())])
+table = catalog.create_table(("analytics", "events"), schema=schema)
+table.append(pa.table({"id": [1, 2], "name": ["a", "b"], "amount": [80.0, 92.5]}))
+print(table.scan().to_arrow().to_pandas())
+```
+
+## PyIceberg: Partitioned Time-Series Table
+
+```python
+from pyiceberg.schema import Schema
+from pyiceberg.types import NestedField, TimestampType, StringType
+from pyiceberg.partitioning import PartitionSpec, PartitionField
+from pyiceberg.transforms import DayTransform
+
+schema = Schema(
+    NestedField(1, "timestamp", TimestampType(), required=True),
+    NestedField(2, "level", StringType(), required=True),
+    NestedField(3, "message", StringType(), required=False),
+)
+spec = PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name="day"))
+table = catalog.create_table(("logs", "app_logs"), schema=schema, partition_spec=spec)
+errors = table.scan(row_filter="level = 'ERROR'").to_pandas()   # partition pruning
+```
+
+## PySpark Session
+
+Verified template — requires Iceberg **1.6.1** and vended credentials. S3 keys are only needed for orphan-file removal. (If this drifts, cross-check `config-examples/spark-python/`.)
+
+```python
+from pyspark.sql import SparkSession
+
+spark = SparkSession.builder \
+    .appName("R2DataCatalog") \
+    .config('spark.jars.packages',
+        'org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.6.1,'
+        'org.apache.iceberg:iceberg-aws-bundle:1.6.1,'
+        'org.apache.hadoop:hadoop-aws:3.3.4,'
+        'com.amazonaws:aws-java-sdk-bundle:1.12.262') \
+    .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") \
+    .config("spark.sql.catalog.r2dc", "org.apache.iceberg.spark.SparkCatalog") \
+    .config("spark.sql.catalog.r2dc.type", "rest") \
+    .config("spark.sql.catalog.r2dc.uri", CATALOG_URI) \
+    .config("spark.sql.catalog.r2dc.warehouse", WAREHOUSE) \
+    .config("spark.sql.catalog.r2dc.token", TOKEN) \
+    .config("spark.sql.catalog.r2dc.header.X-Iceberg-Access-Delegation", "vended-credentials") \
+    .config("spark.sql.catalog.r2dc.s3.remote-signing-enabled", "false") \
+    .config("spark.sql.defaultCatalog", "r2dc") \
+    .config("spark.hadoop.fs.s3a.access.key", S3_ACCESS_KEY) \
+    .config("spark.hadoop.fs.s3a.secret.key", S3_SECRET_KEY) \
+    .config("spark.hadoop.fs.s3a.endpoint", S3_ENDPOINT) \
+    .config("spark.hadoop.fs.s3a.path.style.access", "true") \
+    .getOrCreate()
+spark.sql("USE r2dc")
+```
+
+> `X-Iceberg-Access-Delegation: vended-credentials` is required; `s3.remote-signing-enabled` must be `false`. First startup ~30–60s for JAR downloads (cached after).
+
+## PySpark: Batch ETL
+
+```python
+spark.sql("""
+CREATE TABLE IF NOT EXISTS my_ns.events (
+    __ingest_ts TIMESTAMP, event_id STRING, category STRING, amount DOUBLE
+) PARTITIONED BY (days(__ingest_ts))
+""")
+
+spark.read.option("header","true").csv("data.csv").writeTo("my_ns.events").append()
+spark.read.parquet("data.parquet").writeTo("my_ns.events").append()
+spark.sql("INSERT INTO my_ns.target SELECT col1, col2 FROM my_ns.source WHERE col1 > 0")
+spark.sql("DELETE FROM my_ns.events WHERE amount < 0")
+```
+
+> Partition large tables (`PARTITIONED BY (days(__ingest_ts))`). Unpartitioned works for small datasets (<1000 files) but degrades at scale.
+
+## Concurrent Writes with Retry (PyIceberg)
+
+```python
+from pyiceberg.exceptions import CommitFailedException
+import time
+
+def append_with_retry(table, data, max_retries=3):
+    for attempt in range(max_retries):
+        try:
+            table.append(data); return
+        except CommitFailedException:
+            if attempt == max_retries - 1: raise
+            time.sleep(2 ** attempt)
+```
+
+Optimistic locking: concurrent commits to the same table may conflict; different-partition writes are safe.
+
+## Connecting Any Iceberg Engine
+
+Engines connect with the Iceberg REST catalog config — Catalog URI `https://catalog.cloudflarestorage.com/{ACCOUNT_ID}/{BUCKET}`, warehouse `{ACCOUNT_ID}_{BUCKET}`, your token, and header `X-Iceberg-Access-Delegation: vended-credentials`. Copy-paste configs per engine: `config-examples/`.
+
+## See Also
+
+- [api.md](api.md) · [gotchas.md](gotchas.md) · [pipelines/patterns.md](../pipelines/patterns.md) · [r2-sql/patterns.md](../r2-sql/patterns.md)