schift-cli 0.1.0__py3-none-any.whl

schift_cli/data/schift-best-practices/references/cost-storage-tiers.md

@@ -0,0 +1,183 @@
+---
+title: Understand Storage Tier Lifecycle to Optimize Costs
+impact: MEDIUM
+impactDescription: Using the wrong storage tier causes either data loss (Free tier deletes S3 data after 30 days) or unnecessary cost (storing cold-access data in FAISS hot storage). Matching tier to access pattern is essential for production reliability.
+tags:
+  - cost
+  - storage
+  - tiers
+  - lifecycle
+  - faiss
+  - s3
+---
+
+## Understand Storage Tier Lifecycle to Optimize Costs
+
+Schift stores your vector data in two layers: FAISS (hot, in-memory vector index for fast search) and S3 (durable object storage for the original chunks and metadata). Each pricing tier has different retention policies for both layers. Mismatching your use case to the wrong tier causes data loss or unnecessary cost.
+
+The critical mistake is treating the Free tier as suitable for production data — S3 data is deleted after 30 days, and once deleted, your collection cannot be rebuilt without re-uploading and re-embedding all source documents.
+
+### Storage Tier Reference
+
+| Tier | FAISS Hot Retention | S3 Retention | Use Case |
+|------|--------------------|--------------|-|
+| Free | 7 days | Deleted after 30 days | Development, demos, prototypes |
+| Pro | 30 days | → IA after 30d → Glacier after 90d | Production applications |
+| Enterprise | 90 days | → IA after 90d | High-frequency search workloads |
+
+**Key terms:**
+- **FAISS hot**: Vector index loaded in memory — sub-millisecond search latency
+- **FAISS cold**: Index evicted from memory, reloaded on next query (adds ~1–3s warmup)
+- **S3 Standard**: Instant retrieval, billed at standard rate
+- **S3 IA** (Infrequent Access): Lower storage cost, slightly higher retrieval cost
+- **S3 Glacier**: Cheapest storage, minutes-to-hours retrieval time (not suitable for live search)
+
+### Incorrect
+
+Using Free tier for production data — collection becomes unreachable after 30 days:
+
+```python
+# Python — Free tier for production: S3 data deleted at day 30
+from schift import Schift
+
+# Free tier API key used in production application
+client = Schift(api_key="sch_free_...")
+
+# Upload 50,000 documents — 3 months of engineering work
+result = client.bucket.upload_bulk(bucket_id, documents)
+print(f"Indexed {result.chunk_count} chunks")
+
+# Day 7:  FAISS evicted — first query adds 2s cold-start latency (no warning)
+# Day 30: S3 data deleted — collection.search() returns empty results with no error
+# Day 31: client.search(collection_id, query) → [] (silently empty, data gone)
+
+# There is no recovery path. Source documents must be re-uploaded.
+```
+
+```typescript
+// TypeScript — Free tier with no monitoring: invisible data expiry
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_free_...' });  // Free tier
+
+// Build a production chatbot — works fine for 29 days
+const answer = await client.workflow.run(workflowId, { query: userQuestion });
+
+// Day 30+: search returns no results, LLM generates hallucinated answers
+// because context is empty. No exception thrown — just silent data loss.
+```
+
+### Correct
+
+Use Pro for production data; pin frequently-accessed collections to hot storage:
+
+```python
+# Python — Pro tier with explicit tier awareness
+from schift import Schift
+
+client = Schift(api_key="sch_pro_...")  # Pro tier
+
+# Create collection with tier-appropriate settings
+collection = client.collection.create(
+    name="docs-prod",
+    tier="pro",           # 30d FAISS hot, S3 → IA → Glacier lifecycle
+    region="ap-northeast-2",
+)
+
+# Pin high-traffic collections to hot storage indefinitely
+# (overrides the 30-day FAISS eviction, billed at hot storage rate)
+client.collection.pin(collection.id, hot=True)
+
+# Upload production data — durable beyond 90 days
+result = client.bucket.upload_bulk(
+    bucket_id,
+    documents,
+    metadata={"env": "production", "indexed_at": "2026-03-17"}
+)
+
+# Monitor cold-start risk: check FAISS warmth before high-stakes queries
+status = client.collection.status(collection.id)
+if not status.faiss_hot:
+    client.collection.warm(collection.id)   # pre-warm before query
+    # warm() is async — poll status or add a short wait for latency-sensitive paths
+```
+
+```typescript
+// TypeScript — Pro tier with lifecycle-aware patterns
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_pro_...' });  // Pro tier
+
+// Setup: create production collection with hot pin for frequently searched data
+async function setupProductionCollection() {
+  const collection = await client.collection.create({
+    name: 'docs-prod',
+    tier: 'pro',
+    region: 'ap-northeast-2',
+  });
+
+  // Pin to hot storage if this collection is queried frequently (>100 queries/day)
+  await client.collection.pin(collection.id, { hot: true });
+
+  return collection.id;
+}
+
+// Runtime: check warmth before user-facing queries to avoid cold-start surprises
+async function searchWithWarmGuard(collectionId: string, query: string) {
+  const status = await client.collection.status(collectionId);
+
+  if (!status.faissHot) {
+    // Trigger warm-up for low-latency access on next query
+    // For latency-sensitive paths, await the warm before proceeding
+    await client.collection.warm(collectionId);
+  }
+
+  return client.search(collectionId, query, { topK: 5 });
+}
+```
+
+**Tier selection guide:**
+
+```python
+# Python — choose tier based on access pattern
+def choose_tier(access_frequency: str, data_criticality: str) -> str:
+    """
+    access_frequency: "daily" | "weekly" | "monthly" | "one-time"
+    data_criticality: "production" | "staging" | "dev"
+    """
+    if data_criticality == "dev":
+        return "free"    # OK to lose after 30 days
+
+    if access_frequency == "daily":
+        return "pro"     # FAISS stays hot 30d, pin for longer
+
+    if access_frequency in ("weekly", "monthly") and data_criticality == "production":
+        return "pro"     # S3 lifecycle keeps data, FAISS cold-starts acceptable
+
+    if access_frequency == "one-time":
+        return "free"    # Temporary indexing, no need for retention
+
+    return "pro"         # Default to Pro for any production use
+```
+
+**Cost impact of correct tier usage (10GB collection):**
+
+| Tier | Storage Cost/Month | Risk |
+|------|--------------------|------|
+| Free | $0 | Data deleted at day 30 |
+| Pro | $1.50 (10GB × $0.15) | Safe for production |
+| Enterprise | Custom | High-frequency, SLA-backed |
+
+**Production checklist:**
+
+- Never use a Free tier key in any environment where data loss is unacceptable
+- Pin collections to hot storage if queried more than 50 times per day
+- Monitor `collection.status().faissHot` before latency-sensitive query paths
+- Set up alerts for S3 lifecycle transitions if you use collections infrequently
+- Store collection IDs and creation dates — helps audit what tier data lives on
+
+## Reference
+
+- https://docs.schift.io/storage/tiers
+- https://docs.schift.io/storage/lifecycle
+- https://docs.schift.io/pricing

schift_cli/data/schift-best-practices/references/deploy-cloudrun.md

@@ -0,0 +1,140 @@
+---
+title: Deploy Schift-Powered Apps on Cloud Run for Zero Idle Cost
+impact: LOW-MEDIUM
+impactDescription: Zero idle cost with proper Cloud Run configuration
+tags:
+  - deployment
+  - cloud-run
+  - docker
+  - infrastructure
+  - cost
+  - scaling
+---
+
+## Deploy Schift-Powered Apps on Cloud Run for Zero Idle Cost
+
+Cloud Run scales to zero when no traffic is coming in, so you pay nothing while your app is idle. This is ideal for Schift-backed services that handle sporadic traffic — chatbots, search APIs, document Q&A endpoints. The common mistake is deploying these on an always-on VM, which bills 24/7 regardless of actual usage.
+
+### Incorrect
+
+Running a Schift-powered chatbot on an always-on VM that pays for compute even during zero-traffic hours:
+
+```bash
+# Always-on EC2 / Compute Engine VM approach
+# Costs ~$15–30/month even when nobody is using the app
+# Manual scaling, manual restarts, no request-based autoscaling
+
+ssh ec2-user@your-server
+pm2 start server.js  # just stays running forever
+```
+
+```typescript
+// server.ts — no health check, no graceful shutdown
+// Deployed on an always-on VM: billed 24/7
+import { Hono } from 'hono';
+import { Schift } from '@schift-io/sdk';
+
+const app = new Hono();
+const client = new Schift({ apiKey: process.env.SCHIFT_API_KEY! });
+
+app.post('/chat', async (c) => {
+  const { query } = await c.req.json();
+  const results = await client.search('col_...', query, { topK: 3 });
+  return c.json({ results });
+});
+
+export default app;
+// No /health endpoint → load balancer cannot confirm readiness
+// No memory limit set → OOM-killed silently
+```
+
+This approach pays for a full VM instance even when traffic is zero, lacks health checks for Cloud Run readiness probes, and requires manual scaling during traffic spikes.
+
+### Correct
+
+Deploy on Cloud Run with `min-instances=0`, set memory for embedding operations, and expose a `/health` endpoint:
+
+```dockerfile
+# Dockerfile — multi-stage build for a minimal runtime image
+FROM node:20-alpine AS builder
+WORKDIR /app
+COPY package*.json ./
+RUN npm ci --only=production
+COPY . .
+RUN npm run build
+
+FROM node:20-alpine AS runtime
+WORKDIR /app
+# Copy only compiled output and production deps
+COPY --from=builder /app/dist ./dist
+COPY --from=builder /app/node_modules ./node_modules
+COPY package.json ./
+
+# Cloud Run injects PORT; default to 8080
+ENV PORT=8080
+EXPOSE 8080
+
+CMD ["node", "dist/server.js"]
+```
+
+```typescript
+// server.ts — production-ready Cloud Run handler
+import { Hono } from 'hono';
+import { serve } from '@hono/node-server';
+import { Schift } from '@schift-io/sdk';
+
+const app = new Hono();
+
+// SCHIFT_API_KEY is injected from Cloud Run secret at runtime
+const client = new Schift({ apiKey: process.env.SCHIFT_API_KEY! });
+
+// Required: Cloud Run sends readiness/liveness probes to /health
+app.get('/health', (c) => c.json({ status: 'ok' }));
+
+app.post('/search', async (c) => {
+  const { query, collectionId } = await c.req.json();
+  const results = await client.search(collectionId, query, {
+    topK: 5,
+    scoreThreshold: 0.72,
+  });
+  return c.json({ results });
+});
+
+const port = Number(process.env.PORT) || 8080;
+serve({ fetch: app.fetch, port });
+```
+
+```bash
+# Build and push the container image
+gcloud builds submit --tag gcr.io/YOUR_PROJECT/schift-app
+
+# Deploy to Cloud Run
+gcloud run deploy schift-app \
+  --image gcr.io/YOUR_PROJECT/schift-app \
+  --platform managed \
+  --region us-central1 \
+  --allow-unauthenticated \
+  --min-instances 0 \         # scale to zero → $0 idle cost
+  --max-instances 10 \
+  --concurrency 80 \          # requests handled per instance before scaling out
+  --timeout 300 \             # 5 min max for long embedding/search ops
+  --memory 512Mi \            # minimum for embedding operations
+  --set-secrets SCHIFT_API_KEY=schift-api-key:latest  # inject from Secret Manager
+```
+
+**Key settings explained:**
+
+| Setting | Value | Why |
+|---------|-------|-----|
+| `min-instances` | `0` | Scales to zero when idle — $0 cost |
+| `concurrency` | `80` | Cloud Run default; safe for I/O-bound search APIs |
+| `timeout` | `300s` | Allows time for cold starts + embedding latency |
+| `memory` | `512Mi` | Minimum for Schift SDK + embedding response buffers |
+| `SCHIFT_API_KEY` | Secret Manager | Never hardcode API keys in the image |
+
+**Cold start note:** With `min-instances=0`, the first request after idle incurs a cold start (~1–3 seconds). If your use case requires consistent low latency, set `min-instances=1` — this costs ~$5–10/month but eliminates cold starts.
+
+## Reference
+
+- https://docs.schift.io/deployment/cloud-run
+- https://cloud.google.com/run/docs/configuring/min-instances

schift_cli/data/schift-best-practices/references/embed-batch-processing.md

@@ -0,0 +1,86 @@
+---
+title: Use embed_batch() for multiple texts
+impact: CRITICAL
+impactDescription: Looping embed() one-by-one produces 10-50x more API calls than a single embed_batch() call, increasing latency and cost proportionally.
+tags: [embedding, performance, cost, batch]
+---
+
+## Use embed_batch() for multiple texts
+
+When embedding more than one piece of text, always use `embed_batch()` instead of calling `embed()` in a loop. Each `embed()` call is a separate HTTP round-trip. `embed_batch()` sends all texts in a single request (up to 100 per call) and returns embeddings in the same order.
+
+**Incorrect** — calling `embed()` in a loop creates one HTTP request per text:
+
+```python
+# Python - DON'T do this
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+texts = ["cat", "dog", "bird", "fish"]
+
+embeddings = []
+for text in texts:
+    result = client.embed(text)       # 4 separate API calls
+    embeddings.append(result.embedding)
+```
+
+```typescript
+// TypeScript - DON'T do this
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+const texts = ['cat', 'dog', 'bird', 'fish'];
+
+const embeddings = [];
+for (const text of texts) {
+  const result = await client.embed(text);   // 4 separate API calls
+  embeddings.push(result.embedding);
+}
+```
+
+**Correct** — one `embed_batch()` call handles the entire list:
+
+```python
+# Python - DO this
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+texts = ["cat", "dog", "bird", "fish"]
+
+results = client.embed_batch(texts)           # 1 API call
+embeddings = [r.embedding for r in results]  # same order as input
+
+# For large datasets, chunk into batches of up to 100
+BATCH_SIZE = 100
+all_embeddings = []
+for i in range(0, len(texts), BATCH_SIZE):
+    batch = texts[i : i + BATCH_SIZE]
+    results = client.embed_batch(batch)
+    all_embeddings.extend(r.embedding for r in results)
+```
+
+```typescript
+// TypeScript - DO this
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+const texts = ['cat', 'dog', 'bird', 'fish'];
+
+const results = await client.embedBatch(texts);         // 1 API call
+const embeddings = results.map((r) => r.embedding);     // same order as input
+
+// For large datasets, chunk into batches of up to 100
+const BATCH_SIZE = 100;
+const allEmbeddings: number[][] = [];
+for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+  const batch = texts.slice(i, i + BATCH_SIZE);
+  const batchResults = await client.embedBatch(batch);
+  allEmbeddings.push(...batchResults.map((r) => r.embedding));
+}
+```
+
+The `embed_batch()` / `embedBatch()` limit is 100 texts per call. For larger datasets, split into chunks of 100 as shown above. Results are always returned in the same order as the input list.
+
+## Reference
+
+- https://docs.schift.io/api/embed#batch

schift_cli/data/schift-best-practices/references/embed-error-handling.md

@@ -0,0 +1,155 @@
+---
+title: Handle AuthError, QuotaError, and rate limits properly
+impact: HIGH
+impactDescription: Unhandled quota or rate-limit errors crash production jobs and waste partially processed batches. Proper error handling with exponential backoff recovers automatically and avoids re-processing already-embedded documents.
+tags: [embedding, error-handling, rate-limits, quota, retry]
+---
+
+## Handle AuthError, QuotaError, and rate limits properly
+
+Schift raises three primary error types you must handle:
+
+| Error | HTTP status | Cause |
+|---|---|---|
+| `AuthError` | 401 | Invalid or missing API key |
+| `QuotaError` | 402 | Monthly credit limit reached |
+| `RateLimitError` | 429 | Too many requests per second |
+
+`AuthError` is a hard failure — check your key. `QuotaError` means you need to top up credits or wait for the next billing cycle. `RateLimitError` is transient and should be retried with exponential backoff.
+
+**Incorrect** — no error handling; a single quota error crashes the entire job:
+
+```python
+# Python - DON'T do this
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+texts = load_large_dataset()   # 10,000 documents
+
+# If quota runs out halfway through, the whole job crashes
+# and there is no way to resume from where it stopped
+results = client.embed_batch(texts)
+```
+
+```typescript
+// TypeScript - DON'T do this
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+const texts = await loadLargeDataset();
+
+// Unhandled promise rejection on quota exceeded
+const results = await client.embedBatch(texts);
+```
+
+**Correct** — catch specific errors and apply exponential backoff for rate limits:
+
+```python
+# Python - DO this
+import time
+from schift import Schift, AuthError, QuotaError, SchiftError
+
+client = Schift(api_key="sch_...")
+
+def embed_with_retry(texts: list[str], max_retries: int = 5) -> list:
+    """Embed texts with exponential backoff on rate limit errors."""
+    for attempt in range(max_retries):
+        try:
+            return client.embed_batch(texts)
+
+        except AuthError:
+            # Hard failure — wrong API key, do not retry
+            raise RuntimeError(
+                "Invalid Schift API key. Check your SCHIFT_API_KEY env var."
+            )
+
+        except QuotaError as e:
+            # Credits exhausted — log and stop; retrying won't help
+            raise RuntimeError(
+                f"Schift quota exceeded. Top up credits at https://app.schift.io/billing. "
+                f"Details: {e}"
+            )
+
+        except SchiftError as e:
+            if e.status_code == 429:
+                # Rate limit — wait and retry with exponential backoff
+                wait = 2 ** attempt          # 1s, 2s, 4s, 8s, 16s
+                print(f"Rate limited. Retrying in {wait}s (attempt {attempt + 1}/{max_retries})")
+                time.sleep(wait)
+            else:
+                raise   # Unexpected error — surface it
+
+    raise RuntimeError(f"embed_with_retry failed after {max_retries} attempts")
+
+
+# Check remaining credits before starting a large batch job
+def safe_batch_embed(texts: list[str]) -> list:
+    account = client.account.get()
+    if account.credits_remaining < len(texts):
+        raise RuntimeError(
+            f"Insufficient credits: need {len(texts)}, have {account.credits_remaining}"
+        )
+    return embed_with_retry(texts)
+```
+
+```typescript
+// TypeScript - DO this
+import { Schift, AuthError, QuotaError, SchiftError } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+
+async function embedWithRetry(
+  texts: string[],
+  maxRetries = 5,
+): Promise<{ embedding: number[] }[]> {
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      return await client.embedBatch(texts);
+
+    } catch (err) {
+      if (err instanceof AuthError) {
+        // Hard failure — wrong API key, do not retry
+        throw new Error(
+          'Invalid Schift API key. Check your SCHIFT_API_KEY env var.',
+        );
+      }
+
+      if (err instanceof QuotaError) {
+        // Credits exhausted — log and stop
+        throw new Error(
+          `Schift quota exceeded. Top up credits at https://app.schift.io/billing. ${err.message}`,
+        );
+      }
+
+      if (err instanceof SchiftError && err.statusCode === 429) {
+        // Rate limit — exponential backoff
+        const wait = 2 ** attempt * 1000;   // 1s, 2s, 4s, 8s, 16s
+        console.warn(`Rate limited. Retrying in ${wait / 1000}s (attempt ${attempt + 1}/${maxRetries})`);
+        await new Promise((resolve) => setTimeout(resolve, wait));
+        continue;
+      }
+
+      throw err;   // Unexpected error — surface it
+    }
+  }
+  throw new Error(`embedWithRetry failed after ${maxRetries} attempts`);
+}
+
+// Check credits before a large batch job
+async function safeBatchEmbed(texts: string[]) {
+  const account = await client.account.get();
+  if (account.creditsRemaining < texts.length) {
+    throw new Error(
+      `Insufficient credits: need ${texts.length}, have ${account.creditsRemaining}`,
+    );
+  }
+  return embedWithRetry(texts);
+}
+```
+
+For long-running indexing jobs, track which documents have already been embedded (e.g. store their IDs in a database) so you can resume from where you left off after a `QuotaError` rather than restarting from scratch.
+
+## Reference
+
+- https://docs.schift.io/api/errors
+- https://docs.schift.io/guides/rate-limits

schift_cli/data/schift-best-practices/references/embed-multimodal.md

@@ -0,0 +1,100 @@
+---
+title: Use modality parameter for non-text inputs
+impact: CRITICAL
+impactDescription: Passing raw image bytes or file content as a plain text string produces garbage embeddings. The modality parameter routes the input through the correct encoder before projection into canonical space.
+tags: [embedding, multimodal, image, document, audio]
+---
+
+## Use modality parameter for non-text inputs
+
+Schift supports five modalities: `text`, `image`, `audio`, `video`, and `document`. Every modality is projected into the same canonical 1024-dimensional space, so cross-modal search (e.g. text query against image collection) works out of the box — but only if you specify the correct `modality` when embedding.
+
+Passing raw binary content without setting `modality` causes Schift to treat the input as UTF-8 text, producing meaningless embeddings.
+
+**Incorrect** — passing image bytes as plain text produces garbage vectors:
+
+```python
+# Python - DON'T do this
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+
+with open("photo.jpg", "rb") as f:
+    image_bytes = f.read()
+
+# Wrong: embed() with no modality treats input as text
+result = client.embed(image_bytes)   # garbage embedding
+```
+
+```typescript
+// TypeScript - DON'T do this
+import { Schift } from '@schift-io/sdk';
+import { readFileSync } from 'fs';
+
+const client = new Schift({ apiKey: 'sch_...' });
+
+const imageBuffer = readFileSync('photo.jpg');
+
+// Wrong: no modality — treated as text
+const result = await client.embed(imageBuffer.toString());  // garbage embedding
+```
+
+**Correct** — specify `modality="image"` (or the appropriate modality) and pass the file path or bytes:
+
+```python
+# Python - DO this
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+
+# Embed a single image
+result = client.embed("photo.jpg", modality="image")
+image_vector = result.embedding   # 1024d, same space as text embeddings
+
+# Embed a PDF document
+doc_result = client.embed("report.pdf", modality="document")
+
+# Cross-modal search: text query against image collection
+text_query = client.embed("a golden retriever running in a park", modality="text")
+# text_query.embedding can now be used to search an image collection directly
+
+# Batch embed multiple images
+image_paths = ["img1.jpg", "img2.jpg", "img3.jpg"]
+results = client.embed_batch(image_paths, modality="image")
+vectors = [r.embedding for r in results]
+```
+
+```typescript
+// TypeScript - DO this
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+
+// Embed a single image by file path
+const result = await client.embed('photo.jpg', { modality: 'image' });
+const imageVector = result.embedding;   // 1024d, same space as text
+
+// Embed a PDF document
+const docResult = await client.embed('report.pdf', { modality: 'document' });
+
+// Cross-modal: text query against image collection
+const textQuery = await client.embed(
+  'a golden retriever running in a park',
+  { modality: 'text' },
+);
+// textQuery.embedding searches image collections directly
+
+// Batch embed multiple images
+const imagePaths = ['img1.jpg', 'img2.jpg', 'img3.jpg'];
+const results = await client.embedBatch(imagePaths, { modality: 'image' });
+const vectors = results.map((r) => r.embedding);
+```
+
+Because all modalities share the same canonical 1024d space, you can search across modalities without any additional conversion step. A text embedding and an image embedding are directly comparable with cosine similarity.
+
+Supported modality values: `text` (default), `image`, `audio`, `video`, `document`.
+
+## Reference
+
+- https://docs.schift.io/api/embed#modality
+- https://docs.schift.io/concepts/canonical-space