schift-cli 0.1.0__py3-none-any.whl

schift_cli/data/schift-best-practices/references/search-collection-design.md

@@ -0,0 +1,229 @@
+---
+title: Design Collections by Domain/Use-Case, Not by Data Source
+impact: HIGH
+impactDescription: A single monolithic collection forces every query to search irrelevant data, bloating latency and degrading result quality. Per-file collections create management overhead and prevent cross-document ranking within a domain. Domain-aligned collections with consistent metadata schemas give you both search quality and operational simplicity.
+tags:
+  - collection
+  - schema
+  - design
+  - organization
+  - metadata
+---
+
+## Design Collections by Domain/Use-Case, Not by Data Source
+
+Collection design is a structural decision that affects search quality, filter performance, and long-term maintainability. The two common anti-patterns are opposites of each other: one giant collection that holds everything (poor signal-to-noise ratio per query), and one collection per uploaded file (combinatorial overhead, no cross-document ranking).
+
+The correct approach is to group documents by the queries they serve. A user asking "how do I reset my API key?" should search `support-docs` — not wade through API reference and release notes in the same index.
+
+### Incorrect
+
+**Anti-pattern A: One monolithic collection**
+
+```python
+# Python — everything in one collection
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+
+# Dumping all company knowledge into a single collection
+client.collection.add("everything", documents=[
+    {"id": "pd-001", "text": "Product feature: drag-and-drop upload..."},
+    {"id": "st-001", "text": "Support ticket: user cannot login..."},
+    {"id": "api-001", "text": "POST /v1/embed — Creates embeddings..."},
+    {"id": "rn-001", "text": "Release notes v2.3: added batch support..."},
+])
+
+# Query now competes against all domains — noisy results
+results = client.search("everything", "how to embed documents")
+# Returns: API reference, support tickets, release notes, all mixed together
+```
+
+**Anti-pattern B: One collection per file**
+
+```python
+# Python — a new collection for every uploaded file
+for filename, content in uploaded_files:
+    collection_id = filename.replace(".pdf", "").replace(" ", "-")
+    client.collection.create(collection_id, schema={})  # no schema
+    client.collection.add(collection_id, documents=[{"id": "doc", "text": content}])
+
+# Now you must search 200 collections separately — impossible to rank across files
+```
+
+```typescript
+// TypeScript — same anti-patterns
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+
+// Anti-pattern A: monolithic
+await client.collection.add('everything', { documents: allMyDocuments });
+
+// Anti-pattern B: per-file
+for (const file of uploadedFiles) {
+  const id = file.name.replace('.pdf', '');
+  await client.collection.create(id, { schema: {} });  // no schema
+  await client.collection.add(id, { documents: [{ id: 'doc', text: file.content }] });
+}
+```
+
+### Correct
+
+Create collections per domain/use-case with a defined metadata schema:
+
+```python
+# Python — domain-aligned collections with schemas
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+
+# Define schema for each domain at creation — only these fields are filter-indexed
+client.collection.create("product-docs", schema={
+    "fields": {
+        "version":    {"type": "string",  "filterable": True},
+        "category":   {"type": "string",  "filterable": True},  # "feature", "guide", "faq"
+        "updated_at": {"type": "date",    "filterable": True},
+        "language":   {"type": "string",  "filterable": True},
+    }
+})
+
+client.collection.create("support-tickets", schema={
+    "fields": {
+        "status":       {"type": "string",  "filterable": True},  # "open", "closed"
+        "priority":     {"type": "string",  "filterable": True},
+        "created_at":   {"type": "date",    "filterable": True},
+        "product_area": {"type": "string",  "filterable": True},
+    }
+})
+
+client.collection.create("api-reference", schema={
+    "fields": {
+        "endpoint":    {"type": "string",  "filterable": True},
+        "method":      {"type": "string",  "filterable": True},  # "GET", "POST", etc.
+        "version":     {"type": "string",  "filterable": True},
+        "deprecated":  {"type": "boolean", "filterable": True},
+    }
+})
+
+# Ingest respects the schema — metadata is validated at write time
+client.collection.add("product-docs", documents=[
+    {
+        "id": "pd-drag-drop-001",
+        "text": "Drag-and-drop upload lets you add files directly to a bucket...",
+        "metadata": {
+            "version": "2.3",
+            "category": "feature",
+            "updated_at": "2026-02-10",
+            "language": "en",
+        }
+    },
+])
+
+# Queries are scoped to the right domain — clean, relevant results
+product_results = client.search(
+    "product-docs",
+    "how to upload files",
+    top_k=5,
+    filters={"category": "guide", "language": "en"}
+)
+
+support_results = client.search(
+    "support-tickets",
+    "upload fails with 413 error",
+    top_k=8,
+    filters={"status": "open", "product_area": "ingestion"}
+)
+
+api_results = client.search(
+    "api-reference",
+    "create embedding endpoint",
+    top_k=3,
+    filters={"deprecated": False, "version": "v1"}
+)
+```
+
+```typescript
+// TypeScript — domain-aligned collections with schemas
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+
+// Create collections with explicit schemas
+await client.collection.create('product-docs', {
+  schema: {
+    fields: {
+      version:    { type: 'string',  filterable: true },
+      category:   { type: 'string',  filterable: true },
+      updated_at: { type: 'date',    filterable: true },
+      language:   { type: 'string',  filterable: true },
+    },
+  },
+});
+
+await client.collection.create('support-tickets', {
+  schema: {
+    fields: {
+      status:       { type: 'string',  filterable: true },
+      priority:     { type: 'string',  filterable: true },
+      created_at:   { type: 'date',    filterable: true },
+      product_area: { type: 'string',  filterable: true },
+    },
+  },
+});
+
+await client.collection.create('api-reference', {
+  schema: {
+    fields: {
+      endpoint:   { type: 'string',  filterable: true },
+      method:     { type: 'string',  filterable: true },
+      version:    { type: 'string',  filterable: true },
+      deprecated: { type: 'boolean', filterable: true },
+    },
+  },
+});
+
+// Ingest with required metadata fields
+await client.collection.add('product-docs', {
+  documents: [
+    {
+      id: 'pd-drag-drop-001',
+      text: 'Drag-and-drop upload lets you add files directly to a bucket...',
+      metadata: {
+        version: '2.3',
+        category: 'feature',
+        updated_at: '2026-02-10',
+        language: 'en',
+      },
+    },
+  ],
+});
+
+// Query scoped to the right domain
+const productResults = await client.search('product-docs', 'how to upload files', {
+  topK: 5,
+  filters: { category: 'guide', language: 'en' },
+});
+
+const supportResults = await client.search('support-tickets', 'upload fails with 413 error', {
+  topK: 8,
+  filters: { status: 'open', product_area: 'ingestion' },
+});
+```
+
+**Collection design guidelines:**
+
+| Principle | Rationale |
+|-----------|-----------|
+| One collection per domain/product area | Keeps semantic similarity scores meaningful within a shared context |
+| Define schema before ingesting | Filterable fields must be declared at creation; adding them later requires re-indexing |
+| Use consistent metadata keys | Inconsistent keys (`doc_type` vs `docType`) break filter queries silently |
+| Aim for 10k–1M documents per collection | Smaller: consider merging. Larger: consider sharding by time or sub-domain |
+| Name collections for query intent | `product-docs` not `confluence-export-2025` |
+
+If a single query would logically need to search across multiple domains (e.g., a universal search bar), run parallel searches against each relevant collection and merge results by score on the client side.
+
+## Reference
+
+- https://docs.schift.io/collections/schema
+- https://docs.schift.io/collections/design

schift_cli/data/schift-best-practices/references/search-hybrid.md

@@ -0,0 +1,163 @@
+---
+title: Combine Semantic Search with Metadata Filters for Precise Retrieval
+impact: HIGH
+impactDescription: Pure semantic search returns results that are thematically similar but contextually wrong — e.g., querying "quarterly revenue" returns documents from the wrong year or department. Adding metadata filters eliminates these false positives without sacrificing semantic ranking quality.
+tags:
+  - search
+  - filters
+  - metadata
+  - hybrid-search
+  - retrieval
+---
+
+## Combine Semantic Search with Metadata Filters for Precise Retrieval
+
+Semantic search ranks results by vector similarity, which captures meaning well but is blind to structured attributes like dates, departments, document types, or user ownership. Without filters, a query for "quarterly revenue" might surface a 2019 report with slightly higher similarity than the 2025 report you actually need.
+
+The `filters` parameter in `client.search()` applies exact-match or range constraints on document metadata before semantic ranking. The semantic model then ranks only the pre-filtered candidates, giving you both precision and relevance.
+
+### Incorrect
+
+Pure semantic search without filters returns semantically similar but contextually irrelevant results:
+
+```python
+# Python — semantic-only search, no filters
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+
+# Searching for 2025 finance data — but will return results from any year/department
+results = client.search(
+    "finance-reports",
+    "quarterly revenue growth",
+    top_k=5
+)
+
+# Results may include 2019 marketing reports that mention "revenue" prominently
+for r in results:
+    print(r.text, r.score)
+```
+
+```typescript
+// TypeScript — semantic-only search, no filters
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+
+// No filters — returns semantically similar but potentially wrong-year docs
+const results = await client.search('finance-reports', 'quarterly revenue growth', {
+  topK: 5,
+});
+```
+
+This wastes top_k slots on irrelevant documents and requires the LLM to reason about document provenance instead of just answering.
+
+### Correct
+
+Use the `filters` parameter to narrow candidates by metadata before semantic ranking:
+
+```python
+# Python — hybrid search with metadata filters
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+
+# Narrow to 2025 finance department docs, then rank by semantic similarity
+results = client.search(
+    "finance-reports",
+    "quarterly revenue growth",
+    top_k=5,
+    filters={
+        "year": 2025,
+        "department": "finance",
+    }
+)
+
+# All results are from the right context; semantic score reflects true relevance
+for r in results:
+    print(r.metadata["year"], r.metadata["department"], r.score, r.text[:100])
+
+
+# Multi-value filter: search across multiple years
+results = client.search(
+    "finance-reports",
+    "revenue trend analysis",
+    top_k=10,
+    filters={
+        "year": {"$in": [2023, 2024, 2025]},
+        "department": "finance",
+        "doc_type": "quarterly-report",
+    }
+)
+
+# Filter by recency for support tickets
+recent_tickets = client.search(
+    "support-tickets",
+    "payment gateway timeout error",
+    top_k=8,
+    filters={
+        "status": "open",
+        "created_after": "2026-01-01",
+        "priority": {"$in": ["high", "critical"]},
+    }
+)
+```
+
+```typescript
+// TypeScript — hybrid search with metadata filters
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+
+// Narrow to 2025 finance docs, then semantic ranking
+const results = await client.search('finance-reports', 'quarterly revenue growth', {
+  topK: 5,
+  filters: {
+    year: 2025,
+    department: 'finance',
+  },
+});
+
+results.forEach(r => {
+  console.log(r.metadata.year, r.metadata.department, r.score, r.text.slice(0, 100));
+});
+
+// Multi-value filter across years
+const trendResults = await client.search('finance-reports', 'revenue trend analysis', {
+  topK: 10,
+  filters: {
+    year: { $in: [2023, 2024, 2025] },
+    department: 'finance',
+    doc_type: 'quarterly-report',
+  },
+});
+
+// Filter for recent high-priority support tickets
+const recentTickets = await client.search(
+  'support-tickets',
+  'payment gateway timeout error',
+  {
+    topK: 8,
+    filters: {
+      status: 'open',
+      created_after: '2026-01-01',
+      priority: { $in: ['high', 'critical'] },
+    },
+  }
+);
+```
+
+**Supported filter operators:**
+
+| Operator | Meaning | Example |
+|----------|---------|---------|
+| exact match | `field: value` | `"year": 2025` |
+| `$in` | any of these values | `"status": { "$in": ["open", "pending"] }` |
+| `$gt` / `$gte` | greater than | `"score": { "$gte": 90 }` |
+| `$lt` / `$lte` | less than | `"created_at": { "$lt": "2025-06-01" }` |
+
+Define filterable fields in your collection schema at creation time — only schema-declared fields are indexed for filtering.
+
+## Reference
+
+- https://docs.schift.io/search/filters

schift_cli/data/schift-best-practices/references/search-similarity-tuning.md

@@ -0,0 +1,134 @@
+---
+title: Tune top_k and Similarity Threshold for Your Use Case
+impact: CRITICAL
+impactDescription: Using wrong top_k or no score threshold causes either noisy results (too many irrelevant chunks passed to LLM) or missed relevant results, directly degrading answer quality and wasting tokens.
+tags:
+  - search
+  - retrieval
+  - top_k
+  - score_threshold
+  - precision
+  - recall
+---
+
+## Tune top_k and Similarity Threshold for Your Use Case
+
+The default `top_k=10` is a reasonable starting point but rarely optimal. Every use case has a different precision/recall trade-off: a chatbot answering factual questions needs tight, high-confidence results, while a research tool benefits from broader recall. Passing too many low-quality chunks to an LLM inflates costs and degrades answer quality; returning too few misses the relevant context entirely.
+
+Use `score_threshold` to discard results below a minimum similarity score. Schift returns cosine similarity scores in `[0, 1]` — results below ~0.65 are typically noise.
+
+### Incorrect
+
+Using default `top_k=10` everywhere with no threshold filter:
+
+```python
+# Python — no threshold, fixed top_k for all use cases
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+
+# Same settings for everything: chatbot QA, exploratory search, code lookup
+results = client.search(collection_id, query)  # top_k defaults to 10, no threshold
+
+for r in results:
+    context += r.text  # may include chunks with score=0.42 (largely irrelevant)
+```
+
+```typescript
+// TypeScript — no threshold, fixed top_k for all use cases
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+
+// Same settings for everything
+const results = await client.search(collectionId, query);  // top_k=10, no threshold
+
+const context = results.map(r => r.text).join('\n');  // includes noise
+```
+
+This passes irrelevant chunks to the LLM, increases prompt token count, and produces hallucinated or unfocused answers.
+
+### Correct
+
+Set `top_k` and `score_threshold` based on the retrieval goal:
+
+```python
+# Python — tuned per use case
+from schift import Schift
+
+client = Schift(api_key="sch_...")
+
+# Chatbot Q&A: tight precision, high confidence
+qa_results = client.search(
+    collection_id,
+    query,
+    top_k=3,            # few but precise chunks
+    score_threshold=0.78  # discard anything below 78% similarity
+)
+
+# Research / exploratory: wider recall, lower threshold
+explore_results = client.search(
+    collection_id,
+    query,
+    top_k=30,           # broad recall for exploration
+    score_threshold=0.55  # accept weaker matches
+)
+
+# Code retrieval: moderate, function-level precision
+code_results = client.search(
+    collection_id,
+    query,
+    top_k=5,
+    score_threshold=0.70
+)
+
+# Always check: if nothing passes threshold, surface a "no results" message
+if not qa_results:
+    return "I couldn't find relevant information for your question."
+```
+
+```typescript
+// TypeScript — tuned per use case
+import { Schift } from '@schift-io/sdk';
+
+const client = new Schift({ apiKey: 'sch_...' });
+
+// Chatbot Q&A: tight precision, high confidence
+const qaResults = await client.search(collectionId, query, {
+  topK: 3,
+  scoreThreshold: 0.78,
+});
+
+// Research / exploratory: wider recall
+const exploreResults = await client.search(collectionId, query, {
+  topK: 30,
+  scoreThreshold: 0.55,
+});
+
+// Code retrieval
+const codeResults = await client.search(collectionId, query, {
+  topK: 5,
+  scoreThreshold: 0.70,
+});
+
+// Handle empty results gracefully
+if (qaResults.length === 0) {
+  return "I couldn't find relevant information for your question.";
+}
+```
+
+**Rule of thumb by use case:**
+
+| Use Case | top_k | score_threshold |
+|----------|-------|-----------------|
+| Chatbot Q&A | 3–5 | 0.75–0.82 |
+| Document Q&A (long docs) | 5–8 | 0.70–0.78 |
+| Code retrieval | 3–6 | 0.68–0.75 |
+| Research / exploration | 20–50 | 0.50–0.62 |
+| Fact verification | 1–3 | 0.80–0.88 |
+
+Start with these ranges and adjust based on observed result quality. Log score distributions during development to calibrate thresholds for your data.
+
+## Reference
+
+- https://docs.schift.io/search/parameters

schift_cli/display.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Any, Sequence
+
+from rich.console import Console
+from rich.panel import Panel
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeRemainingColumn,
+)
+from rich.table import Table
+
+console = Console()
+error_console = Console(stderr=True)
+
+
+# -- Tables -------------------------------------------------------------------
+
+def print_table(
+    title: str,
+    columns: Sequence[str],
+    rows: Sequence[Sequence[Any]],
+    *,
+    caption: str | None = None,
+) -> None:
+    table = Table(title=title, caption=caption, show_lines=False)
+    for col in columns:
+        table.add_column(col, overflow="fold")
+    for row in rows:
+        table.add_row(*(str(v) for v in row))
+    console.print(table)
+
+
+def print_kv(title: str, data: dict[str, Any]) -> None:
+    """Print a key-value panel (e.g. model details, db stats)."""
+    table = Table(show_header=False, box=None, padding=(0, 2))
+    table.add_column("Key", style="bold cyan", no_wrap=True)
+    table.add_column("Value")
+    for k, v in data.items():
+        table.add_row(k, str(v))
+    console.print(Panel(table, title=title, border_style="blue"))
+
+
+# -- Status messages ----------------------------------------------------------
+
+def success(msg: str) -> None:
+    console.print(f"[bold green]OK[/]  {msg}")
+
+
+def info(msg: str) -> None:
+    console.print(f"[bold blue]--[/]  {msg}")
+
+
+def warn(msg: str) -> None:
+    error_console.print(f"[bold yellow]WARN[/]  {msg}")
+
+
+def error(msg: str) -> None:
+    error_console.print(f"[bold red]ERROR[/]  {msg}")
+
+
+# -- Progress -----------------------------------------------------------------
+
+def make_progress() -> Progress:
+    return Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        BarColumn(),
+        MofNCompleteColumn(),
+        TimeRemainingColumn(),
+        console=console,
+    )
+
+
+def spinner(description: str = "Working...") -> Progress:
+    """A simple indeterminate spinner for operations without a known total."""
+    return Progress(
+        SpinnerColumn(),
+        TextColumn("[progress.description]{task.description}"),
+        console=console,
+    )

schift_cli/main.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import click
+
+from schift_cli import __version__
+from schift_cli.commands.auth import auth
+from schift_cli.commands.bench import bench
+from schift_cli.commands.catalog import catalog
+from schift_cli.commands.db import db
+from schift_cli.commands.embed import embed
+from schift_cli.commands.migrate import migrate
+from schift_cli.commands.query import query
+from schift_cli.commands.skill import skill
+from schift_cli.commands.usage import usage
+
+
+@click.group()
+@click.version_option(version=__version__, prog_name="schift")
+def cli() -> None:
+    """Schift CLI -- AI Agent Framework.
+
+    Manage agents, embedding models, vector collections, and migrations
+    from the command line.
+    """
+
+
+cli.add_command(auth)
+cli.add_command(catalog)
+cli.add_command(embed)
+cli.add_command(bench)
+cli.add_command(migrate)
+cli.add_command(db)
+cli.add_command(query)
+cli.add_command(skill)
+cli.add_command(usage)
+
+
+if __name__ == "__main__":
+    cli()

schift_cli-0.1.0.dist-info/METADATA

@@ -0,0 +1,12 @@
+Metadata-Version: 2.4
+Name: schift-cli
+Version: 0.1.0
+Summary: Schift CLI — manage agents, embeddings, and vector collections
+License: MIT
+Requires-Python: >=3.10
+Requires-Dist: click>=8.1
+Requires-Dist: httpx>=0.27
+Requires-Dist: rich>=13.0
+Provides-Extra: dev
+Requires-Dist: pytest>=8.0; extra == "dev"
+Requires-Dist: pytest-cov>=5.0; extra == "dev"