purestrata 0.1.0__py3-none-any.whl

purestrata/__init__.py

@@ -0,0 +1,67 @@
+"""PureStrata: the open data layer for manufacturing AI.
+
+Ingest drawings, spec sheets, and failure knowledge into a portable,
+provenance-tagged corpus, query it from any LLM (MCP server, OpenAI-style
+tools, or plain Python), and export the same corpus as fine-tuning
+datasets.
+"""
+
+from .corpus import Corpus, write_chunks
+from .export.finetune import (
+    build_chat_records,
+    build_cot_records,
+    build_tool_trace_records,
+    write_jsonl,
+)
+from .ingest.cad import ingest_dxf
+from .ingest.failures import ingest_failures
+from .ingest.manual import ingest_manual
+from .search import SearchIndex, has_cjk, tokenize
+from .tools import TOOLS, ToolError, ToolExecutor, anthropic_tools, json_schemas, openai_tools
+from .types import (
+    ALL_KINDS,
+    DRAWING,
+    FAILURE_CASE,
+    MANUAL_SECTION,
+    PART_RECORD,
+    PROCEDURE,
+    SAFETY_NOTICE,
+    Chunk,
+    Provenance,
+    derive_doc_id,
+    new_chunk,
+)
+
+__version__ = "0.1.0"
+
+__all__ = [
+    "ALL_KINDS",
+    "DRAWING",
+    "FAILURE_CASE",
+    "MANUAL_SECTION",
+    "PART_RECORD",
+    "PROCEDURE",
+    "SAFETY_NOTICE",
+    "TOOLS",
+    "Chunk",
+    "Corpus",
+    "Provenance",
+    "SearchIndex",
+    "ToolError",
+    "ToolExecutor",
+    "anthropic_tools",
+    "build_chat_records",
+    "build_cot_records",
+    "build_tool_trace_records",
+    "derive_doc_id",
+    "has_cjk",
+    "ingest_dxf",
+    "ingest_failures",
+    "ingest_manual",
+    "json_schemas",
+    "new_chunk",
+    "openai_tools",
+    "tokenize",
+    "write_chunks",
+    "write_jsonl",
+]

purestrata/cli.py

@@ -0,0 +1,161 @@
+"""Command line interface.
+
+purestrata ingest-manual <files...> --out corpus/
+purestrata ingest-dxf <files...> --out corpus/
+purestrata ingest-failures <files...> --out corpus/
+purestrata inspect --corpus corpus/
+purestrata search "spindle vibration" --corpus corpus/
+purestrata tools openai            # print tool schemas for Grok/OpenAI
+purestrata export cot --corpus corpus/ --out dataset.jsonl
+purestrata serve --corpus corpus/  # MCP server over stdio
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from .corpus import Corpus, write_chunks
+from .ingest.cad import ingest_dxf
+from .ingest.failures import ingest_failures
+from .ingest.manual import ingest_manual
+from .tools import ToolError, anthropic_tools, json_schemas, openai_tools
+from .types import ALL_KINDS, derive_doc_id
+
+INGEST_COMMANDS = {
+    "ingest-manual": ingest_manual,
+    "ingest-dxf": ingest_dxf,
+    "ingest-failures": ingest_failures,
+}
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="purestrata",
+        description="The open data layer for manufacturing AI.",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    for name, help_text in (
+        ("ingest-manual", "Ingest Markdown/PDF manuals and spec sheets"),
+        ("ingest-dxf", "Ingest ASCII DXF CAD drawings"),
+        ("ingest-failures", "Ingest CSV/JSONL failure logs"),
+    ):
+        p = sub.add_parser(name, help=help_text)
+        p.add_argument("paths", nargs="+", help="Input file paths")
+        p.add_argument("--out", required=True, help="Output corpus directory")
+
+    p = sub.add_parser("search", help="Full-text search across the corpus")
+    p.add_argument("query", help="Search text (English or Japanese)")
+    p.add_argument("--corpus", default="corpus", help="Corpus directory (default: ./corpus)")
+    p.add_argument("--kind", choices=ALL_KINDS, help="Restrict search to one record kind")
+    p.add_argument("--top-k", type=int, default=5)
+
+    p = sub.add_parser("inspect", help="Print corpus statistics")
+    p.add_argument("--corpus", default="corpus", help="Corpus directory (default: ./corpus)")
+
+    p = sub.add_parser("tools", help="Print tool schemas for LLM integrations")
+    p.add_argument("format", choices=["openai", "anthropic", "json"])
+
+    p = sub.add_parser("export", help="Export fine-tuning datasets from a corpus")
+    p.add_argument("format", choices=["chat", "tool_trace", "cot"])
+    p.add_argument("--corpus", default="corpus", help="Corpus directory (default: ./corpus)")
+    p.add_argument("--out", required=True, help="Output JSONL path")
+    p.add_argument("--limit", type=int, help="Maximum number of records")
+
+    p = sub.add_parser("serve", help="Run the MCP server (stdio) over a corpus")
+    p.add_argument("--corpus", default="corpus", help="Corpus directory (default: ./corpus)")
+    return parser
+
+
+def _run_ingest(args: argparse.Namespace) -> int:
+    ingest = INGEST_COMMANDS[args.command]
+    total = 0
+    for path in args.paths:
+        if not Path(path).exists():
+            print(f"error: file not found: {path}", file=sys.stderr)
+            return 1
+        chunks = ingest(path)
+        doc_id = derive_doc_id(path)
+        out_path = write_chunks(chunks, args.out, doc_id)
+        total += len(chunks)
+        print(f"{path}: {len(chunks)} records -> {out_path}")
+    print(f"Total: {total} records written to {args.out}")
+    return 0
+
+
+def _run_search(args: argparse.Namespace) -> int:
+    corpus = Corpus.load(args.corpus)
+    results = corpus.search(args.query, kinds=args.kind, top_k=args.top_k)
+    if not results:
+        print("No records matched.")
+        return 0
+    for rank, (chunk, score) in enumerate(results, start=1):
+        preview = chunk.text[:200].replace("\n", " ")
+        print(f"{rank}. [{chunk.kind}] score={score:.3f} id={chunk.chunk_id}")
+        print(f"   {preview}")
+        print(f"   source: {chunk.provenance}")
+    return 0
+
+
+def _run_export(args: argparse.Namespace) -> int:
+    from .export.finetune import (
+        build_chat_records,
+        build_cot_records,
+        build_tool_trace_records,
+        write_jsonl,
+    )
+
+    builders = {
+        "chat": build_chat_records,
+        "tool_trace": build_tool_trace_records,
+        "cot": build_cot_records,
+    }
+    corpus = Corpus.load(args.corpus)
+    records = builders[args.format](corpus, limit=args.limit)
+    out_path = write_jsonl(records, args.out)
+    print(f"Exported {len(records)} {args.format} records -> {out_path}")
+    return 0
+
+
+def _run_serve(args: argparse.Namespace) -> int:
+    from .mcp_server import serve
+
+    serve(args.corpus)
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = _build_parser()
+    args = parser.parse_args(argv)
+    try:
+        if args.command in INGEST_COMMANDS:
+            return _run_ingest(args)
+        if args.command == "search":
+            return _run_search(args)
+        if args.command == "inspect":
+            print(json.dumps(Corpus.load(args.corpus).stats(), indent=2, ensure_ascii=False))
+            return 0
+        if args.command == "tools":
+            schema = {
+                "openai": openai_tools,
+                "anthropic": anthropic_tools,
+                "json": json_schemas,
+            }[args.format]
+            print(json.dumps(schema(), indent=2, ensure_ascii=False))
+            return 0
+        if args.command == "export":
+            return _run_export(args)
+        if args.command == "serve":
+            return _run_serve(args)
+        parser.error(f"Unknown command: {args.command}")
+        return 2
+    except (FileNotFoundError, ToolError, RuntimeError, ValueError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

purestrata/corpus.py

@@ -0,0 +1,78 @@
+"""Corpus storage: a directory of portable, auditable JSONL files.
+
+A corpus is just ``*.strata.jsonl`` files, one JSON object per line per
+chunk. No database, no server, no lock-in: companies can drop a corpus on a
+file share, ship it to an air-gapped network, or commit small ones to git.
+"""
+
+from __future__ import annotations
+
+import json
+from collections import Counter
+from pathlib import Path
+
+from .search import SearchIndex
+from .types import ALL_KINDS, Chunk
+
+CORPUS_SUFFIX = ".strata.jsonl"
+
+
+def write_chunks(chunks: list[Chunk], out_dir: str | Path, doc_id: str) -> Path:
+    """Write one ``<doc_id>.strata.jsonl`` file for an ingested document."""
+    out_dir = Path(out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+    out_path = out_dir / f"{doc_id}{CORPUS_SUFFIX}"
+    with open(out_path, "w", encoding="utf-8") as handle:
+        for chunk in chunks:
+            handle.write(json.dumps(chunk.to_dict(), ensure_ascii=False))
+            handle.write("\n")
+    return out_path
+
+
+class Corpus:
+    """An in-memory view of a directory of strata files, with search."""
+
+    def __init__(self, chunks: list[Chunk]):
+        self.chunks = list(chunks)
+        self._index: SearchIndex | None = None
+
+    @classmethod
+    def load(cls, corpus_dir: str | Path) -> Corpus:
+        """Load every ``*.strata.jsonl`` file under ``corpus_dir``."""
+        corpus_dir = Path(corpus_dir)
+        if not corpus_dir.exists():
+            raise FileNotFoundError(f"Corpus directory not found: {corpus_dir}")
+        chunks: list[Chunk] = []
+        for path in sorted(corpus_dir.glob(f"*{CORPUS_SUFFIX}")):
+            for line in path.read_text(encoding="utf-8").splitlines():
+                if line.strip():
+                    chunks.append(Chunk.from_dict(json.loads(line)))
+        return cls(chunks)
+
+    @property
+    def index(self) -> SearchIndex:
+        if self._index is None:
+            self._index = SearchIndex(self.chunks)
+        return self._index
+
+    def stats(self) -> dict:
+        by_kind = Counter(chunk.kind for chunk in self.chunks)
+        by_doc = Counter(chunk.provenance.doc_id for chunk in self.chunks)
+        return {
+            "total_chunks": len(self.chunks),
+            "by_kind": {kind: by_kind[kind] for kind in ALL_KINDS if by_kind[kind]},
+            "documents": sorted(by_doc),
+        }
+
+    def search(self, query: str, kinds=None, top_k: int = 5) -> list[tuple[Chunk, float]]:
+        return self.index.search(query, kinds=kinds, top_k=top_k)
+
+    def get(self, chunk_id: str) -> Chunk | None:
+        for chunk in self.chunks:
+            if chunk.chunk_id == chunk_id:
+                return chunk
+        return None
+
+    def by_kind(self, *kinds: str) -> list[Chunk]:
+        wanted = set(kinds)
+        return [chunk for chunk in self.chunks if chunk.kind in wanted]

purestrata/export/__init__.py

@@ -0,0 +1,15 @@
+"""Fine-tuning dataset exporters."""
+
+from .finetune import (
+    build_chat_records,
+    build_cot_records,
+    build_tool_trace_records,
+    write_jsonl,
+)
+
+__all__ = [
+    "build_chat_records",
+    "build_cot_records",
+    "build_tool_trace_records",
+    "write_jsonl",
+]

purestrata/export/finetune.py

@@ -0,0 +1,214 @@
+"""Export a corpus into fine-tuning dataset formats.
+
+The same corpus that serves live RAG queries is also the raw material for
+training data. Three shapes are produced:
+
+- ``chat``: system/user/assistant message JSONL, the format accepted by
+  OpenAI's and xAI's fine-tuning APIs.
+- ``tool_trace``: message JSONL that includes a model-issued tool call,
+  the tool result, and the grounded final answer, for tool-use training.
+- ``cot``: instruction / reasoning / output records with sources, a
+  scaffold for chain-of-thought refinement by domain experts.
+
+Every record carries provenance, so dataset quality can be audited and
+reproduced from the source documents.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from ..corpus import Corpus
+from ..tools.executor import ToolError, ToolExecutor
+from ..types import FAILURE_CASE, PROCEDURE, SAFETY_NOTICE, Chunk
+
+SYSTEM_PROMPT = (
+    "You are a manufacturing maintenance assistant. Answer using verified maintenance "
+    "and failure knowledge, and cite the source locator of every record you rely on."
+)
+
+
+def _sources(chunk: Chunk) -> list[dict]:
+    return [chunk.provenance.to_dict()]
+
+
+def _failure_question(chunk: Chunk) -> str:
+    meta = chunk.metadata
+    equipment = meta.get("equipment", "")
+    symptom = meta.get("symptom", "")
+    where = f" on equipment '{equipment}'" if equipment else ""
+    observed = f" We observe: {symptom.rstrip('.。')}." if symptom else ""
+    return (
+        f"We are troubleshooting a problem{where}.{observed}"
+        " What is the likely cause and the countermeasure?"
+    )
+
+
+def _failure_answer(chunk: Chunk) -> str:
+    meta = chunk.metadata
+    lines = []
+    part = meta.get("part_number")
+    if part:
+        lines.append(f"Involved part: {part}")
+    lines.append(f"Likely cause: {meta.get('cause') or 'not recorded'}")
+    lines.append(f"Countermeasure: {meta.get('countermeasure') or 'not recorded'}")
+    lines.append(f"Severity: {meta.get('severity', 'unspecified')}")
+    lines.append(f"Source: {chunk.provenance}")
+    return "\n".join(lines)
+
+
+def _procedure_qa(chunk: Chunk) -> dict | None:
+    meta = chunk.metadata
+    steps = meta.get("steps") or []
+    if not steps:
+        return None
+    equipment = meta.get("equipment")
+    scope = f" on {equipment}" if equipment else ""
+    question = f"List the steps of '{meta.get('title', 'the procedure')}'{scope}."
+    answer = "\n".join(f"{i}. {step}" for i, step in enumerate(steps, start=1))
+    answer += f"\nSource: {chunk.provenance}"
+    return {
+        "messages": [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": question},
+            {"role": "assistant", "content": answer},
+        ],
+        "sources": _sources(chunk),
+    }
+
+
+def _safety_qa(chunk: Chunk) -> dict | None:
+    if not chunk.text.strip():
+        return None
+    question = (
+        f"What safety precaution applies to '{chunk.metadata.get('section', 'this equipment')}'?"
+    )
+    answer = f"{chunk.text}\nSource: {chunk.provenance}"
+    return {
+        "messages": [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": question},
+            {"role": "assistant", "content": answer},
+        ],
+        "sources": _sources(chunk),
+    }
+
+
+def build_chat_records(corpus: Corpus, limit: int | None = None) -> list[dict]:
+    """Build chat-format fine-tuning records from failure cases, procedures, and notices."""
+    records: list[dict] = []
+    for chunk in corpus.chunks:
+        record = None
+        if chunk.kind == FAILURE_CASE:
+            record = {
+                "messages": [
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {"role": "user", "content": _failure_question(chunk)},
+                    {"role": "assistant", "content": _failure_answer(chunk)},
+                ],
+                "sources": _sources(chunk),
+            }
+        elif chunk.kind == PROCEDURE:
+            record = _procedure_qa(chunk)
+        elif chunk.kind == SAFETY_NOTICE:
+            record = _safety_qa(chunk)
+        if record is not None:
+            records.append(record)
+    return records[:limit] if limit else records
+
+
+def build_tool_trace_records(corpus: Corpus, limit: int | None = None) -> list[dict]:
+    """Build tool-use fine-tuning records from failure cases.
+
+    Each record shows the model choosing ``search_failure_cases`` with a
+    symptom-driven query, receiving provenance-tagged results, and
+    answering from them.
+    """
+    executor = ToolExecutor(corpus)
+    records: list[dict] = []
+    for i, chunk in enumerate(corpus.by_kind(FAILURE_CASE), start=1):
+        meta = chunk.metadata
+        symptom = meta.get("symptom") or meta.get("cause") or chunk.text[:120]
+        question = _failure_question(chunk)
+        call_args = {"query": symptom, "top_k": 3}
+        try:
+            tool_result = executor.execute("search_failure_cases", call_args)
+        except ToolError:
+            continue
+        call_id = f"call_{i:04d}"
+        final = _failure_answer(chunk)
+        record = {
+            "messages": [
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user", "content": question},
+                {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [
+                        {
+                            "id": call_id,
+                            "type": "function",
+                            "function": {
+                                "name": "search_failure_cases",
+                                "arguments": json.dumps(call_args, ensure_ascii=False),
+                            },
+                        }
+                    ],
+                },
+                {
+                    "role": "tool",
+                    "tool_call_id": call_id,
+                    "content": json.dumps(tool_result, ensure_ascii=False, default=str),
+                },
+                {"role": "assistant", "content": final},
+            ],
+            "sources": _sources(chunk),
+        }
+        records.append(record)
+    return records[:limit] if limit else records
+
+
+def build_cot_records(corpus: Corpus, limit: int | None = None) -> list[dict]:
+    """Build chain-of-thought records from failure cases.
+
+    The reasoning trace is a transparent scaffold over structured fields,
+    designed to be reviewed and sharpened by maintenance engineers before
+    training. That expert-in-the-loop pass is where dataset quality comes
+    from; these records make the work auditable.
+    """
+    records: list[dict] = []
+    for chunk in corpus.by_kind(FAILURE_CASE):
+        meta = chunk.metadata
+        cause = meta.get("cause") or "not recorded"
+        countermeasure = meta.get("countermeasure") or "not recorded"
+        reasoning = (
+            f"The report concerns equipment '{meta.get('equipment', 'unspecified')}'. "
+            f"The observed symptom is: {meta.get('symptom', 'unspecified')}. "
+            f"Failure case {meta.get('case_id', chunk.chunk_id)} in the verified history "
+            f"matches this equipment and symptom. Its recorded root cause was: {cause}. "
+            f"The applied countermeasure was: {countermeasure}. "
+            "Since the symptom matches the recorded case, the same root cause is the most "
+            "likely explanation, and the same countermeasure applies before further "
+            "investigation."
+        )
+        records.append(
+            {
+                "instruction": _failure_question(chunk),
+                "reasoning": reasoning,
+                "output": _failure_answer(chunk),
+                "sources": _sources(chunk),
+            }
+        )
+    return records[:limit] if limit else records
+
+
+def write_jsonl(records: list[dict], out_path: str | Path) -> Path:
+    """Write records as JSONL and return the path."""
+    out_path = Path(out_path)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(out_path, "w", encoding="utf-8") as handle:
+        for record in records:
+            handle.write(json.dumps(record, ensure_ascii=False, default=str))
+            handle.write("\n")
+    return out_path

purestrata/ingest/__init__.py

@@ -0,0 +1,7 @@
+"""Ingestion adapters: manuals, CAD drawings, and failure logs."""
+
+from .cad import ingest_dxf
+from .failures import ingest_failures
+from .manual import ingest_manual
+
+__all__ = ["ingest_dxf", "ingest_failures", "ingest_manual"]