botstash 0.1.0__tar.gz

botstash-0.1.0/.gitignore

@@ -0,0 +1,39 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.egg-info/
+*.egg
+dist/
+build/
+*.whl
+
+# Virtual environments
+.venv/
+venv/
+env/
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+*~
+
+# Testing
+.pytest_cache/
+.coverage
+htmlcov/
+.mypy_cache/
+.ruff_cache/
+
+# Environment / secrets
+.env
+.vectorboard.env
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Staging (runtime output)
+staging/

botstash-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Michael
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

botstash-0.1.0/PKG-INFO

@@ -0,0 +1,86 @@
+Metadata-Version: 2.4
+Name: botstash
+Version: 0.1.0
+Summary: CLI tool and WebUI that ingests LMS course exports and Echo360 transcripts into an AnythingLLM workspace for embedded course chatbots.
+Project-URL: Homepage, https://github.com/michael-borck/botstash
+Project-URL: Repository, https://github.com/michael-borck/botstash
+Project-URL: Issues, https://github.com/michael-borck/botstash/issues
+Author-email: Michael <michael@example.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: anythingllm,botstash,chatbot,education,lms,rag
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Education
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Education
+Requires-Python: >=3.11
+Requires-Dist: click>=8.1
+Requires-Dist: fastapi>=0.115
+Requires-Dist: httpx>=0.27
+Requires-Dist: jinja2>=3.1
+Requires-Dist: pdfminer-six>=20231228
+Requires-Dist: python-docx>=1.1
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: python-pptx>=1.0
+Requires-Dist: uvicorn>=0.32
+Requires-Dist: webvtt-py>=0.5
+Description-Content-Type: text/markdown
+
+# BotStash
+
+A CLI tool and lightweight WebUI that ingests LMS course exports (Blackboard IMSCC / Canvas) and Echo360 VTT transcripts, uploads content to an [AnythingLLM](https://anythingllm.com/) workspace, and returns embeddable chatbot code for pasting into a course page.
+
+## Features
+
+- **IMSCC ingestion** — unzips and walks Blackboard/Canvas common cartridge exports
+- **Transcript ingestion** — processes folders of Echo360 VTT files
+- **Multi-format extraction** — PPTX, DOCX, PDF, VTT, QTI quizzes
+- **Auto-classification** — heuristic tagging of content types (lecture, worksheet, assignment, etc.)
+- **AnythingLLM integration** — uploads documents, manages workspaces, retrieves embed code
+- **WebUI** — FastAPI + Jinja2 interface for non-terminal users
+
+## Installation
+
+```bash
+pip install botstash
+```
+
+## Quick Start
+
+```bash
+# Full pipeline
+botstash run course.zip transcripts/ \
+  --workspace ISYS2001 \
+  --url https://your-anythingllm.instance \
+  --key YOUR_API_KEY
+
+# Two-step workflow (extract, review, embed)
+botstash extract course.zip transcripts/ --output ./staging/
+# ... review staging/tags.json ...
+botstash embed ./staging/ --workspace ISYS2001
+
+# Launch WebUI
+botstash serve
+```
+
+## Development
+
+```bash
+# Clone and install in dev mode
+git clone https://github.com/michael-borck/botstash.git
+cd botstash
+uv sync --dev
+
+# Run checks
+uv run ruff check src/ tests/
+uv run mypy src/
+uv run pytest
+```
+
+## License
+
+MIT

botstash-0.1.0/README.md

@@ -0,0 +1,54 @@
+# BotStash
+
+A CLI tool and lightweight WebUI that ingests LMS course exports (Blackboard IMSCC / Canvas) and Echo360 VTT transcripts, uploads content to an [AnythingLLM](https://anythingllm.com/) workspace, and returns embeddable chatbot code for pasting into a course page.
+
+## Features
+
+- **IMSCC ingestion** — unzips and walks Blackboard/Canvas common cartridge exports
+- **Transcript ingestion** — processes folders of Echo360 VTT files
+- **Multi-format extraction** — PPTX, DOCX, PDF, VTT, QTI quizzes
+- **Auto-classification** — heuristic tagging of content types (lecture, worksheet, assignment, etc.)
+- **AnythingLLM integration** — uploads documents, manages workspaces, retrieves embed code
+- **WebUI** — FastAPI + Jinja2 interface for non-terminal users
+
+## Installation
+
+```bash
+pip install botstash
+```
+
+## Quick Start
+
+```bash
+# Full pipeline
+botstash run course.zip transcripts/ \
+  --workspace ISYS2001 \
+  --url https://your-anythingllm.instance \
+  --key YOUR_API_KEY
+
+# Two-step workflow (extract, review, embed)
+botstash extract course.zip transcripts/ --output ./staging/
+# ... review staging/tags.json ...
+botstash embed ./staging/ --workspace ISYS2001
+
+# Launch WebUI
+botstash serve
+```
+
+## Development
+
+```bash
+# Clone and install in dev mode
+git clone https://github.com/michael-borck/botstash.git
+cd botstash
+uv sync --dev
+
+# Run checks
+uv run ruff check src/ tests/
+uv run mypy src/
+uv run pytest
+```
+
+## License
+
+MIT

botstash-0.1.0/VECTORBOARD_SPEC.md

@@ -0,0 +1,257 @@
+# VectorBoard — Project Specification v0.1
+
+> A CLI tool and lightweight WebUI that ingests an LMS course export (Blackboard IMSCC or Canvas) and a folder of Echo360 VTT transcripts, extracts and classifies content, uploads it to an AnythingLLM workspace, and returns embeddable chatbot code for pasting into a course page.
+
+---
+
+## Name Rationale
+
+**VectorBoard** — vector embeddings + "board" as a generic term for a course/learning board. Intentionally LMS-agnostic so it survives the move from Blackboard to Canvas without renaming. Python package name: `vectorboard`.
+
+---
+
+## Goals
+
+- Give students a citeable, course-aware AI chatbot embedded directly in their LMS page
+- Require minimal technical effort from the lecturer (one command or a simple WebUI)
+- Support both automated and manually-reviewed classification workflows
+- Be LMS-portable: IMSCC is the common cartridge format shared by Blackboard and Canvas
+
+---
+
+## Architecture
+
+```
+vectorboard/
+├── cli.py                  # Click-based entry points
+├── config.py               # Config file + env var handling
+├── pipeline.py             # Orchestrates the full run
+├── ingester/
+│   ├── imscc.py            # Unzip + walk IMSCC/common cartridge structure
+│   └── transcript.py       # VTT folder ingestion
+├── extractors/
+│   ├── docx.py
+│   ├── pdf.py
+│   ├── pptx.py
+│   ├── vtt.py              # VTT → clean text (timestamps stripped)
+│   ├── qti.py              # Blackboard/Canvas quiz XML → questions only
+│   ├── url_tracker.py      # Log video/external URLs found in manifests
+│   └── unit_outline.py     # Bespoke plugin (to be provided at build time)
+├── classifier/
+│   └── auto.py             # Heuristic + optional AI classification
+├── anythingllm/
+│   └── client.py           # AnythingLLM REST API wrapper
+└── webui/
+    └── app.py              # FastAPI + Jinja2 simple UI
+```
+
+---
+
+## CLI Commands
+
+### Full pipeline (default usage)
+
+```bash
+vectorboard run course.zip transcripts/ \
+  --workspace ISYS2001 \
+  --url https://your-anythingllm.instance \
+  --key YOUR_API_KEY
+```
+
+### Two-step workflow (for manual tag review)
+
+```bash
+# Step 1: extract and auto-classify, write tags file
+vectorboard extract course.zip transcripts/ --output ./staging/
+
+# Step 2 (optional): open staging/tags.json in any editor and adjust types/titles
+
+# Step 3: embed using the (possibly edited) tags file
+vectorboard embed ./staging/ --workspace ISYS2001 --tags staging/tags.json
+
+# Step 4: retrieve embed code
+vectorboard chatbot ISYS2001
+```
+
+### Reset a workspace (new semester)
+
+```bash
+vectorboard embed ./staging/ --workspace ISYS2001 --reset
+```
+
+`--reset` clears all documents from the workspace before re-uploading. The workspace itself (and its embed code URL) is preserved, so the LMS page never needs updating mid-year.
+
+---
+
+## Workspace Strategy
+
+- **One workspace per unit code** (e.g. `ISYS2001`, `ISYS6020`)
+- Semester changeover is handled by running with `--reset` — workspace identity and embed URL remain stable
+- Archiving, deletion, and multi-semester history management are intentionally out of scope; these are handled manually in the AnythingLLM UI if needed
+
+---
+
+## Document Classification
+
+Auto-classification uses a two-pass approach:
+
+**Pass 1 — Filename and path heuristics**
+
+Keywords matched against filename and parent folder name (case-insensitive):
+
+| Keywords | Assigned type |
+|---|---|
+| `lecture`, `slides`, `week` | `lecture` |
+| `worksheet`, `tutorial`, `lab` | `worksheet` |
+| `assignment`, `task`, `project` | `assignment` |
+| `rubric`, `marking`, `criteria` | `rubric` |
+| `outline`, `unit guide`, `course guide` | `unit_outline` |
+| `quiz`, `test` | `quiz` |
+| `reading`, `article`, `chapter` | `reading` |
+| *(VTT file)* | `transcript` |
+| *(URL-only entry)* | `video_url` |
+
+**Pass 2 — Content heuristics**
+
+First 500 characters of extracted text are inspected for structural signals (e.g. QTI XML namespace → always `quiz`; "Learning Outcomes" header → likely `unit_outline`). Pass 2 overrides Pass 1 on high-confidence matches.
+
+**Fallback:** `misc`
+
+### Tags file format (`tags.json`)
+
+Generated automatically after extraction. Edit before the embed step to override any classification.
+
+```json
+[
+  {
+    "source_file": "Week1_Intro.pptx",
+    "extracted_as": "staging/Week1_Intro.txt",
+    "type": "lecture",
+    "title": "Week 1: Introduction to Information Systems",
+    "week": 1
+  },
+  {
+    "source_file": "Assessment1_Rubric.docx",
+    "extracted_as": "staging/Assessment1_Rubric.txt",
+    "type": "rubric",
+    "title": "Assessment 1 Rubric",
+    "week": null
+  }
+]
+```
+
+Valid `type` values: `lecture`, `worksheet`, `assignment`, `rubric`, `unit_outline`, `quiz`, `reading`, `transcript`, `video_url`, `misc`
+
+---
+
+## Extractor Behaviour
+
+| Source format | Output | Notes |
+|---|---|---|
+| PPTX | Plain text per slide, slide number preserved as a heading | Speaker notes included |
+| DOCX | Plain text, headings preserved as markdown-style markers | Unit outline uses bespoke plugin |
+| PDF | Plain text via `pdfminer.six` or `pymupdf` | `--ocr` flag available as fallback |
+| VTT | Clean plain text, timestamps stripped, filename used as title | One text file per transcript |
+| QTI XML (quizzes) | Question text only, one question per line | Answer choices and correct answers excluded (student-facing) |
+| Video / external URLs | Written to `urls_log.txt` with the page/item context they appeared in | Not fetched or scraped |
+
+### Unit outline extraction
+
+A bespoke Python plugin handles unit outlines. Interface expected at build time:
+
+```python
+def extract_unit_outline(file_path: str) -> str:
+    """Takes an absolute path to a DOCX/PDF unit outline.
+    Returns extracted plain text as a single string."""
+```
+
+---
+
+## AnythingLLM Integration
+
+Uses the AnythingLLM REST API. Key operations:
+
+| Operation | API call |
+|---|---|
+| Check workspace exists | `GET /api/v1/workspaces` |
+| Create workspace | `POST /api/v1/workspace/new` |
+| Upload document | `POST /api/v1/document/upload` |
+| Move document to workspace | `POST /api/v1/workspace/{slug}/update-embeddings` |
+| Reset workspace documents | `DELETE` or re-embed with `--reset` flag |
+| Get chatbot embed code | `GET /api/v1/workspace/{slug}/chatbot-embed` *(confirm endpoint)* |
+
+### Configuration
+
+Credentials are never passed as plain CLI arguments. Set via:
+
+- A `.vectorboard.env` file in the working directory, or
+- Environment variables: `ANYTHINGLLM_URL`, `ANYTHINGLLM_KEY`
+
+A `vectorboard init` command scaffolds the `.vectorboard.env` file interactively.
+
+---
+
+## Staging and Cleanup
+
+- Extracted text files are written to a `./staging/` directory during processing
+- After a successful embed, staging files are deleted automatically unless `--keep-staging` is passed
+- The `tags.json` file is always retained after a run (useful for auditing what was embedded)
+
+---
+
+## WebUI
+
+A lightweight FastAPI + Jinja2 interface for lecturers who prefer not to use the terminal.
+
+**Workflow:**
+
+1. Upload course ZIP and transcript folder
+2. Review extracted items with auto-assigned types (inline dropdowns to override before embedding)
+3. Click **Embed to AnythingLLM**
+4. Copy the chatbot embed code from the result panel
+
+The WebUI is a thin wrapper around the same pipeline — no separate logic. Launch with:
+
+```bash
+vectorboard serve
+```
+
+---
+
+## Out of Scope (v1)
+
+- Fetching or scraping external URLs
+- Semester archiving or workspace deletion
+- Multi-workspace merge
+- Direct LMS API integration (embed code is copied manually)
+- LMS platforms other than Blackboard/Canvas IMSCC format
+- Student authentication or access control on the WebUI
+
+---
+
+## Open Questions (resolve at build time)
+
+1. **Unit outline plugin** — confirm function signature and whether it handles both DOCX and PDF inputs
+2. **AnythingLLM embed code** — confirm whether the embed snippet is retrievable via API or only from the UI; if UI-only, the `chatbot` command prints the workspace URL and instructs the user to copy from there
+3. **AnythingLLM document metadata** — determine which metadata fields (week number, doc type) can be passed at upload time to improve citation quality in responses
+4. **Canvas differences** — confirm whether Canvas IMSCC exports use the same QTI format and manifest structure as Blackboard; adjust `imscc.py` accordingly
+
+---
+
+## Suggested Dependency Stack
+
+| Purpose | Library |
+|---|---|
+| CLI framework | `click` |
+| WebUI | `fastapi` + `jinja2` + `uvicorn` |
+| PPTX extraction | `python-pptx` |
+| DOCX extraction | `python-docx` |
+| PDF extraction | `pdfminer.six` or `pymupdf` |
+| VTT parsing | `webvtt-py` |
+| HTTP client (AnythingLLM API) | `httpx` |
+| Config / env | `python-dotenv` |
+| Packaging | `pyproject.toml` with `pip install -e .` |
+
+---
+
+*VectorBoard v0.1 spec — generated April 2026. Continue build in a Claude Code session.*

botstash-0.1.0/pyproject.toml

@@ -0,0 +1,74 @@
+[project]
+name = "botstash"
+version = "0.1.0"
+description = "CLI tool and WebUI that ingests LMS course exports and Echo360 transcripts into an AnythingLLM workspace for embedded course chatbots."
+readme = "README.md"
+license = "MIT"
+requires-python = ">=3.11"
+authors = [
+    { name = "Michael", email = "michael@example.com" },
+]
+keywords = ["lms", "chatbot", "anythingllm", "botstash", "education", "rag"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Education",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Education",
+]
+dependencies = [
+    "click>=8.1",
+    "httpx>=0.27",
+    "python-dotenv>=1.0",
+    "python-pptx>=1.0",
+    "python-docx>=1.1",
+    "pdfminer.six>=20231228",
+    "webvtt-py>=0.5",
+    "fastapi>=0.115",
+    "jinja2>=3.1",
+    "uvicorn>=0.32",
+]
+
+[project.scripts]
+botstash = "botstash.cli:cli"
+
+[project.urls]
+Homepage = "https://github.com/michael-borck/botstash"
+Repository = "https://github.com/michael-borck/botstash"
+Issues = "https://github.com/michael-borck/botstash/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/botstash"]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 88
+src = ["src", "tests"]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "N", "W", "UP", "B", "SIM", "TCH"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+addopts = "-v --tb=short"
+
+[dependency-groups]
+dev = [
+    "mypy>=1.20.0",
+    "pytest>=9.0.3",
+    "ruff>=0.15.9",
+    "twine>=6.2.0",
+]

botstash-0.1.0/src/botstash/__init__.py

@@ -0,0 +1,3 @@
+"""BotStash — LMS course content to AnythingLLM chatbot pipeline."""
+
+__version__ = "0.1.0"

botstash-0.1.0/src/botstash/anythingllm/client.py

@@ -0,0 +1 @@
+"""AnythingLLM REST API wrapper."""

botstash-0.1.0/src/botstash/classifier/auto.py

@@ -0,0 +1 @@
+"""Heuristic and optional AI classification."""

botstash-0.1.0/src/botstash/cli.py

@@ -0,0 +1,70 @@
+"""Click-based CLI entry points."""
+
+import click
+
+from botstash import __version__
+
+
+@click.group()
+@click.version_option(version=__version__, prog_name="botstash")
+def cli() -> None:
+    """BotStash — LMS course content to AnythingLLM chatbot pipeline."""
+
+
+@cli.command()
+@click.argument("course_zip", type=click.Path(exists=True))
+@click.argument("transcripts", type=click.Path(exists=True))
+@click.option("--workspace", required=True, help="AnythingLLM workspace name.")
+@click.option("--url", envvar="ANYTHINGLLM_URL", help="AnythingLLM instance URL.")
+@click.option("--key", envvar="ANYTHINGLLM_KEY", help="AnythingLLM API key.")
+@click.option("--keep-staging", is_flag=True, help="Keep staging files after embed.")
+def run(
+    course_zip: str,
+    transcripts: str,
+    workspace: str,
+    url: str | None,
+    key: str | None,
+    keep_staging: bool,
+) -> None:
+    """Run the full extract-classify-embed pipeline."""
+    click.echo("botstash run: not yet implemented")
+
+
+@cli.command()
+@click.argument("course_zip", type=click.Path(exists=True))
+@click.argument("transcripts", type=click.Path(exists=True))
+@click.option("--output", default="./staging", help="Output directory.")
+def extract(course_zip: str, transcripts: str, output: str) -> None:
+    """Extract and auto-classify content from a course export."""
+    click.echo("botstash extract: not yet implemented")
+
+
+@cli.command()
+@click.argument("staging_dir", type=click.Path(exists=True))
+@click.option("--workspace", required=True, help="AnythingLLM workspace name.")
+@click.option("--tags", type=click.Path(exists=True), help="Path to tags.json.")
+@click.option("--reset", is_flag=True, help="Clear workspace before uploading.")
+def embed(staging_dir: str, workspace: str, tags: str | None, reset: bool) -> None:
+    """Embed staged documents into an AnythingLLM workspace."""
+    click.echo("botstash embed: not yet implemented")
+
+
+@cli.command()
+@click.argument("workspace")
+def chatbot(workspace: str) -> None:
+    """Retrieve the chatbot embed code for a workspace."""
+    click.echo("botstash chatbot: not yet implemented")
+
+
+@cli.command()
+def init() -> None:
+    """Scaffold a .botstash.env configuration file."""
+    click.echo("botstash init: not yet implemented")
+
+
+@cli.command()
+@click.option("--host", default="127.0.0.1", help="Host to bind to.")
+@click.option("--port", default=8000, help="Port to bind to.")
+def serve(host: str, port: int) -> None:
+    """Launch the BotStash WebUI."""
+    click.echo("botstash serve: not yet implemented")

botstash-0.1.0/src/botstash/config.py

@@ -0,0 +1 @@
+"""Configuration file and environment variable handling."""

botstash-0.1.0/src/botstash/extractors/docx.py

@@ -0,0 +1 @@
+"""DOCX text extraction."""

botstash-0.1.0/src/botstash/extractors/pdf.py

@@ -0,0 +1 @@
+"""PDF text extraction."""

botstash-0.1.0/src/botstash/extractors/pptx.py

@@ -0,0 +1 @@
+"""PPTX text extraction."""

botstash-0.1.0/src/botstash/extractors/qti.py

@@ -0,0 +1 @@
+"""QTI XML quiz extraction — question text only."""

botstash-0.1.0/src/botstash/extractors/unit_outline.py

@@ -0,0 +1 @@
+"""Bespoke unit outline extraction plugin."""

botstash-0.1.0/src/botstash/extractors/url_tracker.py

@@ -0,0 +1 @@
+"""Log video/external URLs found in manifests."""

botstash-0.1.0/src/botstash/extractors/vtt.py

@@ -0,0 +1 @@
+"""VTT to clean text conversion (timestamps stripped)."""

botstash-0.1.0/src/botstash/ingester/imscc.py

@@ -0,0 +1 @@
+"""Unzip and walk IMSCC/common cartridge structure."""

botstash-0.1.0/src/botstash/ingester/transcript.py

@@ -0,0 +1 @@
+"""VTT folder ingestion."""

botstash-0.1.0/src/botstash/pipeline.py

@@ -0,0 +1 @@
+"""Orchestrates the full extract-classify-embed pipeline."""

botstash-0.1.0/src/botstash/webui/app.py

@@ -0,0 +1 @@
+"""FastAPI + Jinja2 WebUI."""

botstash-0.1.0/tests/test_version.py

@@ -0,0 +1,7 @@
+"""Basic smoke tests."""
+
+from botstash import __version__
+
+
+def test_version() -> None:
+    assert __version__ == "0.1.0"