moorcheh-client 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- moorcheh_client-0.1.0/PKG-INFO +134 -0
- moorcheh_client-0.1.0/README.md +124 -0
- moorcheh_client-0.1.0/moorcheh/__init__.py +6 -0
- moorcheh_client-0.1.0/moorcheh/__main__.py +4 -0
- moorcheh_client-0.1.0/moorcheh/api.py +109 -0
- moorcheh_client-0.1.0/moorcheh/cli.py +365 -0
- moorcheh_client-0.1.0/moorcheh/compose/docker-compose.yml +33 -0
- moorcheh_client-0.1.0/moorcheh/docker_runtime.py +174 -0
- moorcheh_client-0.1.0/moorcheh_client.egg-info/PKG-INFO +134 -0
- moorcheh_client-0.1.0/moorcheh_client.egg-info/SOURCES.txt +14 -0
- moorcheh_client-0.1.0/moorcheh_client.egg-info/dependency_links.txt +1 -0
- moorcheh_client-0.1.0/moorcheh_client.egg-info/entry_points.txt +2 -0
- moorcheh_client-0.1.0/moorcheh_client.egg-info/requires.txt +4 -0
- moorcheh_client-0.1.0/moorcheh_client.egg-info/top_level.txt +1 -0
- moorcheh_client-0.1.0/pyproject.toml +31 -0
- moorcheh_client-0.1.0/setup.cfg +4 -0
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: moorcheh-client
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Moorcheh client and runtime launcher for on-prem server containers.
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: requests>=2.31.0
|
|
8
|
+
Provides-Extra: web
|
|
9
|
+
Requires-Dist: Flask>=3.0.0; extra == "web"
|
|
10
|
+
|
|
11
|
+
# moorcheh-client (Python)
|
|
12
|
+
|
|
13
|
+
This directory publishes the **`moorcheh-client`** distribution on PyPI. The importable module is still `moorcheh` (`from moorcheh import MoorchehApiClient`).
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
- `pip install moorcheh-client` (from this directory: `pip install .`)
|
|
18
|
+
|
|
19
|
+
Default server image: `moorcheh/server:latest` (override with `moorcheh up --server-image …` or `MOORCHEH_SERVER_IMAGE` to pin a specific release).
|
|
20
|
+
|
|
21
|
+
## Data directory
|
|
22
|
+
|
|
23
|
+
`moorcheh up` stores your vectors and documents at:
|
|
24
|
+
|
|
25
|
+
- `~/.moorcheh/data` (e.g. `C:\Users\<you>\.moorcheh\data` on Windows)
|
|
26
|
+
- `moorcheh_data_store.json`, `namespace_registry.json`
|
|
27
|
+
|
|
28
|
+
`moorcheh down` stops containers but **keeps** `~/.moorcheh`. Back up that folder to save everything.
|
|
29
|
+
|
|
30
|
+
## CLI Commands
|
|
31
|
+
|
|
32
|
+
- `moorcheh up` — start Moorcheh; uses host Ollama on `127.0.0.1:11434` when already running
|
|
33
|
+
- `moorcheh down` — stop Moorcheh containers (does not stop host Ollama)
|
|
34
|
+
- `moorcheh status` — `GET /health` (items, max_items, remaining)
|
|
35
|
+
- `moorcheh namespace-create` — create a text or vector namespace
|
|
36
|
+
- `moorcheh namespace-list` — list namespaces and per-namespace `item_count`
|
|
37
|
+
- `moorcheh upload-documents` — `POST /namespaces/{namespace_name}/documents` (async job)
|
|
38
|
+
- `moorcheh upload-vectors` — `POST /namespaces/{namespace_name}/vectors` (async job)
|
|
39
|
+
- `moorcheh upload-job-status` — poll upload job (documents or vectors)
|
|
40
|
+
- `moorcheh items-get` — get items by id within a namespace
|
|
41
|
+
- `moorcheh items-delete` — delete items by id within a namespace
|
|
42
|
+
- `moorcheh namespace-delete` — delete a namespace and all its items
|
|
43
|
+
- `moorcheh search` — semantic search
|
|
44
|
+
|
|
45
|
+
## Global item limit (100k)
|
|
46
|
+
|
|
47
|
+
Moorcheh stores at most **100,000 items total** across all namespaces (text + vectors).
|
|
48
|
+
|
|
49
|
+
- Check quota: `moorcheh status` → `items`, `max_items`, `remaining`
|
|
50
|
+
- Uploads that would add **new** ids over the cap return **409** (entire batch rejected)
|
|
51
|
+
- Re-uploading an existing id in the same namespace is an update and does not use extra quota
|
|
52
|
+
- Deleting items or a namespace frees quota immediately
|
|
53
|
+
|
|
54
|
+
Item ids are **unique per namespace** (the same id string may exist in different namespaces).
|
|
55
|
+
|
|
56
|
+
## Examples
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
moorcheh up
|
|
60
|
+
moorcheh status
|
|
61
|
+
moorcheh namespace-create --name docs --type text
|
|
62
|
+
moorcheh namespace-create --name products_vec --type vector --vector-dimension 768
|
|
63
|
+
moorcheh upload-documents --namespace-name docs --documents-file docs-upload.json
|
|
64
|
+
moorcheh upload-vectors --namespace-name products_vec --vectors-file vectors-upload.json
|
|
65
|
+
moorcheh upload-job-status --namespace-name docs --job-id job-abc123
|
|
66
|
+
moorcheh items-get --namespace-name docs --ids-json "[\"doc-1\"]"
|
|
67
|
+
moorcheh items-delete --namespace-name docs --ids-json "[\"doc-1\"]"
|
|
68
|
+
moorcheh search --query "on prem retrieval" --namespaces docs --top-k 5
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Documents upload payload (`docs-upload.json`):
|
|
72
|
+
|
|
73
|
+
```json
|
|
74
|
+
{
|
|
75
|
+
"documents": [
|
|
76
|
+
{
|
|
77
|
+
"id": "doc-1",
|
|
78
|
+
"text": "Moorcheh on-prem retrieval test",
|
|
79
|
+
"team": "ai"
|
|
80
|
+
}
|
|
81
|
+
]
|
|
82
|
+
}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Python API
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from moorcheh import MoorchehApiClient, MoorchehApiError
|
|
89
|
+
|
|
90
|
+
client = MoorchehApiClient("http://localhost:8080")
|
|
91
|
+
health = client.health()
|
|
92
|
+
print(health["items"], health["remaining"])
|
|
93
|
+
|
|
94
|
+
try:
|
|
95
|
+
client.upload_namespace_vectors("products_vec", {"vectors": [...]})
|
|
96
|
+
except MoorchehApiError as e:
|
|
97
|
+
if e.is_item_limit_exceeded:
|
|
98
|
+
print(e.body) # items, max_items, requested_new
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Optional Flask Demo App
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
pip install .[web]
|
|
105
|
+
python client/app.py
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Ollama: host vs Docker
|
|
109
|
+
|
|
110
|
+
By default, `moorcheh up` uses Ollama on `http://127.0.0.1:11434` when it is already running (typical on Windows/macOS). It does **not** start a second Ollama container in that case.
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
moorcheh up
|
|
114
|
+
# Using Ollama already running at http://127.0.0.1:11434 (moorcheh-ollama container not started)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Force behavior:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
moorcheh up --use-host-ollama # never start moorcheh-ollama
|
|
121
|
+
moorcheh up --bundled-ollama # always start moorcheh-ollama container
|
|
122
|
+
moorcheh up --bundled-ollama --ollama-port 11435 # bundled Ollama on another host port
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Port conflict fallback
|
|
126
|
+
|
|
127
|
+
If **8080** is busy:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
moorcheh up --server-port 8081
|
|
131
|
+
moorcheh status --base-url http://localhost:8081
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`moorcheh up` removes stale `moorcheh-onprem-server` (and `moorcheh-ollama` only when starting bundled Ollama). Data persists under `~/.moorcheh/data` across restarts.
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# moorcheh-client (Python)
|
|
2
|
+
|
|
3
|
+
This directory publishes the **`moorcheh-client`** distribution on PyPI. The importable module is still `moorcheh` (`from moorcheh import MoorchehApiClient`).
|
|
4
|
+
|
|
5
|
+
## Install
|
|
6
|
+
|
|
7
|
+
- `pip install moorcheh-client` (from this directory: `pip install .`)
|
|
8
|
+
|
|
9
|
+
Default server image: `moorcheh/server:latest` (override with `moorcheh up --server-image …` or `MOORCHEH_SERVER_IMAGE` to pin a specific release).
|
|
10
|
+
|
|
11
|
+
## Data directory
|
|
12
|
+
|
|
13
|
+
`moorcheh up` stores your vectors and documents at:
|
|
14
|
+
|
|
15
|
+
- `~/.moorcheh/data` (e.g. `C:\Users\<you>\.moorcheh\data` on Windows)
|
|
16
|
+
- `moorcheh_data_store.json`, `namespace_registry.json`
|
|
17
|
+
|
|
18
|
+
`moorcheh down` stops containers but **keeps** `~/.moorcheh`. Back up that folder to save everything.
|
|
19
|
+
|
|
20
|
+
## CLI Commands
|
|
21
|
+
|
|
22
|
+
- `moorcheh up` — start Moorcheh; uses host Ollama on `127.0.0.1:11434` when already running
|
|
23
|
+
- `moorcheh down` — stop Moorcheh containers (does not stop host Ollama)
|
|
24
|
+
- `moorcheh status` — `GET /health` (items, max_items, remaining)
|
|
25
|
+
- `moorcheh namespace-create` — create a text or vector namespace
|
|
26
|
+
- `moorcheh namespace-list` — list namespaces and per-namespace `item_count`
|
|
27
|
+
- `moorcheh upload-documents` — `POST /namespaces/{namespace_name}/documents` (async job)
|
|
28
|
+
- `moorcheh upload-vectors` — `POST /namespaces/{namespace_name}/vectors` (async job)
|
|
29
|
+
- `moorcheh upload-job-status` — poll upload job (documents or vectors)
|
|
30
|
+
- `moorcheh items-get` — get items by id within a namespace
|
|
31
|
+
- `moorcheh items-delete` — delete items by id within a namespace
|
|
32
|
+
- `moorcheh namespace-delete` — delete a namespace and all its items
|
|
33
|
+
- `moorcheh search` — semantic search
|
|
34
|
+
|
|
35
|
+
## Global item limit (100k)
|
|
36
|
+
|
|
37
|
+
Moorcheh stores at most **100,000 items total** across all namespaces (text + vectors).
|
|
38
|
+
|
|
39
|
+
- Check quota: `moorcheh status` → `items`, `max_items`, `remaining`
|
|
40
|
+
- Uploads that would add **new** ids over the cap return **409** (entire batch rejected)
|
|
41
|
+
- Re-uploading an existing id in the same namespace is an update and does not use extra quota
|
|
42
|
+
- Deleting items or a namespace frees quota immediately
|
|
43
|
+
|
|
44
|
+
Item ids are **unique per namespace** (the same id string may exist in different namespaces).
|
|
45
|
+
|
|
46
|
+
## Examples
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
moorcheh up
|
|
50
|
+
moorcheh status
|
|
51
|
+
moorcheh namespace-create --name docs --type text
|
|
52
|
+
moorcheh namespace-create --name products_vec --type vector --vector-dimension 768
|
|
53
|
+
moorcheh upload-documents --namespace-name docs --documents-file docs-upload.json
|
|
54
|
+
moorcheh upload-vectors --namespace-name products_vec --vectors-file vectors-upload.json
|
|
55
|
+
moorcheh upload-job-status --namespace-name docs --job-id job-abc123
|
|
56
|
+
moorcheh items-get --namespace-name docs --ids-json "[\"doc-1\"]"
|
|
57
|
+
moorcheh items-delete --namespace-name docs --ids-json "[\"doc-1\"]"
|
|
58
|
+
moorcheh search --query "on prem retrieval" --namespaces docs --top-k 5
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Documents upload payload (`docs-upload.json`):
|
|
62
|
+
|
|
63
|
+
```json
|
|
64
|
+
{
|
|
65
|
+
"documents": [
|
|
66
|
+
{
|
|
67
|
+
"id": "doc-1",
|
|
68
|
+
"text": "Moorcheh on-prem retrieval test",
|
|
69
|
+
"team": "ai"
|
|
70
|
+
}
|
|
71
|
+
]
|
|
72
|
+
}
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Python API
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from moorcheh import MoorchehApiClient, MoorchehApiError
|
|
79
|
+
|
|
80
|
+
client = MoorchehApiClient("http://localhost:8080")
|
|
81
|
+
health = client.health()
|
|
82
|
+
print(health["items"], health["remaining"])
|
|
83
|
+
|
|
84
|
+
try:
|
|
85
|
+
client.upload_namespace_vectors("products_vec", {"vectors": [...]})
|
|
86
|
+
except MoorchehApiError as e:
|
|
87
|
+
if e.is_item_limit_exceeded:
|
|
88
|
+
print(e.body) # items, max_items, requested_new
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Optional Flask Demo App
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
pip install .[web]
|
|
95
|
+
python client/app.py
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## Ollama: host vs Docker
|
|
99
|
+
|
|
100
|
+
By default, `moorcheh up` uses Ollama on `http://127.0.0.1:11434` when it is already running (typical on Windows/macOS). It does **not** start a second Ollama container in that case.
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
moorcheh up
|
|
104
|
+
# Using Ollama already running at http://127.0.0.1:11434 (moorcheh-ollama container not started)
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
Force behavior:
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
moorcheh up --use-host-ollama # never start moorcheh-ollama
|
|
111
|
+
moorcheh up --bundled-ollama # always start moorcheh-ollama container
|
|
112
|
+
moorcheh up --bundled-ollama --ollama-port 11435 # bundled Ollama on another host port
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Port conflict fallback
|
|
116
|
+
|
|
117
|
+
If **8080** is busy:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
moorcheh up --server-port 8081
|
|
121
|
+
moorcheh status --base-url http://localhost:8081
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`moorcheh up` removes stale `moorcheh-onprem-server` (and `moorcheh-ollama` only when starting bundled Ollama). Data persists under `~/.moorcheh/data` across restarts.
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
import requests
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class MoorchehApiError(Exception):
|
|
9
|
+
"""HTTP error from the Moorcheh API (includes parsed JSON body when available)."""
|
|
10
|
+
|
|
11
|
+
def __init__(
|
|
12
|
+
self,
|
|
13
|
+
message: str,
|
|
14
|
+
status_code: int,
|
|
15
|
+
body: dict[str, Any] | None = None,
|
|
16
|
+
) -> None:
|
|
17
|
+
super().__init__(message)
|
|
18
|
+
self.status_code = status_code
|
|
19
|
+
self.body = body
|
|
20
|
+
|
|
21
|
+
@property
|
|
22
|
+
def is_item_limit_exceeded(self) -> bool:
|
|
23
|
+
return self.status_code == 409 and bool(self.body)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class MoorchehApiClient:
|
|
27
|
+
def __init__(self, base_url: str, timeout: int = 30) -> None:
|
|
28
|
+
self.base_url = base_url.rstrip("/")
|
|
29
|
+
self.timeout = timeout
|
|
30
|
+
|
|
31
|
+
def _raise_for_status(self, response: requests.Response) -> None:
|
|
32
|
+
if response.ok:
|
|
33
|
+
return
|
|
34
|
+
body: dict[str, Any] | None = None
|
|
35
|
+
try:
|
|
36
|
+
parsed = response.json()
|
|
37
|
+
if isinstance(parsed, dict):
|
|
38
|
+
body = parsed
|
|
39
|
+
except requests.JSONDecodeError:
|
|
40
|
+
body = None
|
|
41
|
+
message = (
|
|
42
|
+
str(body.get("message"))
|
|
43
|
+
if body and body.get("message") is not None
|
|
44
|
+
else response.text or response.reason
|
|
45
|
+
)
|
|
46
|
+
raise MoorchehApiError(message, response.status_code, body)
|
|
47
|
+
|
|
48
|
+
def _post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
49
|
+
response = requests.post(
|
|
50
|
+
f"{self.base_url}{path}",
|
|
51
|
+
json=payload,
|
|
52
|
+
timeout=self.timeout,
|
|
53
|
+
)
|
|
54
|
+
self._raise_for_status(response)
|
|
55
|
+
return response.json()
|
|
56
|
+
|
|
57
|
+
def _get(self, path: str) -> dict[str, Any]:
|
|
58
|
+
response = requests.get(f"{self.base_url}{path}", timeout=self.timeout)
|
|
59
|
+
self._raise_for_status(response)
|
|
60
|
+
return response.json()
|
|
61
|
+
|
|
62
|
+
def health(self) -> dict[str, Any]:
|
|
63
|
+
"""
|
|
64
|
+
GET /health — includes global item quota:
|
|
65
|
+
items, max_items, remaining, model, status.
|
|
66
|
+
"""
|
|
67
|
+
return self._get("/health")
|
|
68
|
+
|
|
69
|
+
def create_namespace(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
70
|
+
return self._post("/namespaces", payload)
|
|
71
|
+
|
|
72
|
+
def list_namespaces(self) -> dict[str, Any]:
|
|
73
|
+
return self._get("/namespaces")
|
|
74
|
+
|
|
75
|
+
def delete_namespace(self, namespace_name: str) -> dict[str, Any]:
|
|
76
|
+
response = requests.delete(
|
|
77
|
+
f"{self.base_url}/namespaces/{namespace_name}",
|
|
78
|
+
timeout=self.timeout,
|
|
79
|
+
)
|
|
80
|
+
self._raise_for_status(response)
|
|
81
|
+
return response.json()
|
|
82
|
+
|
|
83
|
+
def delete_namespace_job_status(self, namespace_name: str, job_id: str) -> dict[str, Any]:
|
|
84
|
+
return self._get(f"/namespaces/{namespace_name}/delete-jobs/{job_id}")
|
|
85
|
+
|
|
86
|
+
def upload_namespace_documents(self, namespace_name: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
87
|
+
"""POST documents (async job). May raise MoorchehApiError with status 409 if global item cap exceeded."""
|
|
88
|
+
return self._post(f"/namespaces/{namespace_name}/documents", payload)
|
|
89
|
+
|
|
90
|
+
def upload_namespace_vectors(self, namespace_name: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
91
|
+
"""POST vectors (async job). May raise MoorchehApiError with status 409 if global item cap exceeded."""
|
|
92
|
+
return self._post(f"/namespaces/{namespace_name}/vectors", payload)
|
|
93
|
+
|
|
94
|
+
def get_namespace_items(self, namespace_name: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
95
|
+
return self._post(f"/namespaces/{namespace_name}/items/get", payload)
|
|
96
|
+
|
|
97
|
+
def delete_namespace_items(self, namespace_name: str, payload: dict[str, Any]) -> dict[str, Any]:
|
|
98
|
+
"""Delete by item id within namespace (ids are unique per namespace, not globally)."""
|
|
99
|
+
return self._post(f"/namespaces/{namespace_name}/items/delete", payload)
|
|
100
|
+
|
|
101
|
+
def upload_job_status(self, namespace_name: str, job_id: str) -> dict[str, Any]:
|
|
102
|
+
"""Poll document or vector upload job status."""
|
|
103
|
+
return self._get(f"/namespaces/{namespace_name}/upload-jobs/{job_id}")
|
|
104
|
+
|
|
105
|
+
def upload_namespace_documents_job_status(self, namespace_name: str, job_id: str) -> dict[str, Any]:
|
|
106
|
+
return self.upload_job_status(namespace_name, job_id)
|
|
107
|
+
|
|
108
|
+
def search(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
109
|
+
return self._post("/search", payload)
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from moorcheh.api import MoorchehApiClient, MoorchehApiError
|
|
10
|
+
from moorcheh.docker_runtime import (
|
|
11
|
+
DEFAULT_OLLAMA_IMAGE,
|
|
12
|
+
DEFAULT_OLLAMA_MODEL,
|
|
13
|
+
DEFAULT_SERVER_IMAGE,
|
|
14
|
+
ComposeCommandError,
|
|
15
|
+
down,
|
|
16
|
+
up,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _print_json(payload: dict[str, Any]) -> None:
|
|
21
|
+
print(json.dumps(payload, indent=2))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _parse_json(text: str, field_name: str) -> dict[str, Any]:
|
|
25
|
+
try:
|
|
26
|
+
data = json.loads(text)
|
|
27
|
+
except json.JSONDecodeError as exc:
|
|
28
|
+
raise ValueError(f"{field_name} must be valid JSON: {exc}") from exc
|
|
29
|
+
if not isinstance(data, dict):
|
|
30
|
+
raise ValueError(f"{field_name} must be a JSON object")
|
|
31
|
+
return data
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _parse_json_array(text: str, field_name: str) -> list[Any]:
|
|
35
|
+
try:
|
|
36
|
+
data = json.loads(text)
|
|
37
|
+
except json.JSONDecodeError as exc:
|
|
38
|
+
raise ValueError(f"{field_name} must be valid JSON: {exc}") from exc
|
|
39
|
+
if not isinstance(data, list):
|
|
40
|
+
raise ValueError(f"{field_name} must be a JSON array")
|
|
41
|
+
return data
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _api_client(base_url: str) -> MoorchehApiClient:
|
|
45
|
+
return MoorchehApiClient(base_url=base_url)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def cmd_up(args: argparse.Namespace) -> int:
|
|
49
|
+
bundled_ollama: bool | None
|
|
50
|
+
if args.bundled_ollama and args.use_host_ollama:
|
|
51
|
+
raise ValueError("Use only one of --bundled-ollama or --use-host-ollama")
|
|
52
|
+
if args.bundled_ollama:
|
|
53
|
+
bundled_ollama = True
|
|
54
|
+
elif args.use_host_ollama:
|
|
55
|
+
bundled_ollama = False
|
|
56
|
+
else:
|
|
57
|
+
bundled_ollama = None
|
|
58
|
+
|
|
59
|
+
result, started_bundled_ollama, data_dir = up(
|
|
60
|
+
server_image=args.server_image,
|
|
61
|
+
ollama_image=args.ollama_image,
|
|
62
|
+
server_port=args.server_port,
|
|
63
|
+
ollama_port=args.ollama_port,
|
|
64
|
+
ollama_model=args.ollama_model,
|
|
65
|
+
bundled_ollama=bundled_ollama,
|
|
66
|
+
ollama_host=args.ollama_host,
|
|
67
|
+
)
|
|
68
|
+
if result.stdout.strip():
|
|
69
|
+
print(result.stdout.strip())
|
|
70
|
+
if result.stderr.strip():
|
|
71
|
+
print(result.stderr.strip(), file=sys.stderr)
|
|
72
|
+
print(f"Data directory: {data_dir}")
|
|
73
|
+
if started_bundled_ollama:
|
|
74
|
+
print(
|
|
75
|
+
f"Started Moorcheh server + Ollama container (Ollama on host port {args.ollama_port})"
|
|
76
|
+
)
|
|
77
|
+
else:
|
|
78
|
+
print(
|
|
79
|
+
f"Using Ollama already running at http://{args.ollama_host}:{args.ollama_port} "
|
|
80
|
+
"(moorcheh-ollama container not started)"
|
|
81
|
+
)
|
|
82
|
+
print(f"Moorcheh API: http://localhost:{args.server_port}")
|
|
83
|
+
return 0
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def cmd_down(args: argparse.Namespace) -> int:
|
|
87
|
+
bundled_ollama: bool | None
|
|
88
|
+
if args.bundled_ollama and args.use_host_ollama:
|
|
89
|
+
raise ValueError("Use only one of --bundled-ollama or --use-host-ollama")
|
|
90
|
+
if args.bundled_ollama:
|
|
91
|
+
bundled_ollama = True
|
|
92
|
+
elif args.use_host_ollama:
|
|
93
|
+
bundled_ollama = False
|
|
94
|
+
else:
|
|
95
|
+
bundled_ollama = None
|
|
96
|
+
|
|
97
|
+
include_ollama: bool | None
|
|
98
|
+
if bundled_ollama is True:
|
|
99
|
+
include_ollama = True
|
|
100
|
+
elif bundled_ollama is False:
|
|
101
|
+
include_ollama = False
|
|
102
|
+
else:
|
|
103
|
+
include_ollama = None
|
|
104
|
+
|
|
105
|
+
result = down(include_ollama=include_ollama)
|
|
106
|
+
if result.stdout.strip():
|
|
107
|
+
print(result.stdout.strip())
|
|
108
|
+
if result.stderr.strip():
|
|
109
|
+
print(result.stderr.strip(), file=sys.stderr)
|
|
110
|
+
return 0
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def cmd_status(args: argparse.Namespace) -> int:
|
|
114
|
+
client = _api_client(args.base_url)
|
|
115
|
+
health = client.health()
|
|
116
|
+
items = health.get("items")
|
|
117
|
+
max_items = health.get("max_items")
|
|
118
|
+
remaining = health.get("remaining")
|
|
119
|
+
if items is not None and max_items is not None:
|
|
120
|
+
print(
|
|
121
|
+
f"items: {items} / {max_items} | remaining: {remaining} | model: {health.get('model')}"
|
|
122
|
+
)
|
|
123
|
+
_print_json(health)
|
|
124
|
+
return 0
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def cmd_namespace_create(args: argparse.Namespace) -> int:
|
|
128
|
+
client = _api_client(args.base_url)
|
|
129
|
+
payload: dict[str, Any] = {
|
|
130
|
+
"namespace_name": args.name,
|
|
131
|
+
"type": args.type,
|
|
132
|
+
}
|
|
133
|
+
if args.vector_dimension is not None:
|
|
134
|
+
payload["vector_dimension"] = args.vector_dimension
|
|
135
|
+
_print_json(client.create_namespace(payload))
|
|
136
|
+
return 0
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def cmd_namespace_list(args: argparse.Namespace) -> int:
|
|
140
|
+
client = _api_client(args.base_url)
|
|
141
|
+
_print_json(client.list_namespaces())
|
|
142
|
+
return 0
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def cmd_namespace_delete(args: argparse.Namespace) -> int:
|
|
146
|
+
client = _api_client(args.base_url)
|
|
147
|
+
_print_json(client.delete_namespace(args.namespace_name))
|
|
148
|
+
return 0
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def cmd_namespace_delete_job_status(args: argparse.Namespace) -> int:
|
|
152
|
+
client = _api_client(args.base_url)
|
|
153
|
+
_print_json(client.delete_namespace_job_status(args.namespace_name, args.job_id))
|
|
154
|
+
return 0
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def cmd_upload_documents(args: argparse.Namespace) -> int:
|
|
158
|
+
client = _api_client(args.base_url)
|
|
159
|
+
documents_path = Path(args.documents_file)
|
|
160
|
+
payload = json.loads(documents_path.read_text(encoding="utf-8"))
|
|
161
|
+
if not isinstance(payload, dict):
|
|
162
|
+
raise ValueError("documents file must contain a JSON object with a 'documents' array")
|
|
163
|
+
_print_json(client.upload_namespace_documents(args.namespace_name, payload))
|
|
164
|
+
return 0
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def cmd_upload_vectors(args: argparse.Namespace) -> int:
|
|
168
|
+
client = _api_client(args.base_url)
|
|
169
|
+
vectors_path = Path(args.vectors_file)
|
|
170
|
+
payload = json.loads(vectors_path.read_text(encoding="utf-8"))
|
|
171
|
+
if not isinstance(payload, dict):
|
|
172
|
+
raise ValueError("vectors file must contain a JSON object with a 'vectors' array")
|
|
173
|
+
_print_json(client.upload_namespace_vectors(args.namespace_name, payload))
|
|
174
|
+
return 0
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def cmd_upload_job_status(args: argparse.Namespace) -> int:
|
|
178
|
+
client = _api_client(args.base_url)
|
|
179
|
+
_print_json(client.upload_job_status(args.namespace_name, args.job_id))
|
|
180
|
+
return 0
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def cmd_items_get(args: argparse.Namespace) -> int:
|
|
184
|
+
client = _api_client(args.base_url)
|
|
185
|
+
ids = _parse_json_array(args.ids_json, "ids_json")
|
|
186
|
+
if any(not isinstance(item, str) for item in ids):
|
|
187
|
+
raise ValueError("ids_json must be an array of strings")
|
|
188
|
+
payload = {"ids": ids}
|
|
189
|
+
_print_json(client.get_namespace_items(args.namespace_name, payload))
|
|
190
|
+
return 0
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def cmd_items_delete(args: argparse.Namespace) -> int:
|
|
194
|
+
client = _api_client(args.base_url)
|
|
195
|
+
ids = _parse_json_array(args.ids_json, "ids_json")
|
|
196
|
+
if any(not isinstance(item, str) for item in ids):
|
|
197
|
+
raise ValueError("ids_json must be an array of strings")
|
|
198
|
+
payload = {"ids": ids}
|
|
199
|
+
_print_json(client.delete_namespace_items(args.namespace_name, payload))
|
|
200
|
+
return 0
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def cmd_search(args: argparse.Namespace) -> int:
|
|
204
|
+
metadata = _parse_json(args.metadata_json, "metadata_json")
|
|
205
|
+
client = _api_client(args.base_url)
|
|
206
|
+
query: Any
|
|
207
|
+
if args.query_vector_json is not None:
|
|
208
|
+
query = json.loads(args.query_vector_json)
|
|
209
|
+
if not isinstance(query, list):
|
|
210
|
+
raise ValueError("query_vector_json must be a JSON array")
|
|
211
|
+
else:
|
|
212
|
+
if not args.query:
|
|
213
|
+
raise ValueError("query is required when query_vector_json is not provided")
|
|
214
|
+
query = args.query
|
|
215
|
+
namespaces = [n.strip() for n in args.namespaces.split(",") if n.strip()]
|
|
216
|
+
payload: dict[str, Any] = {
|
|
217
|
+
"query": query,
|
|
218
|
+
"top_k": args.top_k,
|
|
219
|
+
"threshold": args.threshold,
|
|
220
|
+
"metadata": metadata,
|
|
221
|
+
"namespaces": namespaces,
|
|
222
|
+
}
|
|
223
|
+
_print_json(
|
|
224
|
+
client.search(payload)
|
|
225
|
+
)
|
|
226
|
+
return 0
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
230
|
+
parser = argparse.ArgumentParser(prog="moorcheh", description="Moorcheh on-prem client and runtime CLI.")
|
|
231
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
232
|
+
|
|
233
|
+
p_up = sub.add_parser("up", help="Start server + ollama containers.")
|
|
234
|
+
p_up.add_argument("--server-image", default=DEFAULT_SERVER_IMAGE)
|
|
235
|
+
p_up.add_argument("--ollama-image", default=DEFAULT_OLLAMA_IMAGE)
|
|
236
|
+
p_up.add_argument("--server-port", type=int, default=8080)
|
|
237
|
+
p_up.add_argument(
|
|
238
|
+
"--ollama-port",
|
|
239
|
+
type=int,
|
|
240
|
+
default=11434,
|
|
241
|
+
help="Host port to probe for existing Ollama, or to publish bundled Ollama on.",
|
|
242
|
+
)
|
|
243
|
+
p_up.add_argument(
|
|
244
|
+
"--ollama-host",
|
|
245
|
+
default="127.0.0.1",
|
|
246
|
+
help="Host to probe for existing Ollama (default: 127.0.0.1).",
|
|
247
|
+
)
|
|
248
|
+
p_up.add_argument(
|
|
249
|
+
"--bundled-ollama",
|
|
250
|
+
action="store_true",
|
|
251
|
+
help="Always start the moorcheh-ollama Docker container.",
|
|
252
|
+
)
|
|
253
|
+
p_up.add_argument(
|
|
254
|
+
"--use-host-ollama",
|
|
255
|
+
action="store_true",
|
|
256
|
+
help="Never start moorcheh-ollama; server uses Ollama on the host (port 11434).",
|
|
257
|
+
)
|
|
258
|
+
p_up.add_argument("--ollama-model", default=DEFAULT_OLLAMA_MODEL)
|
|
259
|
+
p_up.set_defaults(func=cmd_up)
|
|
260
|
+
|
|
261
|
+
p_down = sub.add_parser("down", help="Stop and remove runtime containers.")
|
|
262
|
+
p_down.add_argument("--bundled-ollama", action="store_true")
|
|
263
|
+
p_down.add_argument("--use-host-ollama", action="store_true")
|
|
264
|
+
p_down.set_defaults(func=cmd_down)
|
|
265
|
+
|
|
266
|
+
p_status = sub.add_parser("status", help="Check server health endpoint.")
|
|
267
|
+
p_status.add_argument("--base-url", default="http://localhost:8080")
|
|
268
|
+
p_status.set_defaults(func=cmd_status)
|
|
269
|
+
|
|
270
|
+
p_namespace_create = sub.add_parser("namespace-create", help="Create a namespace.")
|
|
271
|
+
p_namespace_create.add_argument("--base-url", default="http://localhost:8080")
|
|
272
|
+
p_namespace_create.add_argument("--name", required=True)
|
|
273
|
+
p_namespace_create.add_argument("--type", choices=["text", "vector"], required=True)
|
|
274
|
+
p_namespace_create.add_argument("--vector-dimension", type=int)
|
|
275
|
+
p_namespace_create.set_defaults(func=cmd_namespace_create)
|
|
276
|
+
|
|
277
|
+
p_namespace_list = sub.add_parser("namespace-list", help="List namespaces.")
|
|
278
|
+
p_namespace_list.add_argument("--base-url", default="http://localhost:8080")
|
|
279
|
+
p_namespace_list.set_defaults(func=cmd_namespace_list)
|
|
280
|
+
|
|
281
|
+
p_namespace_delete = sub.add_parser("namespace-delete", help="Call DELETE /namespaces/{namespace_name}.")
|
|
282
|
+
p_namespace_delete.add_argument("--base-url", default="http://localhost:8080")
|
|
283
|
+
p_namespace_delete.add_argument("--namespace-name", required=True)
|
|
284
|
+
p_namespace_delete.set_defaults(func=cmd_namespace_delete)
|
|
285
|
+
|
|
286
|
+
p_namespace_delete_job_status = sub.add_parser(
|
|
287
|
+
"namespace-delete-job-status",
|
|
288
|
+
help="Call GET /namespaces/{namespace_name}/delete-jobs/{job_id}.",
|
|
289
|
+
)
|
|
290
|
+
p_namespace_delete_job_status.add_argument("--base-url", default="http://localhost:8080")
|
|
291
|
+
p_namespace_delete_job_status.add_argument("--namespace-name", required=True)
|
|
292
|
+
p_namespace_delete_job_status.add_argument("--job-id", required=True)
|
|
293
|
+
p_namespace_delete_job_status.set_defaults(func=cmd_namespace_delete_job_status)
|
|
294
|
+
|
|
295
|
+
p_upload_documents = sub.add_parser("upload-documents", help="Call POST /namespaces/{namespace_name}/documents.")
|
|
296
|
+
p_upload_documents.add_argument("--base-url", default="http://localhost:8080")
|
|
297
|
+
p_upload_documents.add_argument("--namespace-name", required=True)
|
|
298
|
+
p_upload_documents.add_argument("--documents-file", required=True, help="Path to JSON body with {'documents': [...]} payload.")
|
|
299
|
+
p_upload_documents.set_defaults(func=cmd_upload_documents)
|
|
300
|
+
|
|
301
|
+
p_upload_vectors = sub.add_parser("upload-vectors", help="Call POST /namespaces/{namespace_name}/vectors.")
|
|
302
|
+
p_upload_vectors.add_argument("--base-url", default="http://localhost:8080")
|
|
303
|
+
p_upload_vectors.add_argument("--namespace-name", required=True)
|
|
304
|
+
p_upload_vectors.add_argument("--vectors-file", required=True, help="Path to JSON body with {'vectors': [...]} payload.")
|
|
305
|
+
p_upload_vectors.set_defaults(func=cmd_upload_vectors)
|
|
306
|
+
|
|
307
|
+
p_upload_job_status = sub.add_parser(
|
|
308
|
+
"upload-job-status",
|
|
309
|
+
help="Poll document or vector upload job (GET .../upload-jobs/{job_id}).",
|
|
310
|
+
)
|
|
311
|
+
p_upload_job_status.add_argument("--base-url", default="http://localhost:8080")
|
|
312
|
+
p_upload_job_status.add_argument("--namespace-name", required=True)
|
|
313
|
+
p_upload_job_status.add_argument("--job-id", required=True)
|
|
314
|
+
p_upload_job_status.set_defaults(func=cmd_upload_job_status)
|
|
315
|
+
|
|
316
|
+
p_items_get = sub.add_parser("items-get", help="Call POST /namespaces/{namespace_name}/items/get.")
|
|
317
|
+
p_items_get.add_argument("--base-url", default="http://localhost:8080")
|
|
318
|
+
p_items_get.add_argument("--namespace-name", required=True)
|
|
319
|
+
p_items_get.add_argument("--ids-json", required=True, help='JSON array string, e.g. ["id1","id2"].')
|
|
320
|
+
p_items_get.set_defaults(func=cmd_items_get)
|
|
321
|
+
|
|
322
|
+
p_items_delete = sub.add_parser("items-delete", help="Call POST /namespaces/{namespace_name}/items/delete.")
|
|
323
|
+
p_items_delete.add_argument("--base-url", default="http://localhost:8080")
|
|
324
|
+
p_items_delete.add_argument("--namespace-name", required=True)
|
|
325
|
+
p_items_delete.add_argument("--ids-json", required=True, help='JSON array string, e.g. ["id1","id2"].')
|
|
326
|
+
p_items_delete.set_defaults(func=cmd_items_delete)
|
|
327
|
+
|
|
328
|
+
p_search = sub.add_parser("search", help="Call /search endpoint.")
|
|
329
|
+
p_search.add_argument("--base-url", default="http://localhost:8080")
|
|
330
|
+
p_search.add_argument("--query", help="Text query.")
|
|
331
|
+
p_search.add_argument("--query-vector-json", help="JSON array string vector query.")
|
|
332
|
+
p_search.add_argument("--namespaces", default="", help="Comma-separated namespace list.")
|
|
333
|
+
p_search.add_argument("--top-k", type=int, default=5)
|
|
334
|
+
p_search.add_argument("--threshold", type=float, default=0.0)
|
|
335
|
+
p_search.add_argument("--metadata-json", default="{}")
|
|
336
|
+
p_search.set_defaults(func=cmd_search)
|
|
337
|
+
|
|
338
|
+
return parser
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def main() -> None:
|
|
342
|
+
parser = build_parser()
|
|
343
|
+
args = parser.parse_args()
|
|
344
|
+
try:
|
|
345
|
+
code = args.func(args)
|
|
346
|
+
except ComposeCommandError as exc:
|
|
347
|
+
if exc.stdout.strip():
|
|
348
|
+
print(exc.stdout.strip())
|
|
349
|
+
if exc.stderr.strip():
|
|
350
|
+
print(exc.stderr.strip(), file=sys.stderr)
|
|
351
|
+
print(f"Error: docker compose failed (exit {exc.returncode})", file=sys.stderr)
|
|
352
|
+
raise SystemExit(1) from exc
|
|
353
|
+
except MoorchehApiError as exc:
|
|
354
|
+
if exc.body:
|
|
355
|
+
_print_json(exc.body)
|
|
356
|
+
print(f"Error ({exc.status_code}): {exc}", file=sys.stderr)
|
|
357
|
+
raise SystemExit(1) from exc
|
|
358
|
+
except Exception as exc: # pragma: no cover - surfaced to CLI user
|
|
359
|
+
print(f"Error: {exc}", file=sys.stderr)
|
|
360
|
+
raise SystemExit(1) from exc
|
|
361
|
+
raise SystemExit(code)
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
if __name__ == "__main__":
|
|
365
|
+
main()
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
services:
|
|
2
|
+
ollama:
|
|
3
|
+
profiles: ["bundled-ollama"]
|
|
4
|
+
image: ${OLLAMA_IMAGE:-ollama/ollama:latest}
|
|
5
|
+
container_name: moorcheh-ollama
|
|
6
|
+
ports:
|
|
7
|
+
- "${OLLAMA_PORT:-11434}:11434"
|
|
8
|
+
volumes:
|
|
9
|
+
- ollama_data:/root/.ollama
|
|
10
|
+
|
|
11
|
+
server:
|
|
12
|
+
image: ${MOORCHEH_SERVER_IMAGE:-moorcheh/server:latest}
|
|
13
|
+
container_name: moorcheh-onprem-server
|
|
14
|
+
depends_on:
|
|
15
|
+
ollama:
|
|
16
|
+
condition: service_started
|
|
17
|
+
required: false
|
|
18
|
+
ports:
|
|
19
|
+
- "${SERVER_PORT:-8080}:8080"
|
|
20
|
+
extra_hosts:
|
|
21
|
+
- "host.docker.internal:host-gateway"
|
|
22
|
+
environment:
|
|
23
|
+
SERVER_HOST: 0.0.0.0
|
|
24
|
+
SERVER_PORT: 8080
|
|
25
|
+
OLLAMA_URL: ${OLLAMA_URL:-http://ollama:11434}
|
|
26
|
+
OLLAMA_MODEL: ${OLLAMA_MODEL:-nomic-embed-text}
|
|
27
|
+
DATA_FILE: /app/data/moorcheh_data_store.json
|
|
28
|
+
NAMESPACE_REGISTRY_FILE: /app/data/namespace_registry.json
|
|
29
|
+
volumes:
|
|
30
|
+
- ${MOORCHEH_DATA_DIR}:/app/data
|
|
31
|
+
|
|
32
|
+
volumes:
|
|
33
|
+
ollama_data:
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import subprocess
|
|
5
|
+
import urllib.error
|
|
6
|
+
import urllib.request
|
|
7
|
+
from importlib import resources
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
DEFAULT_SERVER_IMAGE = "moorcheh/server:latest"
|
|
12
|
+
DEFAULT_OLLAMA_IMAGE = "ollama/ollama:latest"
|
|
13
|
+
DEFAULT_OLLAMA_MODEL = "nomic-embed-text"
|
|
14
|
+
DEFAULT_OLLAMA_HOST = "127.0.0.1"
|
|
15
|
+
DEFAULT_OLLAMA_PORT = 11434
|
|
16
|
+
HOST_OLLAMA_URL = "http://host.docker.internal:11434"
|
|
17
|
+
|
|
18
|
+
# Must match container_name in compose/docker-compose.yml
|
|
19
|
+
COMPOSE_CONTAINER_NAMES = ("moorcheh-ollama", "moorcheh-onprem-server")
|
|
20
|
+
|
|
21
|
+
MOORCHEH_DATA_DIR_ENV = "MOORCHEH_DATA_DIR"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def default_data_dir() -> Path:
|
|
25
|
+
"""Per-user data directory: ~/.moorcheh/data (e.g. C:\\Users\\you\\.moorcheh\\data on Windows)."""
|
|
26
|
+
return Path.home() / ".moorcheh" / "data"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def ensure_data_dir() -> Path:
|
|
30
|
+
"""Create and return ~/.moorcheh/data for the current user."""
|
|
31
|
+
path = default_data_dir().resolve()
|
|
32
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
33
|
+
return path
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def docker_bind_path(path: Path) -> str:
|
|
37
|
+
"""Absolute host path for docker compose bind mounts (forward slashes on Windows)."""
|
|
38
|
+
return path.resolve().as_posix()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class ComposeCommandError(RuntimeError):
|
|
42
|
+
"""docker compose failed; includes captured stdout/stderr."""
|
|
43
|
+
|
|
44
|
+
def __init__(self, command: list[str], returncode: int, stdout: str, stderr: str) -> None:
|
|
45
|
+
self.command = command
|
|
46
|
+
self.returncode = returncode
|
|
47
|
+
self.stdout = stdout
|
|
48
|
+
self.stderr = stderr
|
|
49
|
+
message = stderr.strip() or stdout.strip() or f"exit code {returncode}"
|
|
50
|
+
super().__init__(message)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def compose_file_path() -> str:
|
|
54
|
+
return str(resources.files("moorcheh").joinpath("compose/docker-compose.yml"))
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def ollama_is_reachable(host: str = DEFAULT_OLLAMA_HOST, port: int = DEFAULT_OLLAMA_PORT, timeout: float = 2.0) -> bool:
|
|
58
|
+
"""Return True if an Ollama HTTP server responds on host:port."""
|
|
59
|
+
url = f"http://{host}:{port}/"
|
|
60
|
+
try:
|
|
61
|
+
with urllib.request.urlopen(url, timeout=timeout) as response:
|
|
62
|
+
return 200 <= response.status < 300
|
|
63
|
+
except (urllib.error.URLError, TimeoutError, ValueError):
|
|
64
|
+
return False
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def should_use_bundled_ollama(
|
|
68
|
+
*,
|
|
69
|
+
bundled_ollama: bool | None,
|
|
70
|
+
ollama_host: str,
|
|
71
|
+
ollama_port: int,
|
|
72
|
+
) -> bool:
|
|
73
|
+
"""
|
|
74
|
+
bundled_ollama: True = always start container; False = always use host;
|
|
75
|
+
None = auto-detect (use host Ollama when already reachable).
|
|
76
|
+
"""
|
|
77
|
+
if bundled_ollama is True:
|
|
78
|
+
return True
|
|
79
|
+
if bundled_ollama is False:
|
|
80
|
+
return False
|
|
81
|
+
return not ollama_is_reachable(ollama_host, ollama_port)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def run_compose(command: list[str], env: dict[str, str] | None = None) -> subprocess.CompletedProcess[str]:
|
|
85
|
+
full_env = os.environ.copy()
|
|
86
|
+
if env:
|
|
87
|
+
full_env.update(env)
|
|
88
|
+
|
|
89
|
+
cmd = ["docker", "compose", "-f", compose_file_path(), *command]
|
|
90
|
+
result = subprocess.run(
|
|
91
|
+
cmd,
|
|
92
|
+
text=True,
|
|
93
|
+
capture_output=True,
|
|
94
|
+
env=full_env,
|
|
95
|
+
)
|
|
96
|
+
if result.returncode != 0:
|
|
97
|
+
raise ComposeCommandError(cmd, result.returncode, result.stdout, result.stderr)
|
|
98
|
+
return result
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def remove_stale_compose_containers(*, include_ollama: bool) -> None:
|
|
102
|
+
"""Remove leftover containers that block fixed container_name in compose."""
|
|
103
|
+
names = list(COMPOSE_CONTAINER_NAMES) if include_ollama else ("moorcheh-onprem-server",)
|
|
104
|
+
for name in names:
|
|
105
|
+
subprocess.run(
|
|
106
|
+
["docker", "rm", "-f", name],
|
|
107
|
+
capture_output=True,
|
|
108
|
+
text=True,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def up(
|
|
113
|
+
server_image: str,
|
|
114
|
+
ollama_image: str,
|
|
115
|
+
server_port: int,
|
|
116
|
+
ollama_port: int,
|
|
117
|
+
ollama_model: str,
|
|
118
|
+
*,
|
|
119
|
+
bundled_ollama: bool | None = None,
|
|
120
|
+
ollama_host: str = DEFAULT_OLLAMA_HOST,
|
|
121
|
+
) -> tuple[subprocess.CompletedProcess[str], bool, Path]:
|
|
122
|
+
"""
|
|
123
|
+
Start the Moorcheh stack. Returns (compose result, whether bundled Ollama was started, data_dir).
|
|
124
|
+
|
|
125
|
+
When host Ollama is already running on ollama_host:ollama_port, only the server
|
|
126
|
+
container is started and OLLAMA_URL points at host.docker.internal:11434.
|
|
127
|
+
"""
|
|
128
|
+
use_bundled = should_use_bundled_ollama(
|
|
129
|
+
bundled_ollama=bundled_ollama,
|
|
130
|
+
ollama_host=ollama_host,
|
|
131
|
+
ollama_port=ollama_port,
|
|
132
|
+
)
|
|
133
|
+
remove_stale_compose_containers(include_ollama=use_bundled)
|
|
134
|
+
|
|
135
|
+
resolved_data_dir = ensure_data_dir()
|
|
136
|
+
base_env = {
|
|
137
|
+
"MOORCHEH_SERVER_IMAGE": server_image,
|
|
138
|
+
"OLLAMA_IMAGE": ollama_image,
|
|
139
|
+
"SERVER_PORT": str(server_port),
|
|
140
|
+
"OLLAMA_MODEL": ollama_model,
|
|
141
|
+
MOORCHEH_DATA_DIR_ENV: docker_bind_path(resolved_data_dir),
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if use_bundled:
|
|
145
|
+
result = run_compose(
|
|
146
|
+
["--profile", "bundled-ollama", "up", "-d"],
|
|
147
|
+
env={
|
|
148
|
+
**base_env,
|
|
149
|
+
"OLLAMA_PORT": str(ollama_port),
|
|
150
|
+
"OLLAMA_URL": "http://ollama:11434",
|
|
151
|
+
},
|
|
152
|
+
)
|
|
153
|
+
else:
|
|
154
|
+
result = run_compose(
|
|
155
|
+
["up", "-d", "server"],
|
|
156
|
+
env={
|
|
157
|
+
**base_env,
|
|
158
|
+
"OLLAMA_URL": HOST_OLLAMA_URL,
|
|
159
|
+
},
|
|
160
|
+
)
|
|
161
|
+
return result, use_bundled, resolved_data_dir
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def down(*, include_ollama: bool | None = None) -> subprocess.CompletedProcess[str]:
|
|
165
|
+
"""
|
|
166
|
+
Stop compose services. When include_ollama is False, only stops the server
|
|
167
|
+
(leaves a bundled ollama container running if it was started separately).
|
|
168
|
+
None = stop all services defined in the compose file (including profile services
|
|
169
|
+
that were started).
|
|
170
|
+
"""
|
|
171
|
+
env = {MOORCHEH_DATA_DIR_ENV: docker_bind_path(ensure_data_dir())}
|
|
172
|
+
if include_ollama is False:
|
|
173
|
+
return run_compose(["stop", "server"], env=env)
|
|
174
|
+
return run_compose(["--profile", "bundled-ollama", "down"], env=env)
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: moorcheh-client
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Moorcheh client and runtime launcher for on-prem server containers.
|
|
5
|
+
Requires-Python: >=3.10
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: requests>=2.31.0
|
|
8
|
+
Provides-Extra: web
|
|
9
|
+
Requires-Dist: Flask>=3.0.0; extra == "web"
|
|
10
|
+
|
|
11
|
+
# moorcheh-client (Python)
|
|
12
|
+
|
|
13
|
+
This directory publishes the **`moorcheh-client`** distribution on PyPI. The importable module is still `moorcheh` (`from moorcheh import MoorchehApiClient`).
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
- `pip install moorcheh-client` (from this directory: `pip install .`)
|
|
18
|
+
|
|
19
|
+
Default server image: `moorcheh/server:latest` (override with `moorcheh up --server-image …` or `MOORCHEH_SERVER_IMAGE` to pin a specific release).
|
|
20
|
+
|
|
21
|
+
## Data directory
|
|
22
|
+
|
|
23
|
+
`moorcheh up` stores your vectors and documents at:
|
|
24
|
+
|
|
25
|
+
- `~/.moorcheh/data` (e.g. `C:\Users\<you>\.moorcheh\data` on Windows)
|
|
26
|
+
- `moorcheh_data_store.json`, `namespace_registry.json`
|
|
27
|
+
|
|
28
|
+
`moorcheh down` stops containers but **keeps** `~/.moorcheh`. Back up that folder to save everything.
|
|
29
|
+
|
|
30
|
+
## CLI Commands
|
|
31
|
+
|
|
32
|
+
- `moorcheh up` — start Moorcheh; uses host Ollama on `127.0.0.1:11434` when already running
|
|
33
|
+
- `moorcheh down` — stop Moorcheh containers (does not stop host Ollama)
|
|
34
|
+
- `moorcheh status` — `GET /health` (items, max_items, remaining)
|
|
35
|
+
- `moorcheh namespace-create` — create a text or vector namespace
|
|
36
|
+
- `moorcheh namespace-list` — list namespaces and per-namespace `item_count`
|
|
37
|
+
- `moorcheh upload-documents` — `POST /namespaces/{namespace_name}/documents` (async job)
|
|
38
|
+
- `moorcheh upload-vectors` — `POST /namespaces/{namespace_name}/vectors` (async job)
|
|
39
|
+
- `moorcheh upload-job-status` — poll upload job (documents or vectors)
|
|
40
|
+
- `moorcheh items-get` — get items by id within a namespace
|
|
41
|
+
- `moorcheh items-delete` — delete items by id within a namespace
|
|
42
|
+
- `moorcheh namespace-delete` — delete a namespace and all its items
|
|
43
|
+
- `moorcheh search` — semantic search
|
|
44
|
+
|
|
45
|
+
## Global item limit (100k)
|
|
46
|
+
|
|
47
|
+
Moorcheh stores at most **100,000 items total** across all namespaces (text + vectors).
|
|
48
|
+
|
|
49
|
+
- Check quota: `moorcheh status` → `items`, `max_items`, `remaining`
|
|
50
|
+
- Uploads that would add **new** ids over the cap return **409** (entire batch rejected)
|
|
51
|
+
- Re-uploading an existing id in the same namespace is an update and does not use extra quota
|
|
52
|
+
- Deleting items or a namespace frees quota immediately
|
|
53
|
+
|
|
54
|
+
Item ids are **unique per namespace** (the same id string may exist in different namespaces).
|
|
55
|
+
|
|
56
|
+
## Examples
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
moorcheh up
|
|
60
|
+
moorcheh status
|
|
61
|
+
moorcheh namespace-create --name docs --type text
|
|
62
|
+
moorcheh namespace-create --name products_vec --type vector --vector-dimension 768
|
|
63
|
+
moorcheh upload-documents --namespace-name docs --documents-file docs-upload.json
|
|
64
|
+
moorcheh upload-vectors --namespace-name products_vec --vectors-file vectors-upload.json
|
|
65
|
+
moorcheh upload-job-status --namespace-name docs --job-id job-abc123
|
|
66
|
+
moorcheh items-get --namespace-name docs --ids-json "[\"doc-1\"]"
|
|
67
|
+
moorcheh items-delete --namespace-name docs --ids-json "[\"doc-1\"]"
|
|
68
|
+
moorcheh search --query "on prem retrieval" --namespaces docs --top-k 5
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
Documents upload payload (`docs-upload.json`):
|
|
72
|
+
|
|
73
|
+
```json
|
|
74
|
+
{
|
|
75
|
+
"documents": [
|
|
76
|
+
{
|
|
77
|
+
"id": "doc-1",
|
|
78
|
+
"text": "Moorcheh on-prem retrieval test",
|
|
79
|
+
"team": "ai"
|
|
80
|
+
}
|
|
81
|
+
]
|
|
82
|
+
}
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
## Python API
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
from moorcheh import MoorchehApiClient, MoorchehApiError
|
|
89
|
+
|
|
90
|
+
client = MoorchehApiClient("http://localhost:8080")
|
|
91
|
+
health = client.health()
|
|
92
|
+
print(health["items"], health["remaining"])
|
|
93
|
+
|
|
94
|
+
try:
|
|
95
|
+
client.upload_namespace_vectors("products_vec", {"vectors": [...]})
|
|
96
|
+
except MoorchehApiError as e:
|
|
97
|
+
if e.is_item_limit_exceeded:
|
|
98
|
+
print(e.body) # items, max_items, requested_new
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## Optional Flask Demo App
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
pip install .[web]
|
|
105
|
+
python client/app.py
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
## Ollama: host vs Docker
|
|
109
|
+
|
|
110
|
+
By default, `moorcheh up` uses Ollama on `http://127.0.0.1:11434` when it is already running (typical on Windows/macOS). It does **not** start a second Ollama container in that case.
|
|
111
|
+
|
|
112
|
+
```bash
|
|
113
|
+
moorcheh up
|
|
114
|
+
# Using Ollama already running at http://127.0.0.1:11434 (moorcheh-ollama container not started)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Force behavior:
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
moorcheh up --use-host-ollama # never start moorcheh-ollama
|
|
121
|
+
moorcheh up --bundled-ollama # always start moorcheh-ollama container
|
|
122
|
+
moorcheh up --bundled-ollama --ollama-port 11435 # bundled Ollama on another host port
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Port conflict fallback
|
|
126
|
+
|
|
127
|
+
If **8080** is busy:
|
|
128
|
+
|
|
129
|
+
```bash
|
|
130
|
+
moorcheh up --server-port 8081
|
|
131
|
+
moorcheh status --base-url http://localhost:8081
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`moorcheh up` removes stale `moorcheh-onprem-server` (and `moorcheh-ollama` only when starting bundled Ollama). Data persists under `~/.moorcheh/data` across restarts.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
moorcheh/__init__.py
|
|
4
|
+
moorcheh/__main__.py
|
|
5
|
+
moorcheh/api.py
|
|
6
|
+
moorcheh/cli.py
|
|
7
|
+
moorcheh/docker_runtime.py
|
|
8
|
+
moorcheh/compose/docker-compose.yml
|
|
9
|
+
moorcheh_client.egg-info/PKG-INFO
|
|
10
|
+
moorcheh_client.egg-info/SOURCES.txt
|
|
11
|
+
moorcheh_client.egg-info/dependency_links.txt
|
|
12
|
+
moorcheh_client.egg-info/entry_points.txt
|
|
13
|
+
moorcheh_client.egg-info/requires.txt
|
|
14
|
+
moorcheh_client.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
moorcheh
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "moorcheh-client"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Moorcheh client and runtime launcher for on-prem server containers."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
dependencies = [
|
|
12
|
+
"requests>=2.31.0",
|
|
13
|
+
]
|
|
14
|
+
|
|
15
|
+
[project.optional-dependencies]
|
|
16
|
+
web = [
|
|
17
|
+
"Flask>=3.0.0",
|
|
18
|
+
]
|
|
19
|
+
|
|
20
|
+
[project.scripts]
|
|
21
|
+
moorcheh = "moorcheh.cli:main"
|
|
22
|
+
|
|
23
|
+
[tool.setuptools]
|
|
24
|
+
include-package-data = true
|
|
25
|
+
|
|
26
|
+
[tool.setuptools.packages.find]
|
|
27
|
+
where = ["."]
|
|
28
|
+
include = ["moorcheh*"]
|
|
29
|
+
|
|
30
|
+
[tool.setuptools.package-data]
|
|
31
|
+
moorcheh = ["compose/docker-compose.yml"]
|