oshepherd 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of oshepherd might be problematic. Click here for more details.
- oshepherd-0.0.1/LICENSE +22 -0
- oshepherd-0.0.1/PKG-INFO +104 -0
- oshepherd-0.0.1/README.md +89 -0
- oshepherd-0.0.1/oshepherd/__init__.py +0 -0
- oshepherd-0.0.1/oshepherd/api/__init__.py +0 -0
- oshepherd-0.0.1/oshepherd/api/app.py +30 -0
- oshepherd-0.0.1/oshepherd/api/config.py +12 -0
- oshepherd-0.0.1/oshepherd/api/generate/__init__.py +0 -0
- oshepherd-0.0.1/oshepherd/api/generate/models.py +30 -0
- oshepherd-0.0.1/oshepherd/api/generate/routes.py +38 -0
- oshepherd-0.0.1/oshepherd/api/utils.py +13 -0
- oshepherd-0.0.1/oshepherd/cli.py +47 -0
- oshepherd-0.0.1/oshepherd/lib.py +23 -0
- oshepherd-0.0.1/oshepherd/worker/__init__.py +0 -0
- oshepherd-0.0.1/oshepherd/worker/app.py +40 -0
- oshepherd-0.0.1/oshepherd/worker/config.py +11 -0
- oshepherd-0.0.1/oshepherd/worker/tasks.py +23 -0
- oshepherd-0.0.1/oshepherd.egg-info/PKG-INFO +104 -0
- oshepherd-0.0.1/oshepherd.egg-info/SOURCES.txt +24 -0
- oshepherd-0.0.1/oshepherd.egg-info/dependency_links.txt +1 -0
- oshepherd-0.0.1/oshepherd.egg-info/top_level.txt +2 -0
- oshepherd-0.0.1/pyproject.toml +18 -0
- oshepherd-0.0.1/setup.cfg +4 -0
- oshepherd-0.0.1/setup.py +27 -0
- oshepherd-0.0.1/tests/__init__.py +0 -0
- oshepherd-0.0.1/tests/test_e2e_api.py +33 -0
oshepherd-0.0.1/LICENSE
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
(The MIT License)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024 mnemonica.ai <mnemonica.ai>
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
|
6
|
+
a copy of this software and associated documentation files (the
|
|
7
|
+
'Software'), to deal in the Software without restriction, including
|
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
|
11
|
+
the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be
|
|
14
|
+
included in all copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
|
19
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
|
20
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
|
21
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
|
22
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
oshepherd-0.0.1/PKG-INFO
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: oshepherd
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: The Oshepherd guiding the Ollama(s) inference orchestration.
|
|
5
|
+
Author: mnemonica.ai
|
|
6
|
+
Author-email: "mnemonica.ai" <info@mnemonica.ai>
|
|
7
|
+
Project-URL: Homepage, https://github.com/mnemonica-ai/oshepherd
|
|
8
|
+
Project-URL: Issues, https://github.com/mnemonica-ai/oshepherd/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.8
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
|
|
16
|
+
# oshepherd
|
|
17
|
+
|
|
18
|
+
> The Oshepherd guiding the Ollama(s) inference orchestration.
|
|
19
|
+
|
|
20
|
+
A centralized [Flask](https://flask.palletsprojects.com) API service, using [Celery](https://docs.celeryq.dev) ([RabbitMQ](https://www.rabbitmq.com) + [Redis](https://redis.com)) to orchestrate multiple [Ollama](https://ollama.com) workers.
|
|
21
|
+
|
|
22
|
+
### Install
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
pip install oshepherd
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
### Usage
|
|
29
|
+
|
|
30
|
+
1. Setup RabbitMQ and Redis:
|
|
31
|
+
|
|
32
|
+
Create instances for free for both:
|
|
33
|
+
* [cloudamqp.com](https://www.cloudamqp.com)
|
|
34
|
+
* [redislabs.com](https://app.redislabs.com)
|
|
35
|
+
|
|
36
|
+
2. Setup Flask API Server:
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
# define configuration env file
|
|
40
|
+
# use credentials for redis and rabbitmq
|
|
41
|
+
cp .api.env.template .api.env
|
|
42
|
+
|
|
43
|
+
# start api
|
|
44
|
+
oshepherd start-api --env-file .api.env
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
3. Setup Celery/Ollama Worker(s):
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
# install ollama https://ollama.com/download
|
|
51
|
+
ollama run mistral
|
|
52
|
+
|
|
53
|
+
# define configuration env file
|
|
54
|
+
# use credentials for redis and rabbitmq
|
|
55
|
+
cp .worker.env.template .worker.env
|
|
56
|
+
|
|
57
|
+
# start worker
|
|
58
|
+
oshepherd start-worker --env-file .worker.env
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### Words of advice 🚨
|
|
62
|
+
|
|
63
|
+
This package is in alpha, its architecture and api might change in the near future. Currently this is getting tested in a closed environment by real users, but haven't been audited, nor tested thorugly. Use it at your own risk.
|
|
64
|
+
|
|
65
|
+
### Disclaimer on Support
|
|
66
|
+
|
|
67
|
+
As this is an alpha version, support and responses might be limited. We'll do our best to address questions and issues as quickly as possible.
|
|
68
|
+
|
|
69
|
+
### Contribution Guidelines
|
|
70
|
+
|
|
71
|
+
We welcome contributions! If you find a bug or have suggestions for improvements, please open an issue or submit a pull request.
|
|
72
|
+
|
|
73
|
+
##### Conda Support
|
|
74
|
+
|
|
75
|
+
To run and build locally you can use [conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html):
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
conda create -n oshepherd python=3.8
|
|
79
|
+
conda activate oshepherd
|
|
80
|
+
pip install -r requirements.txt
|
|
81
|
+
|
|
82
|
+
# install oshepherd
|
|
83
|
+
pip install -e .
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
##### Tests
|
|
87
|
+
|
|
88
|
+
Follow usage instructions to start api server and celery worker using a local ollama, and then run the tests:
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
pytest -s tests/
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### Reporting Issues
|
|
95
|
+
|
|
96
|
+
Please report any issues you encounter on the GitHub issues page. Before creating a new issue, take a moment to search through the existing issues to avoid duplicates.
|
|
97
|
+
|
|
98
|
+
### Author
|
|
99
|
+
|
|
100
|
+
Currently, [mnemonica.ai](mnemonica.ai) is sponsoring the development of this tool.
|
|
101
|
+
|
|
102
|
+
### License
|
|
103
|
+
|
|
104
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# oshepherd
|
|
2
|
+
|
|
3
|
+
> The Oshepherd guiding the Ollama(s) inference orchestration.
|
|
4
|
+
|
|
5
|
+
A centralized [Flask](https://flask.palletsprojects.com) API service, using [Celery](https://docs.celeryq.dev) ([RabbitMQ](https://www.rabbitmq.com) + [Redis](https://redis.com)) to orchestrate multiple [Ollama](https://ollama.com) workers.
|
|
6
|
+
|
|
7
|
+
### Install
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
pip install oshepherd
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
### Usage
|
|
14
|
+
|
|
15
|
+
1. Setup RabbitMQ and Redis:
|
|
16
|
+
|
|
17
|
+
Create instances for free for both:
|
|
18
|
+
* [cloudamqp.com](https://www.cloudamqp.com)
|
|
19
|
+
* [redislabs.com](https://app.redislabs.com)
|
|
20
|
+
|
|
21
|
+
2. Setup Flask API Server:
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
# define configuration env file
|
|
25
|
+
# use credentials for redis and rabbitmq
|
|
26
|
+
cp .api.env.template .api.env
|
|
27
|
+
|
|
28
|
+
# start api
|
|
29
|
+
oshepherd start-api --env-file .api.env
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
3. Setup Celery/Ollama Worker(s):
|
|
33
|
+
|
|
34
|
+
```
|
|
35
|
+
# install ollama https://ollama.com/download
|
|
36
|
+
ollama run mistral
|
|
37
|
+
|
|
38
|
+
# define configuration env file
|
|
39
|
+
# use credentials for redis and rabbitmq
|
|
40
|
+
cp .worker.env.template .worker.env
|
|
41
|
+
|
|
42
|
+
# start worker
|
|
43
|
+
oshepherd start-worker --env-file .worker.env
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
### Words of advice 🚨
|
|
47
|
+
|
|
48
|
+
This package is in alpha, its architecture and api might change in the near future. Currently this is getting tested in a closed environment by real users, but haven't been audited, nor tested thorugly. Use it at your own risk.
|
|
49
|
+
|
|
50
|
+
### Disclaimer on Support
|
|
51
|
+
|
|
52
|
+
As this is an alpha version, support and responses might be limited. We'll do our best to address questions and issues as quickly as possible.
|
|
53
|
+
|
|
54
|
+
### Contribution Guidelines
|
|
55
|
+
|
|
56
|
+
We welcome contributions! If you find a bug or have suggestions for improvements, please open an issue or submit a pull request.
|
|
57
|
+
|
|
58
|
+
##### Conda Support
|
|
59
|
+
|
|
60
|
+
To run and build locally you can use [conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html):
|
|
61
|
+
|
|
62
|
+
```
|
|
63
|
+
conda create -n oshepherd python=3.8
|
|
64
|
+
conda activate oshepherd
|
|
65
|
+
pip install -r requirements.txt
|
|
66
|
+
|
|
67
|
+
# install oshepherd
|
|
68
|
+
pip install -e .
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
##### Tests
|
|
72
|
+
|
|
73
|
+
Follow usage instructions to start api server and celery worker using a local ollama, and then run the tests:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
pytest -s tests/
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Reporting Issues
|
|
80
|
+
|
|
81
|
+
Please report any issues you encounter on the GitHub issues page. Before creating a new issue, take a moment to search through the existing issues to avoid duplicates.
|
|
82
|
+
|
|
83
|
+
### Author
|
|
84
|
+
|
|
85
|
+
Currently, [mnemonica.ai](mnemonica.ai) is sponsoring the development of this tool.
|
|
86
|
+
|
|
87
|
+
### License
|
|
88
|
+
|
|
89
|
+
[MIT](LICENSE)
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from flask import Flask, Blueprint
|
|
2
|
+
from oshepherd.api.config import ApiConfig
|
|
3
|
+
from oshepherd.api.generate.routes import generate_blueprint
|
|
4
|
+
from oshepherd.worker.app import create_celery_app_for_flask
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def start_flask_app(config: ApiConfig):
|
|
8
|
+
app = Flask(config.FLASK_PROJECT_NAME)
|
|
9
|
+
app.config["FLASK_RUN_PORT"] = config.FLASK_RUN_PORT
|
|
10
|
+
app.config["FLASK_DEBUG"] = config.FLASK_DEBUG
|
|
11
|
+
app.config["FLASK_HOST"] = config.FLASK_HOST
|
|
12
|
+
app.config["RABBITMQ_URL"] = config.RABBITMQ_URL
|
|
13
|
+
app.config["REDIS_URL"] = config.REDIS_URL
|
|
14
|
+
|
|
15
|
+
# celery setup
|
|
16
|
+
celery_app = create_celery_app_for_flask(app)
|
|
17
|
+
app.celery = celery_app
|
|
18
|
+
|
|
19
|
+
# endpoints
|
|
20
|
+
api = Blueprint("api", __name__)
|
|
21
|
+
api.register_blueprint(generate_blueprint)
|
|
22
|
+
app.register_blueprint(api)
|
|
23
|
+
|
|
24
|
+
app.run(
|
|
25
|
+
debug=app.config["FLASK_DEBUG"],
|
|
26
|
+
host=app.config["FLASK_HOST"],
|
|
27
|
+
port=app.config["FLASK_RUN_PORT"],
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
return app
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
from pydantic import BaseModel
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class ApiConfig(BaseModel):
|
|
6
|
+
FLASK_PROJECT_NAME: Optional[str] = "oshepherd_api"
|
|
7
|
+
FLASK_ENV: Optional[str] = "development"
|
|
8
|
+
FLASK_DEBUG: Optional[bool] = True
|
|
9
|
+
FLASK_RUN_PORT: Optional[int] = 5001
|
|
10
|
+
FLASK_HOST: Optional[str] = "0.0.0.0"
|
|
11
|
+
RABBITMQ_URL: str
|
|
12
|
+
REDIS_URL: str
|
|
File without changes
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
from pydantic import BaseModel
|
|
2
|
+
from typing import Optional, List
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class GenerateRequest(BaseModel):
|
|
6
|
+
model: str
|
|
7
|
+
prompt: str
|
|
8
|
+
images: Optional[List[str]] = None
|
|
9
|
+
format: Optional[str] = "json"
|
|
10
|
+
options: Optional[dict] = None
|
|
11
|
+
system: Optional[str] = None
|
|
12
|
+
template: Optional[str] = None
|
|
13
|
+
context: Optional[List[int]] = None
|
|
14
|
+
stream: Optional[bool] = False
|
|
15
|
+
raw: Optional[bool] = False
|
|
16
|
+
keep_alive: Optional[str] = "5m"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class GenerateResponse(BaseModel):
|
|
20
|
+
model: str
|
|
21
|
+
created_at: str
|
|
22
|
+
response: str
|
|
23
|
+
done: bool
|
|
24
|
+
context: Optional[List[int]] = None
|
|
25
|
+
total_duration: Optional[int] = None
|
|
26
|
+
load_duration: Optional[int] = None
|
|
27
|
+
prompt_eval_count: Optional[int] = None
|
|
28
|
+
prompt_eval_duration: Optional[int] = None
|
|
29
|
+
eval_count: Optional[int] = None
|
|
30
|
+
eval_duration: Optional[int] = None
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from flask import Blueprint, request
|
|
3
|
+
from oshepherd.api.utils import streamify_json
|
|
4
|
+
from oshepherd.api.generate.models import GenerateRequest
|
|
5
|
+
|
|
6
|
+
generate_blueprint = Blueprint("generate", __name__, url_prefix="/api/generate")
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@generate_blueprint.route("/", methods=["POST"])
|
|
10
|
+
def generate():
|
|
11
|
+
# TODO use .send_task() instead?
|
|
12
|
+
from oshepherd.worker.tasks import make_generate_request
|
|
13
|
+
|
|
14
|
+
print(f" # request.json {request.json}")
|
|
15
|
+
generate_request = GenerateRequest(**request.json)
|
|
16
|
+
|
|
17
|
+
# req as json string ready to be sent though broker
|
|
18
|
+
generate_request_json_str = generate_request.model_dump_json()
|
|
19
|
+
print(generate_request_json_str)
|
|
20
|
+
|
|
21
|
+
# queue request to remote ollama api server though
|
|
22
|
+
task = make_generate_request.delay(generate_request_json_str)
|
|
23
|
+
while not task.ready():
|
|
24
|
+
print(" > waiting for response...")
|
|
25
|
+
time.sleep(1)
|
|
26
|
+
ollama_res = task.get(timeout=1)
|
|
27
|
+
|
|
28
|
+
status = 200
|
|
29
|
+
if ollama_res.get("error"):
|
|
30
|
+
ollama_res = {
|
|
31
|
+
"error": "Internal Server Error",
|
|
32
|
+
"message": f"error triggering llm inference: {ollama_res['error']['message']}",
|
|
33
|
+
}
|
|
34
|
+
status = 500
|
|
35
|
+
|
|
36
|
+
print(f" $ ollama response {status}: {ollama_res}")
|
|
37
|
+
|
|
38
|
+
return streamify_json(ollama_res, status)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from flask import Response, stream_with_context
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def streamify_json(json_data, status=200):
|
|
6
|
+
def json_stream(data):
|
|
7
|
+
yield json.dumps(data)
|
|
8
|
+
|
|
9
|
+
return Response(
|
|
10
|
+
stream_with_context(json_stream(json_data)),
|
|
11
|
+
status=status,
|
|
12
|
+
mimetype="application/json",
|
|
13
|
+
)
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import click
|
|
2
|
+
from oshepherd.lib import load_and_validate_env
|
|
3
|
+
from oshepherd.api.app import start_flask_app
|
|
4
|
+
from oshepherd.api.config import ApiConfig
|
|
5
|
+
from oshepherd.worker.config import WorkerConfig
|
|
6
|
+
from oshepherd.worker.app import create_celery_app
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@click.group()
|
|
10
|
+
def main():
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@main.command()
|
|
15
|
+
@click.option(
|
|
16
|
+
"--env-file",
|
|
17
|
+
type=click.Path(exists=True),
|
|
18
|
+
help="Path to the .env file with environment variables.",
|
|
19
|
+
)
|
|
20
|
+
def start_api(env_file):
|
|
21
|
+
"""Starts the Flask API serving Ollama model's inference."""
|
|
22
|
+
config: ApiConfig = load_and_validate_env(env_file, ApiConfig)
|
|
23
|
+
|
|
24
|
+
start_flask_app(config)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@main.command()
|
|
28
|
+
@click.option(
|
|
29
|
+
"--env-file",
|
|
30
|
+
type=click.Path(exists=True),
|
|
31
|
+
help="Path to the .env file with environment variables.",
|
|
32
|
+
)
|
|
33
|
+
def start_worker(env_file):
|
|
34
|
+
"""Starts the Celery Worker serving local Ollama models."""
|
|
35
|
+
config: WorkerConfig = load_and_validate_env(env_file, WorkerConfig)
|
|
36
|
+
|
|
37
|
+
celery_app = create_celery_app(config)
|
|
38
|
+
worker = celery_app.Worker(
|
|
39
|
+
loglevel=config.LOGLEVEL,
|
|
40
|
+
concurrency=config.CONCURRENCY,
|
|
41
|
+
prefetch_multiplier=config.PREFETCH_MULTIPLIER,
|
|
42
|
+
)
|
|
43
|
+
worker.start()
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
if __name__ == "__main__":
|
|
47
|
+
main()
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from dotenv import load_dotenv
|
|
3
|
+
from pydantic import BaseModel, ValidationError
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def load_and_validate_env(file_path: str, Config: BaseModel) -> Optional[BaseModel]:
|
|
8
|
+
# Load .env file into environment variables
|
|
9
|
+
load_dotenv(file_path)
|
|
10
|
+
|
|
11
|
+
# Create a dictionary of environment variables
|
|
12
|
+
env_vars = {
|
|
13
|
+
key: os.getenv(key) for key in Config.__annotations__.keys() if os.getenv(key)
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
# Validate environment variables against ApiConfig model
|
|
17
|
+
try:
|
|
18
|
+
config = Config(**env_vars)
|
|
19
|
+
except ValidationError as e:
|
|
20
|
+
print(f"Invalid configuration: {e}")
|
|
21
|
+
return None
|
|
22
|
+
|
|
23
|
+
return config
|
|
File without changes
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
from celery import Celery
|
|
2
|
+
from oshepherd.worker.config import WorkerConfig
|
|
3
|
+
|
|
4
|
+
TASKS_MODULE = "oshepherd.worker.tasks"
|
|
5
|
+
|
|
6
|
+
celery_app = None
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def create_celery_app(config: WorkerConfig):
|
|
10
|
+
"""
|
|
11
|
+
Create a Celery app instance given a certain configuration.
|
|
12
|
+
Refs:
|
|
13
|
+
* https://docs.celeryq.dev/en/stable/getting-started/next-steps.html#using-celery-in-your-application
|
|
14
|
+
* https://docs.celeryq.dev/en/stable/getting-started/first-steps-with-celery.html#celerytut-keeping-results
|
|
15
|
+
"""
|
|
16
|
+
global celery_app
|
|
17
|
+
|
|
18
|
+
celery_app = Celery(
|
|
19
|
+
TASKS_MODULE, broker=config.RABBITMQ_URL, backend=config.REDIS_URL
|
|
20
|
+
)
|
|
21
|
+
celery_app.conf.update(
|
|
22
|
+
result_expires=config.RESULTS_EXPIRES,
|
|
23
|
+
)
|
|
24
|
+
celery_app.autodiscover_tasks([TASKS_MODULE])
|
|
25
|
+
|
|
26
|
+
return celery_app
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def create_celery_app_for_flask(flask_app):
|
|
30
|
+
config = WorkerConfig(
|
|
31
|
+
RABBITMQ_URL=flask_app.config["RABBITMQ_URL"],
|
|
32
|
+
REDIS_URL=flask_app.config["REDIS_URL"],
|
|
33
|
+
RESULTS_EXPIRES=flask_app.config.get(
|
|
34
|
+
"RESULTS_EXPIRES", WorkerConfig.__fields__["RESULTS_EXPIRES"].default
|
|
35
|
+
),
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
celery_app = create_celery_app(config)
|
|
39
|
+
|
|
40
|
+
return celery_app
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from pydantic import BaseModel
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class WorkerConfig(BaseModel):
|
|
6
|
+
RABBITMQ_URL: str
|
|
7
|
+
REDIS_URL: str
|
|
8
|
+
LOGLEVEL: Optional[str] = "info"
|
|
9
|
+
CONCURRENCY: Optional[int] = 1
|
|
10
|
+
PREFETCH_MULTIPLIER: Optional[int] = 1
|
|
11
|
+
RESULTS_EXPIRES: Optional[int] = 3600
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import ollama
|
|
3
|
+
from oshepherd.worker.app import celery_app
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@celery_app.task(name="oshepherd.worker.tasks.make_generate_request")
|
|
7
|
+
def make_generate_request(request_str: str):
|
|
8
|
+
try:
|
|
9
|
+
request = json.loads(request_str)
|
|
10
|
+
print(f"# make_generate_request request {request}")
|
|
11
|
+
|
|
12
|
+
response = ollama.generate(**request)
|
|
13
|
+
print(f" $ success {response}")
|
|
14
|
+
except Exception as error:
|
|
15
|
+
print(f" * error make_generate_request {error}")
|
|
16
|
+
response = {
|
|
17
|
+
"error": {"type": str(error.__class__.__name__), "message": str(error)}
|
|
18
|
+
}
|
|
19
|
+
print(
|
|
20
|
+
f" * error response {response}",
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
return response
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: oshepherd
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: The Oshepherd guiding the Ollama(s) inference orchestration.
|
|
5
|
+
Author: mnemonica.ai
|
|
6
|
+
Author-email: "mnemonica.ai" <info@mnemonica.ai>
|
|
7
|
+
Project-URL: Homepage, https://github.com/mnemonica-ai/oshepherd
|
|
8
|
+
Project-URL: Issues, https://github.com/mnemonica-ai/oshepherd/issues
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
11
|
+
Classifier: Operating System :: OS Independent
|
|
12
|
+
Requires-Python: >=3.8
|
|
13
|
+
Description-Content-Type: text/markdown
|
|
14
|
+
License-File: LICENSE
|
|
15
|
+
|
|
16
|
+
# oshepherd
|
|
17
|
+
|
|
18
|
+
> The Oshepherd guiding the Ollama(s) inference orchestration.
|
|
19
|
+
|
|
20
|
+
A centralized [Flask](https://flask.palletsprojects.com) API service, using [Celery](https://docs.celeryq.dev) ([RabbitMQ](https://www.rabbitmq.com) + [Redis](https://redis.com)) to orchestrate multiple [Ollama](https://ollama.com) workers.
|
|
21
|
+
|
|
22
|
+
### Install
|
|
23
|
+
|
|
24
|
+
```
|
|
25
|
+
pip install oshepherd
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
### Usage
|
|
29
|
+
|
|
30
|
+
1. Setup RabbitMQ and Redis:
|
|
31
|
+
|
|
32
|
+
Create instances for free for both:
|
|
33
|
+
* [cloudamqp.com](https://www.cloudamqp.com)
|
|
34
|
+
* [redislabs.com](https://app.redislabs.com)
|
|
35
|
+
|
|
36
|
+
2. Setup Flask API Server:
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
# define configuration env file
|
|
40
|
+
# use credentials for redis and rabbitmq
|
|
41
|
+
cp .api.env.template .api.env
|
|
42
|
+
|
|
43
|
+
# start api
|
|
44
|
+
oshepherd start-api --env-file .api.env
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
3. Setup Celery/Ollama Worker(s):
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
# install ollama https://ollama.com/download
|
|
51
|
+
ollama run mistral
|
|
52
|
+
|
|
53
|
+
# define configuration env file
|
|
54
|
+
# use credentials for redis and rabbitmq
|
|
55
|
+
cp .worker.env.template .worker.env
|
|
56
|
+
|
|
57
|
+
# start worker
|
|
58
|
+
oshepherd start-worker --env-file .worker.env
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
### Words of advice 🚨
|
|
62
|
+
|
|
63
|
+
This package is in alpha, its architecture and api might change in the near future. Currently this is getting tested in a closed environment by real users, but haven't been audited, nor tested thorugly. Use it at your own risk.
|
|
64
|
+
|
|
65
|
+
### Disclaimer on Support
|
|
66
|
+
|
|
67
|
+
As this is an alpha version, support and responses might be limited. We'll do our best to address questions and issues as quickly as possible.
|
|
68
|
+
|
|
69
|
+
### Contribution Guidelines
|
|
70
|
+
|
|
71
|
+
We welcome contributions! If you find a bug or have suggestions for improvements, please open an issue or submit a pull request.
|
|
72
|
+
|
|
73
|
+
##### Conda Support
|
|
74
|
+
|
|
75
|
+
To run and build locally you can use [conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html):
|
|
76
|
+
|
|
77
|
+
```
|
|
78
|
+
conda create -n oshepherd python=3.8
|
|
79
|
+
conda activate oshepherd
|
|
80
|
+
pip install -r requirements.txt
|
|
81
|
+
|
|
82
|
+
# install oshepherd
|
|
83
|
+
pip install -e .
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
##### Tests
|
|
87
|
+
|
|
88
|
+
Follow usage instructions to start api server and celery worker using a local ollama, and then run the tests:
|
|
89
|
+
|
|
90
|
+
```
|
|
91
|
+
pytest -s tests/
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### Reporting Issues
|
|
95
|
+
|
|
96
|
+
Please report any issues you encounter on the GitHub issues page. Before creating a new issue, take a moment to search through the existing issues to avoid duplicates.
|
|
97
|
+
|
|
98
|
+
### Author
|
|
99
|
+
|
|
100
|
+
Currently, [mnemonica.ai](mnemonica.ai) is sponsoring the development of this tool.
|
|
101
|
+
|
|
102
|
+
### License
|
|
103
|
+
|
|
104
|
+
[MIT](LICENSE)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
setup.py
|
|
5
|
+
oshepherd/__init__.py
|
|
6
|
+
oshepherd/cli.py
|
|
7
|
+
oshepherd/lib.py
|
|
8
|
+
oshepherd.egg-info/PKG-INFO
|
|
9
|
+
oshepherd.egg-info/SOURCES.txt
|
|
10
|
+
oshepherd.egg-info/dependency_links.txt
|
|
11
|
+
oshepherd.egg-info/top_level.txt
|
|
12
|
+
oshepherd/api/__init__.py
|
|
13
|
+
oshepherd/api/app.py
|
|
14
|
+
oshepherd/api/config.py
|
|
15
|
+
oshepherd/api/utils.py
|
|
16
|
+
oshepherd/api/generate/__init__.py
|
|
17
|
+
oshepherd/api/generate/models.py
|
|
18
|
+
oshepherd/api/generate/routes.py
|
|
19
|
+
oshepherd/worker/__init__.py
|
|
20
|
+
oshepherd/worker/app.py
|
|
21
|
+
oshepherd/worker/config.py
|
|
22
|
+
oshepherd/worker/tasks.py
|
|
23
|
+
tests/__init__.py
|
|
24
|
+
tests/test_e2e_api.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "oshepherd"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
authors = [
|
|
5
|
+
{ name="mnemonica.ai", email="info@mnemonica.ai" },
|
|
6
|
+
]
|
|
7
|
+
description = "The Oshepherd guiding the Ollama(s) inference orchestration."
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.8"
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Programming Language :: Python :: 3",
|
|
12
|
+
"License :: OSI Approved :: MIT License",
|
|
13
|
+
"Operating System :: OS Independent",
|
|
14
|
+
]
|
|
15
|
+
|
|
16
|
+
[project.urls]
|
|
17
|
+
Homepage = "https://github.com/mnemonica-ai/oshepherd"
|
|
18
|
+
Issues = "https://github.com/mnemonica-ai/oshepherd/issues"
|
oshepherd-0.0.1/setup.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
|
|
3
|
+
setup(
|
|
4
|
+
name="oshepherd",
|
|
5
|
+
version="0.0.1",
|
|
6
|
+
description="The Oshepherd guiding the Ollama(s) inference orchestration.",
|
|
7
|
+
packages=find_packages(),
|
|
8
|
+
install_requires=[
|
|
9
|
+
"flask",
|
|
10
|
+
"celery",
|
|
11
|
+
"click",
|
|
12
|
+
"ollama",
|
|
13
|
+
"amqp",
|
|
14
|
+
"redis",
|
|
15
|
+
"pydantic",
|
|
16
|
+
"pytest",
|
|
17
|
+
"python-dotenv",
|
|
18
|
+
"requests",
|
|
19
|
+
],
|
|
20
|
+
entry_points={
|
|
21
|
+
"console_scripts": [
|
|
22
|
+
"oshepherd=oshepherd.cli:main",
|
|
23
|
+
]
|
|
24
|
+
},
|
|
25
|
+
author="mnemonica.ai",
|
|
26
|
+
author_email="info@mnemonica.ai",
|
|
27
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Basic end to end tests, using an equivalent request as pointing to ollama api server in local:
|
|
3
|
+
curl -X POST -H "Content-Type: application/json" -L http://127.0.0.1:11434/api/generate/ -d '{
|
|
4
|
+
"model": "mistral",
|
|
5
|
+
"prompt":"Why is the sky blue?"
|
|
6
|
+
}'
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import requests
|
|
11
|
+
import ollama
|
|
12
|
+
from oshepherd.api.generate.models import GenerateResponse
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def test_basic_api_worker_queueing_using_ollama():
|
|
16
|
+
params = {"model": "mistral", "prompt": "Why is the sky blue?"}
|
|
17
|
+
client = ollama.Client(host="http://127.0.0.1:5001")
|
|
18
|
+
ollama_res = client.generate(**params)
|
|
19
|
+
|
|
20
|
+
ollama_res = GenerateResponse(**ollama_res)
|
|
21
|
+
assert ollama_res.response, "response should not be empty"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_basic_api_worker_queueing_using_requests():
|
|
25
|
+
url = "http://127.0.0.1:5001/api/generate/"
|
|
26
|
+
headers = {"Content-Type": "application/json"}
|
|
27
|
+
data = {"model": "mistral", "prompt": "Why is the sky blue?"}
|
|
28
|
+
response = requests.post(url, headers=headers, data=json.dumps(data))
|
|
29
|
+
|
|
30
|
+
assert response.status_code == 200
|
|
31
|
+
assert "error" not in response
|
|
32
|
+
ollama_res = GenerateResponse(**response.json())
|
|
33
|
+
assert ollama_res.response, "response should not be empty"
|