oshepherd 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of oshepherd might be problematic. Click here for more details.

oshepherd/__init__.py ADDED
File without changes
File without changes
oshepherd/api/app.py ADDED
@@ -0,0 +1,30 @@
1
+ from flask import Flask, Blueprint
2
+ from oshepherd.api.config import ApiConfig
3
+ from oshepherd.api.generate.routes import generate_blueprint
4
+ from oshepherd.worker.app import create_celery_app_for_flask
5
+
6
+
7
+ def start_flask_app(config: ApiConfig):
8
+ app = Flask(config.FLASK_PROJECT_NAME)
9
+ app.config["FLASK_RUN_PORT"] = config.FLASK_RUN_PORT
10
+ app.config["FLASK_DEBUG"] = config.FLASK_DEBUG
11
+ app.config["FLASK_HOST"] = config.FLASK_HOST
12
+ app.config["RABBITMQ_URL"] = config.RABBITMQ_URL
13
+ app.config["REDIS_URL"] = config.REDIS_URL
14
+
15
+ # celery setup
16
+ celery_app = create_celery_app_for_flask(app)
17
+ app.celery = celery_app
18
+
19
+ # endpoints
20
+ api = Blueprint("api", __name__)
21
+ api.register_blueprint(generate_blueprint)
22
+ app.register_blueprint(api)
23
+
24
+ app.run(
25
+ debug=app.config["FLASK_DEBUG"],
26
+ host=app.config["FLASK_HOST"],
27
+ port=app.config["FLASK_RUN_PORT"],
28
+ )
29
+
30
+ return app
@@ -0,0 +1,12 @@
1
+ from pydantic import BaseModel
2
+ from typing import Optional
3
+
4
+
5
+ class ApiConfig(BaseModel):
6
+ FLASK_PROJECT_NAME: Optional[str] = "oshepherd_api"
7
+ FLASK_ENV: Optional[str] = "development"
8
+ FLASK_DEBUG: Optional[bool] = True
9
+ FLASK_RUN_PORT: Optional[int] = 5001
10
+ FLASK_HOST: Optional[str] = "0.0.0.0"
11
+ RABBITMQ_URL: str
12
+ REDIS_URL: str
File without changes
@@ -0,0 +1,30 @@
1
+ from pydantic import BaseModel
2
+ from typing import Optional, List
3
+
4
+
5
+ class GenerateRequest(BaseModel):
6
+ model: str
7
+ prompt: str
8
+ images: Optional[List[str]] = None
9
+ format: Optional[str] = "json"
10
+ options: Optional[dict] = None
11
+ system: Optional[str] = None
12
+ template: Optional[str] = None
13
+ context: Optional[List[int]] = None
14
+ stream: Optional[bool] = False
15
+ raw: Optional[bool] = False
16
+ keep_alive: Optional[str] = "5m"
17
+
18
+
19
+ class GenerateResponse(BaseModel):
20
+ model: str
21
+ created_at: str
22
+ response: str
23
+ done: bool
24
+ context: Optional[List[int]] = None
25
+ total_duration: Optional[int] = None
26
+ load_duration: Optional[int] = None
27
+ prompt_eval_count: Optional[int] = None
28
+ prompt_eval_duration: Optional[int] = None
29
+ eval_count: Optional[int] = None
30
+ eval_duration: Optional[int] = None
@@ -0,0 +1,38 @@
1
+ import time
2
+ from flask import Blueprint, request
3
+ from oshepherd.api.utils import streamify_json
4
+ from oshepherd.api.generate.models import GenerateRequest
5
+
6
+ generate_blueprint = Blueprint("generate", __name__, url_prefix="/api/generate")
7
+
8
+
9
+ @generate_blueprint.route("/", methods=["POST"])
10
+ def generate():
11
+ # TODO use .send_task() instead?
12
+ from oshepherd.worker.tasks import make_generate_request
13
+
14
+ print(f" # request.json {request.json}")
15
+ generate_request = GenerateRequest(**request.json)
16
+
17
+ # req as json string ready to be sent though broker
18
+ generate_request_json_str = generate_request.model_dump_json()
19
+ print(generate_request_json_str)
20
+
21
+ # queue request to remote ollama api server though
22
+ task = make_generate_request.delay(generate_request_json_str)
23
+ while not task.ready():
24
+ print(" > waiting for response...")
25
+ time.sleep(1)
26
+ ollama_res = task.get(timeout=1)
27
+
28
+ status = 200
29
+ if ollama_res.get("error"):
30
+ ollama_res = {
31
+ "error": "Internal Server Error",
32
+ "message": f"error triggering llm inference: {ollama_res['error']['message']}",
33
+ }
34
+ status = 500
35
+
36
+ print(f" $ ollama response {status}: {ollama_res}")
37
+
38
+ return streamify_json(ollama_res, status)
oshepherd/api/utils.py ADDED
@@ -0,0 +1,13 @@
1
+ import json
2
+ from flask import Response, stream_with_context
3
+
4
+
5
+ def streamify_json(json_data, status=200):
6
+ def json_stream(data):
7
+ yield json.dumps(data)
8
+
9
+ return Response(
10
+ stream_with_context(json_stream(json_data)),
11
+ status=status,
12
+ mimetype="application/json",
13
+ )
oshepherd/cli.py ADDED
@@ -0,0 +1,47 @@
1
+ import click
2
+ from oshepherd.lib import load_and_validate_env
3
+ from oshepherd.api.app import start_flask_app
4
+ from oshepherd.api.config import ApiConfig
5
+ from oshepherd.worker.config import WorkerConfig
6
+ from oshepherd.worker.app import create_celery_app
7
+
8
+
9
+ @click.group()
10
+ def main():
11
+ pass
12
+
13
+
14
+ @main.command()
15
+ @click.option(
16
+ "--env-file",
17
+ type=click.Path(exists=True),
18
+ help="Path to the .env file with environment variables.",
19
+ )
20
+ def start_api(env_file):
21
+ """Starts the Flask API serving Ollama model's inference."""
22
+ config: ApiConfig = load_and_validate_env(env_file, ApiConfig)
23
+
24
+ start_flask_app(config)
25
+
26
+
27
+ @main.command()
28
+ @click.option(
29
+ "--env-file",
30
+ type=click.Path(exists=True),
31
+ help="Path to the .env file with environment variables.",
32
+ )
33
+ def start_worker(env_file):
34
+ """Starts the Celery Worker serving local Ollama models."""
35
+ config: WorkerConfig = load_and_validate_env(env_file, WorkerConfig)
36
+
37
+ celery_app = create_celery_app(config)
38
+ worker = celery_app.Worker(
39
+ loglevel=config.LOGLEVEL,
40
+ concurrency=config.CONCURRENCY,
41
+ prefetch_multiplier=config.PREFETCH_MULTIPLIER,
42
+ )
43
+ worker.start()
44
+
45
+
46
+ if __name__ == "__main__":
47
+ main()
oshepherd/lib.py ADDED
@@ -0,0 +1,23 @@
1
+ import os
2
+ from dotenv import load_dotenv
3
+ from pydantic import BaseModel, ValidationError
4
+ from typing import Optional
5
+
6
+
7
+ def load_and_validate_env(file_path: str, Config: BaseModel) -> Optional[BaseModel]:
8
+ # Load .env file into environment variables
9
+ load_dotenv(file_path)
10
+
11
+ # Create a dictionary of environment variables
12
+ env_vars = {
13
+ key: os.getenv(key) for key in Config.__annotations__.keys() if os.getenv(key)
14
+ }
15
+
16
+ # Validate environment variables against ApiConfig model
17
+ try:
18
+ config = Config(**env_vars)
19
+ except ValidationError as e:
20
+ print(f"Invalid configuration: {e}")
21
+ return None
22
+
23
+ return config
File without changes
@@ -0,0 +1,40 @@
1
+ from celery import Celery
2
+ from oshepherd.worker.config import WorkerConfig
3
+
4
+ TASKS_MODULE = "oshepherd.worker.tasks"
5
+
6
+ celery_app = None
7
+
8
+
9
+ def create_celery_app(config: WorkerConfig):
10
+ """
11
+ Create a Celery app instance given a certain configuration.
12
+ Refs:
13
+ * https://docs.celeryq.dev/en/stable/getting-started/next-steps.html#using-celery-in-your-application
14
+ * https://docs.celeryq.dev/en/stable/getting-started/first-steps-with-celery.html#celerytut-keeping-results
15
+ """
16
+ global celery_app
17
+
18
+ celery_app = Celery(
19
+ TASKS_MODULE, broker=config.RABBITMQ_URL, backend=config.REDIS_URL
20
+ )
21
+ celery_app.conf.update(
22
+ result_expires=config.RESULTS_EXPIRES,
23
+ )
24
+ celery_app.autodiscover_tasks([TASKS_MODULE])
25
+
26
+ return celery_app
27
+
28
+
29
+ def create_celery_app_for_flask(flask_app):
30
+ config = WorkerConfig(
31
+ RABBITMQ_URL=flask_app.config["RABBITMQ_URL"],
32
+ REDIS_URL=flask_app.config["REDIS_URL"],
33
+ RESULTS_EXPIRES=flask_app.config.get(
34
+ "RESULTS_EXPIRES", WorkerConfig.__fields__["RESULTS_EXPIRES"].default
35
+ ),
36
+ )
37
+
38
+ celery_app = create_celery_app(config)
39
+
40
+ return celery_app
@@ -0,0 +1,11 @@
1
+ from pydantic import BaseModel
2
+ from typing import Optional
3
+
4
+
5
+ class WorkerConfig(BaseModel):
6
+ RABBITMQ_URL: str
7
+ REDIS_URL: str
8
+ LOGLEVEL: Optional[str] = "info"
9
+ CONCURRENCY: Optional[int] = 1
10
+ PREFETCH_MULTIPLIER: Optional[int] = 1
11
+ RESULTS_EXPIRES: Optional[int] = 3600
@@ -0,0 +1,23 @@
1
+ import json
2
+ import ollama
3
+ from oshepherd.worker.app import celery_app
4
+
5
+
6
+ @celery_app.task(name="oshepherd.worker.tasks.make_generate_request")
7
+ def make_generate_request(request_str: str):
8
+ try:
9
+ request = json.loads(request_str)
10
+ print(f"# make_generate_request request {request}")
11
+
12
+ response = ollama.generate(**request)
13
+ print(f" $ success {response}")
14
+ except Exception as error:
15
+ print(f" * error make_generate_request {error}")
16
+ response = {
17
+ "error": {"type": str(error.__class__.__name__), "message": str(error)}
18
+ }
19
+ print(
20
+ f" * error response {response}",
21
+ )
22
+
23
+ return response
@@ -0,0 +1,22 @@
1
+ (The MIT License)
2
+
3
+ Copyright (c) 2024 mnemonica.ai <mnemonica.ai>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ 'Software'), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,104 @@
1
+ Metadata-Version: 2.1
2
+ Name: oshepherd
3
+ Version: 0.0.1
4
+ Summary: The Oshepherd guiding the Ollama(s) inference orchestration.
5
+ Author: mnemonica.ai
6
+ Author-email: "mnemonica.ai" <info@mnemonica.ai>
7
+ Project-URL: Homepage, https://github.com/mnemonica-ai/oshepherd
8
+ Project-URL: Issues, https://github.com/mnemonica-ai/oshepherd/issues
9
+ Classifier: Programming Language :: Python :: 3
10
+ Classifier: License :: OSI Approved :: MIT License
11
+ Classifier: Operating System :: OS Independent
12
+ Requires-Python: >=3.8
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+
16
+ # oshepherd
17
+
18
+ > The Oshepherd guiding the Ollama(s) inference orchestration.
19
+
20
+ A centralized [Flask](https://flask.palletsprojects.com) API service, using [Celery](https://docs.celeryq.dev) ([RabbitMQ](https://www.rabbitmq.com) + [Redis](https://redis.com)) to orchestrate multiple [Ollama](https://ollama.com) workers.
21
+
22
+ ### Install
23
+
24
+ ```
25
+ pip install oshepherd
26
+ ```
27
+
28
+ ### Usage
29
+
30
+ 1. Setup RabbitMQ and Redis:
31
+
32
+ Create instances for free for both:
33
+ * [cloudamqp.com](https://www.cloudamqp.com)
34
+ * [redislabs.com](https://app.redislabs.com)
35
+
36
+ 2. Setup Flask API Server:
37
+
38
+ ```
39
+ # define configuration env file
40
+ # use credentials for redis and rabbitmq
41
+ cp .api.env.template .api.env
42
+
43
+ # start api
44
+ oshepherd start-api --env-file .api.env
45
+ ```
46
+
47
+ 3. Setup Celery/Ollama Worker(s):
48
+
49
+ ```
50
+ # install ollama https://ollama.com/download
51
+ ollama run mistral
52
+
53
+ # define configuration env file
54
+ # use credentials for redis and rabbitmq
55
+ cp .worker.env.template .worker.env
56
+
57
+ # start worker
58
+ oshepherd start-worker --env-file .worker.env
59
+ ```
60
+
61
+ ### Words of advice 🚨
62
+
63
+ This package is in alpha, its architecture and api might change in the near future. Currently this is getting tested in a closed environment by real users, but haven't been audited, nor tested thorugly. Use it at your own risk.
64
+
65
+ ### Disclaimer on Support
66
+
67
+ As this is an alpha version, support and responses might be limited. We'll do our best to address questions and issues as quickly as possible.
68
+
69
+ ### Contribution Guidelines
70
+
71
+ We welcome contributions! If you find a bug or have suggestions for improvements, please open an issue or submit a pull request.
72
+
73
+ ##### Conda Support
74
+
75
+ To run and build locally you can use [conda](https://conda.io/projects/conda/en/latest/user-guide/install/index.html):
76
+
77
+ ```
78
+ conda create -n oshepherd python=3.8
79
+ conda activate oshepherd
80
+ pip install -r requirements.txt
81
+
82
+ # install oshepherd
83
+ pip install -e .
84
+ ```
85
+
86
+ ##### Tests
87
+
88
+ Follow usage instructions to start api server and celery worker using a local ollama, and then run the tests:
89
+
90
+ ```
91
+ pytest -s tests/
92
+ ```
93
+
94
+ ### Reporting Issues
95
+
96
+ Please report any issues you encounter on the GitHub issues page. Before creating a new issue, take a moment to search through the existing issues to avoid duplicates.
97
+
98
+ ### Author
99
+
100
+ Currently, [mnemonica.ai](mnemonica.ai) is sponsoring the development of this tool.
101
+
102
+ ### License
103
+
104
+ [MIT](LICENSE)
@@ -0,0 +1,21 @@
1
+ oshepherd/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ oshepherd/cli.py,sha256=udwCa1H3mFIWZHw0j6wRIqlLQ6UI-WjGtOM3GhSzaUs,1208
3
+ oshepherd/lib.py,sha256=tGGOUBDuCFv3MiNSJ__5dYi0KmJYF8jPrdBSgumZa74,665
4
+ oshepherd/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
5
+ oshepherd/api/app.py,sha256=udmBSLJV9msBjew7Yd70FG46oaaIDWVeTTaG97g73DA,932
6
+ oshepherd/api/config.py,sha256=LKt032hFbn-S1UfHRKpWZMuWOOjmKTrHsl0q-wzRSpU,353
7
+ oshepherd/api/utils.py,sha256=vHrELbyBkZjUhKwBE3t0ZRd8zFgbPqO5us9uocdxdQ0,304
8
+ oshepherd/api/generate/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
9
+ oshepherd/api/generate/models.py,sha256=DZaC8c6w8mU4_9tjnXQTDnbsIw8j4_7xqHS-z7dZpjI,847
10
+ oshepherd/api/generate/routes.py,sha256=pYAnN0EdGa8TkcIM6vVQvl2Q1zKVhbkCDdTBXpCXz2U,1262
11
+ oshepherd/worker/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
+ oshepherd/worker/app.py,sha256=SA-YUuToA77ksClQFKXJ-iEm9S8WZ1bpbvpRh1fEC4g,1170
13
+ oshepherd/worker/config.py,sha256=lBC5fMvEq2nYC2D0Q-ad6iwH-DoD2xybRHq1tQCN6uc,290
14
+ oshepherd/worker/tasks.py,sha256=mIMMN9l6lQ4lrzxmPa9V3t59FgO2AC-CA9kwbfYiyFQ,684
15
+ tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
16
+ tests/test_e2e_api.py,sha256=F30sYhvyobIA2xwKfqFmY39cshab0VQ0ep7vq_1AqZs,1197
17
+ oshepherd-0.0.1.dist-info/LICENSE,sha256=jg8MgAFOnTmeA16IT2Un7Vf_OyyzB1foHxnnx-dJmyU,1090
18
+ oshepherd-0.0.1.dist-info/METADATA,sha256=SwZmf291jl_fYBIHpt2_ynaAdgY3SzvcpKLOhIyrlJg,2985
19
+ oshepherd-0.0.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
20
+ oshepherd-0.0.1.dist-info/top_level.txt,sha256=sl8YLK-dPWBDK3Yrv-y_H1UniOTuGF9c0lETQn96_us,16
21
+ oshepherd-0.0.1.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: bdist_wheel (0.43.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ oshepherd
2
+ tests
tests/__init__.py ADDED
File without changes
tests/test_e2e_api.py ADDED
@@ -0,0 +1,33 @@
1
+ """
2
+ Basic end to end tests, using an equivalent request as pointing to ollama api server in local:
3
+ curl -X POST -H "Content-Type: application/json" -L http://127.0.0.1:11434/api/generate/ -d '{
4
+ "model": "mistral",
5
+ "prompt":"Why is the sky blue?"
6
+ }'
7
+ """
8
+
9
+ import json
10
+ import requests
11
+ import ollama
12
+ from oshepherd.api.generate.models import GenerateResponse
13
+
14
+
15
+ def test_basic_api_worker_queueing_using_ollama():
16
+ params = {"model": "mistral", "prompt": "Why is the sky blue?"}
17
+ client = ollama.Client(host="http://127.0.0.1:5001")
18
+ ollama_res = client.generate(**params)
19
+
20
+ ollama_res = GenerateResponse(**ollama_res)
21
+ assert ollama_res.response, "response should not be empty"
22
+
23
+
24
+ def test_basic_api_worker_queueing_using_requests():
25
+ url = "http://127.0.0.1:5001/api/generate/"
26
+ headers = {"Content-Type": "application/json"}
27
+ data = {"model": "mistral", "prompt": "Why is the sky blue?"}
28
+ response = requests.post(url, headers=headers, data=json.dumps(data))
29
+
30
+ assert response.status_code == 200
31
+ assert "error" not in response
32
+ ollama_res = GenerateResponse(**response.json())
33
+ assert ollama_res.response, "response should not be empty"