opendataframework 0.0.3__tar.gz → 0.0.4__tar.gz

{opendataframework-0.0.3 → opendataframework-0.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opendataframework
-Version: 0.0.3
+Version: 0.0.4
 Summary: Open Data Framework is an open source, full stack data framework.
 Home-page: https://github.com/opendataframework/opendataframework
 License: MIT

{opendataframework-0.0.3 → opendataframework-0.0.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "opendataframework"
-version = "0.0.3"
+version = "0.0.4"
 description = "Open Data Framework is an open source, full stack data framework."
 authors = [
     "mykytapavlov <mykytapavlov@users.noreply.github.com>",

{opendataframework-0.0.3 → opendataframework-0.0.4}/setup.py

@@ -8,6 +8,7 @@ packages = \
 ['opendataframework',
  'opendataframework.api',
  'opendataframework.api.api-postgres.app',
+ 'opendataframework.api.inference.app',
  'opendataframework.tests']
 
 package_data = \
@@ -42,7 +43,7 @@ package_data = \
                        'utility/nginx/static/images/*',
                        'utility/texlive/*',
                        'utility/texlive/mnt/*'],
- 'opendataframework.api': ['api-postgres/*']}
+ 'opendataframework.api': ['api-postgres/*', 'inference/*']}
 
 install_requires = \
 ['typer>=0.12.3,<0.13.0']
@@ -52,7 +53,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'opendataframework',
-    'version': '0.0.3',
+    'version': '0.0.4',
     'description': 'Open Data Framework is an open source, full stack data framework.',
     'long_description': '# Open Data Framework\nOpen source, full stack data framework.\n',
     'author': 'mykytapavlov',

{opendataframework-0.0.3 → opendataframework-0.0.4}/src/opendataframework/__init__.py

@@ -1,3 +1,3 @@
 """Open Data Framework package."""
 
-__version__ = "0.0.3"
+__version__ = "0.0.4"

{opendataframework-0.0.3 → opendataframework-0.0.4}/src/opendataframework/__main__.py

@@ -100,6 +100,7 @@ class Component:
 
     # API
     API_POSTGRES: str = "api-postgres"
+    INFERENCE: str = "inference"
 
     # DEVCONTAINERS
     PYTHON: str = "python"
@@ -123,7 +124,7 @@ FILE_FORMATS = {
 COMPONENTS = {
     Layer.ANALYTICS: [Component.SUPERSET],
     Layer.DEVCONTAINERS: [Component.PYTHON, Component.R],
-    Layer.API: [Component.API_POSTGRES],
+    Layer.API: [Component.API_POSTGRES, Component.INFERENCE],
     Layer.STORAGE: [Component.POSTGRES],
     Layer.UTILITY: [Component.NGINX, Component.TEXLIVE],
 }
@@ -154,6 +155,7 @@ PORTS = {
     Component.SUPERSET: "8088",
     # API
     Component.API_POSTGRES: "8000",
+    Component.INFERENCE: "8000",
     # STORAGE
     Component.POSTGRES: "5432",
     # UTILITY
@@ -431,16 +433,20 @@ class Entity:
 class Project:
     """Project."""
 
-    def __init__(self, name: str, path: str = "", data: str = ""):
+    def __init__(self, name: str, path: str = "", data: str = "", models: str = ""):
         """Create project instance."""
         self._name = None
         self._path = None
         self._profile = Profile.CUSTOM
         self._layout = Layout.CUSTOM
+        self._models = None
+
         if data:
             self._data = os.path.join(os.getcwd(), data)
         else:
             self._data = os.path.join(os.getcwd(), "data")
+        if models:
+            self._models = os.path.join(os.getcwd(), models)
 
         self._settings = {
             "opendataframework": __version__,
@@ -505,6 +511,34 @@ class Project:
 
         self._path = path
 
+        if not self._models:
+            models = os.path.join(os.getcwd(), "models")
+            if not os.path.exists(models):
+                return
+            self._models = models
+
+        models_path = os.path.join(path, "models")
+        if not os.path.exists(models_path):
+            os.mkdir(models_path)
+            rprint(f"{models_path}[green] created[/green]")
+
+            for folder in os.listdir(self._models):
+                source_path = os.path.join(self._models, folder)
+                if os.path.isdir(source_path):
+                    dest_path = os.path.join(models_path, folder)
+                    if not os.path.exists(dest_path):
+                        os.mkdir(dest_path)
+
+                    for file_name in os.listdir(source_path):
+                        supported = [
+                            file_name.endswith(".pkl"),
+                            file_name.strip().lower() == "requirements.txt",
+                        ]
+                        if not any(supported):
+                            continue
+                        self.copy(source_path, dest_path, file_name)
+                        rprint(f"{file_name}[green] copied to [/green]{dest_path}")
+
     @property
     def settings(self) -> dict:
         """Get project settings."""
@@ -1340,9 +1374,134 @@ class API:
 
             rprint(f"{to_path}[green] created[/green]")
 
+    def inference(self):
+        """Configure API `INFERENCE` component."""
+        from_path = os.path.join(SRC_PATH, Layer.API, Component.INFERENCE)
+        if not os.path.exists(from_path):
+            raise ValueError(f"{from_path} does not exist")
+
+        entities = self.project.settings.get("entities", {})
+
+        for plural_name, settings in entities.items():
+            components = settings["layers"].get(Layer.API, {})
+            if Component.INFERENCE not in components:
+                continue
+
+            to_path = os.path.join(
+                self.project.path,
+                PLATFORM_FOLDER,
+                Layer.API,
+                Component.INFERENCE,
+                plural_name,
+            )
+            if os.path.exists(to_path):
+                raise ValueError(f"{to_path} already exists")
+
+            shutil.copytree(
+                from_path,
+                to_path,
+                ignore=shutil.ignore_patterns(
+                    *IGNORE_PATTERNS,
+                ),
+            )
+
+            model_path = os.path.join(self.project.path, "models", plural_name)
+            if not os.path.exists(os.path.join(model_path, "model.pkl")):
+                raise ValueError(f"{model_path} does not exist")
+
+            Project.copy(model_path, os.path.join(to_path, "app"), "model.pkl")
+
+            if os.path.exists(os.path.join(model_path, "requirements.txt")):
+                with open(os.path.join(model_path, "requirements.txt"), "r") as file:
+                    filedata = file.read()
+                    new_filedata = []
+                    for line in filedata.split("\n"):
+                        if not line:
+                            continue
+                        packet, version = line.split("==")
+                        new_line = f'{packet} = "{version}"'
+                        new_filedata.append(new_line)
+                    filedata = "\n".join(new_filedata)
+
+                    Project.replace(
+                        os.path.join(to_path, "pyproject.toml"),
+                        "# dependencies",
+                        filedata,
+                    )
+
+            hostname = self.project.settings["project"].replace("_", "-")
+
+            Project.replace(
+                os.path.join(to_path, "docker-compose.yaml"),
+                f"hostname: {PROJECT_NAME}-{Component.INFERENCE}",
+                f"hostname: {hostname}-{Component.INFERENCE}",
+            )
+
+            Project.replace(
+                os.path.join(to_path, "docker-compose.yaml"),
+                f"{PROJECT_NAME}",
+                self.project.settings["project"],
+            )
+
+            Project.replace(
+                os.path.join(to_path, "docker-compose.yaml"), "entity", plural_name
+            )
+
+            port = components[Component.INFERENCE]["port"]
+            Project.replace(
+                os.path.join(to_path, "docker-compose.yaml"),
+                f"{PORTS[Component.INFERENCE]}:",
+                f"{port}:",
+            )
+
+            # model
+            model_path = os.path.join(to_path, "app", "models.py")
+            new_text = "# fields"
+
+            for field_name, field_type in settings["fields"].items():
+                if field_name in Field.RESERVED_FIELDS:
+                    raise ValueError(
+                        f"Field names `{self.RESERVED_FIELDS}` are reserved"
+                    )
+                if "datetime" in field_type:
+                    # TODO: format validator
+                    field_type = "datetime"
+
+                new_text += f"\n    {field_name}: {field_type}"
+
+            Project.replace(model_path, "# extra fields", new_text)
+            Project.replace(model_path, "entities", plural_name)
+            Project.replace(model_path, "Entity", settings["name"].capitalize())
+
+            # crud
+            crud_path = os.path.join(to_path, "app", "crud.py")
+            Project.replace(crud_path, "entity", settings["name"])
+            Project.replace(crud_path, "entities", plural_name)
+            Project.replace(crud_path, "Entity", settings["name"].capitalize())
+
+            # router
+            router_path = os.path.join(to_path, "app", "router.py")
+            Project.replace(router_path, "entity", settings["name"])
+            Project.replace(router_path, "entities", plural_name)
+            Project.replace(router_path, "Entity", settings["name"].capitalize())
+
+            # env
+            env_path = os.path.join(to_path, ".env")
+            Project.replace(
+                env_path, f"{PROJECT_NAME}", self.project.settings["project"]
+            )
+            Project.replace(env_path, "description", settings["description"])
+
+            # main
+            main_path = os.path.join(to_path, "app", "main.py")
+            Project.replace(main_path, "entity", settings["name"])
+
+            rprint(f"{to_path}[green] created[/green]")
+
     def __call__(self):
         """Call layer."""
         self.api_postgres()
+        self.inference()
 
 
 class Devcontainers:

opendataframework-0.0.4/src/opendataframework/api/inference/.env

@@ -0,0 +1,7 @@
+API_V1_PREFIX="/api/v1"
+
+DEBUG=True
+DESCRIPTION="description"
+
+PROJECT_NAME="project_name"
+VERSION="0.1.0"

opendataframework-0.0.4/src/opendataframework/api/inference/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.11.2
+WORKDIR /api
+
+COPY .env pyproject.toml ./
+
+
+ARG poetry_version=1.3.2
+RUN pip install poetry==${poetry_version}
+RUN poetry config virtualenvs.create false
+RUN poetry install
+
+COPY app/ app/
+
+EXPOSE 8000
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

opendataframework-0.0.4/src/opendataframework/api/inference/app/__init__.py

@@ -0,0 +1,11 @@
+"""App package."""
+
+from os import getenv
+
+from dotenv import load_dotenv
+
+from app.config import Settings
+
+load_dotenv(getenv("ENV_FILE"))
+
+settings = Settings()

opendataframework-0.0.4/src/opendataframework/api/inference/app/config.py

@@ -0,0 +1,14 @@
+"""Config module."""
+
+from pydantic import BaseSettings
+
+
+class Settings(BaseSettings):
+    """Settings."""
+
+    # Base
+    api_v1_prefix: str
+    debug: bool
+    project_name: str
+    version: str
+    description: str

opendataframework-0.0.4/src/opendataframework/api/inference/app/crud.py

@@ -0,0 +1,14 @@
+"""CRUD module."""
+
+from typing import Any
+
+import pandas as pd
+
+from app.models import Entity
+
+
+def predict(entity: Entity, inference: Any) -> float:
+    """Predict."""
+    data = pd.DataFrame(entity.dict(), index=[0])
+    predictions = inference.predict(data)
+    return predictions[0]

opendataframework-0.0.4/src/opendataframework/api/inference/app/dependencies.py

@@ -0,0 +1,19 @@
+"""Dependencies module."""
+
+import os
+import pickle
+from pathlib import Path
+
+MODEL_PATH = os.path.join(Path(__file__).parent, "model.pkl")
+INFERENCE_MODEL = None
+
+
+def get_inference():
+    """Get inference model."""
+    global INFERENCE_MODEL
+
+    if INFERENCE_MODEL is None:
+        with open(MODEL_PATH, "rb") as file:
+            INFERENCE_MODEL = pickle.load(file)  # nosec
+
+    yield INFERENCE_MODEL

opendataframework-0.0.4/src/opendataframework/api/inference/app/main.py

@@ -0,0 +1,39 @@
+"""Main module."""
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from app import settings
+from app.models import HealthCheck
+from app.router import inference_router
+
+app = FastAPI(
+    title=settings.project_name,
+    version=settings.version,
+    openapi_url=f"{settings.api_v1_prefix}/openapi.json",
+    debug=settings.debug,
+)
+
+
+origins = ["http://localhost:8080"]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@app.get("/", response_model=HealthCheck, tags=["status"])
+async def health_check():
+    """Health check."""
+    return {
+        "name": settings.project_name,
+        "version": settings.version,
+        "description": settings.description,
+    }
+
+
+app.include_router(inference_router, prefix=settings.api_v1_prefix)

opendataframework-0.0.4/src/opendataframework/api/inference/app/models.py

@@ -0,0 +1,20 @@
+"""Models module."""
+
+from pydantic import BaseModel
+from sqlmodel import SQLModel
+
+
+class HealthCheck(BaseModel):
+    """Health check model."""
+
+    name: str
+    version: str
+    description: str
+
+
+class Entity(SQLModel):
+    """Entity model."""
+
+    __tablename__ = "entities"
+
+    # extra fields

opendataframework-0.0.4/src/opendataframework/api/inference/app/router.py

@@ -0,0 +1,18 @@
+"""Router module."""
+
+from typing import Any
+
+from fastapi import APIRouter, Depends, status
+
+from app.crud import predict
+from app.dependencies import get_inference
+from app.models import Entity
+
+inference_router = APIRouter(prefix="/inference", tags=["inference"])
+
+
+@inference_router.post("/", status_code=status.HTTP_200_OK)
+def get_prediction(entity: Entity, inference: Any = Depends(get_inference)) -> float:
+    """Create prediction."""
+    prediction = predict(entity, inference)
+    return prediction

opendataframework-0.0.4/src/opendataframework/api/inference/docker-compose.yaml

@@ -0,0 +1,16 @@
+version: "3.9"
+services:
+  project_name_entity:
+    profiles: ["api"]
+    image: project_name:entity
+    container_name: project_name_entity
+    hostname: project_name-entity
+    build:
+      context: ./api/inference/entity
+      args:
+        poetry_version: "1.3.2"
+    ports:
+      - 8000:8000
+    networks:
+      - project_name_default
+    restart: always

opendataframework-0.0.4/src/opendataframework/api/inference/pyproject.toml

@@ -0,0 +1,20 @@
+[tool.poetry]
+name = "api"
+version = "0.1.0"
+description = ""
+authors = [""]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+fastapi = "^0.92.0"
+sqlmodel = "^0.0.8"
+python-dotenv = "^0.21.1"
+uvicorn = "^0.20.0"
+pandas = "^2.2.3"
+# dependencies
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"