opendataframework 0.0.2__tar.gz → 0.0.3__tar.gz

{opendataframework-0.0.2 → opendataframework-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: opendataframework
-Version: 0.0.2
+Version: 0.0.3
 Summary: Open Data Framework is an open source, full stack data framework.
 Home-page: https://github.com/opendataframework/opendataframework
 License: MIT

{opendataframework-0.0.2 → opendataframework-0.0.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "opendataframework"
-version = "0.0.2"
+version = "0.0.3"
 description = "Open Data Framework is an open source, full stack data framework."
 authors = [
     "mykytapavlov <mykytapavlov@users.noreply.github.com>",

{opendataframework-0.0.2 → opendataframework-0.0.3}/setup.py

@@ -7,7 +7,8 @@ package_dir = \
 packages = \
 ['opendataframework',
  'opendataframework.api',
- 'opendataframework.api.api-postgres.app']
+ 'opendataframework.api.api-postgres.app',
+ 'opendataframework.tests']
 
 package_data = \
 {'': ['*'],
@@ -16,6 +17,10 @@ package_data = \
                        'analytics/superset/database/postgres/*',
                        'devcontainers/R/.devcontainer/*',
                        'devcontainers/python/.devcontainer/*',
+                       'docs/*',
+                       'docs/stylesheets/*',
+                       'github/workflows/*',
+                       'hooks/*',
                        'layouts/research/.vscode/*',
                        'layouts/research/code/build/*',
                        'layouts/research/code/check/*',
@@ -32,6 +37,9 @@ package_data = \
                        'storage/*',
                        'storage/postgres/*',
                        'utility/*',
+                       'utility/nginx/*',
+                       'utility/nginx/static/*',
+                       'utility/nginx/static/images/*',
                        'utility/texlive/*',
                        'utility/texlive/mnt/*'],
  'opendataframework.api': ['api-postgres/*']}
@@ -44,7 +52,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'opendataframework',
-    'version': '0.0.2',
+    'version': '0.0.3',
     'description': 'Open Data Framework is an open source, full stack data framework.',
     'long_description': '# Open Data Framework\nOpen source, full stack data framework.\n',
     'author': 'mykytapavlov',

opendataframework-0.0.3/src/opendataframework/README.md

@@ -0,0 +1,43 @@
+# project_name
+
+### Build platform
+Command builds platform's images.
+
+```sh
+opendataframework build
+```
+
+### Start platform
+Command starts platform's containers.
+
+```sh
+opendataframework start
+```
+
+### Setup platform
+Command setup platform's containers.
+
+```sh
+opendataframework setup
+```
+
+### Install dependencies
+Command creates `myproject/.venv/` and installs dependencies from `requirements.txt`.
+
+```sh
+opendataframework install
+```
+
+### Docs
+Command starts mkdocs server.
+
+```sh
+opendataframework docs
+```
+
+### Test
+Command runs tests.
+
+```sh
+opendataframework test
+```

{opendataframework-0.0.2 → opendataframework-0.0.3}/src/opendataframework/__init__.py

@@ -1,3 +1,3 @@
 """Open Data Framework package."""
 
-__version__ = "0.0.2"
+__version__ = "0.0.3"

{opendataframework-0.0.2 → opendataframework-0.0.3}/src/opendataframework/__main__.py

@@ -5,8 +5,11 @@ import json
 import os
 import re
 import shutil
+import stat
+import subprocess
 import traceback
 import uuid
+import venv
 from datetime import datetime
 from pathlib import Path
 
@@ -106,6 +109,7 @@ class Component:
     POSTGRES: str = "postgres"
 
     # UTILITY
+    NGINX = "nginx"
     TEXLIVE = "texlive"
 
 
@@ -121,7 +125,7 @@ COMPONENTS = {
     Layer.DEVCONTAINERS: [Component.PYTHON, Component.R],
     Layer.API: [Component.API_POSTGRES],
     Layer.STORAGE: [Component.POSTGRES],
-    Layer.UTILITY: [Component.TEXLIVE],
+    Layer.UTILITY: [Component.NGINX, Component.TEXLIVE],
 }
 
 DEPENDENCIES = {
@@ -140,6 +144,7 @@ DESCRIPTIONS = {
     # STORAGE
     Component.POSTGRES: "Advanced Relational Database",
     # UTILITY
+    Component.NGINX: "HTTP and reverse proxy server",
     Component.TEXLIVE: "TeX Live is intended to be a straightforward way to get up and running with the TeX document production system",  # noqa
 }
 
@@ -151,6 +156,8 @@ PORTS = {
     Component.API_POSTGRES: "8000",
     # STORAGE
     Component.POSTGRES: "5432",
+    # UTILITY
+    Component.NGINX: "80",
 }
 
 
@@ -211,6 +218,14 @@ LAYOUTS = {Layout.CUSTOM, Layout.RESEARCH}
 PROFILES = {Profile.CUSTOM, Profile.RESEARCH}
 
 
+BADGES = {
+    Layer.ANALYTICS: "badge badge-info gap-2",
+    Layer.API: "badge badge-accent gap-2",
+    Layer.STORAGE: "badge badge-warning gap-2",
+    Layer.UTILITY: "badge badge-secondary gap-2",
+}
+
+
 class Field:
     """Field."""
 
@@ -690,6 +705,109 @@ class Project:
 
         rprint(f"{self.name}: {self.layout} layout[green] created[/green]")
 
+    def add_docs(self):
+        """Add project docs."""
+        from_path = os.path.join(SRC_PATH, "docs")
+        if not os.path.exists(from_path):
+            raise ValueError(f"{from_path} does not exist")
+
+        to_path = os.path.join(self.path, "docs")
+        if os.path.exists(to_path):
+            raise ValueError(f"{to_path} already exists")
+
+        shutil.copytree(
+            from_path,
+            to_path,
+            ignore=shutil.ignore_patterns(
+                *IGNORE_PATTERNS, *("mkdocs.yml", "requirements.txt")
+            ),
+        )
+
+        if os.path.exists(os.path.join(self.path, "requirements.txt")):
+            with open(os.path.join(from_path, "requirements.txt"), "r") as file:
+                src_file_data = file.read()
+            with open(os.path.join(self.path, "requirements.txt"), "a") as file:
+                file.write(src_file_data)
+        else:
+            self.copy(from_path, self.path, "requirements.txt")
+
+        self.copy(from_path, self.path, "mkdocs.yml")
+        self.replace(
+            os.path.join(self.path, "docs", "index.md"), PROJECT_NAME, self.name
+        )
+        self.replace(os.path.join(self.path, "mkdocs.yml"), PROJECT_NAME, self.name)
+
+        rprint(f"{self.name}: docs[green] created[/green]")
+
+    def add_hooks(self):
+        """Add project pre-commit hooks."""
+        from_path = os.path.join(SRC_PATH, "hooks")
+        if not os.path.exists(from_path):
+            raise ValueError(f"{from_path} does not exist")
+
+        if os.path.exists(os.path.join(self.path, ".pre-commit-config.yaml")):
+            raise ValueError(
+                f'{os.path.join(self.path, ".pre-commit-config.yaml")} already exists'
+            )
+
+        self.copy(from_path, self.path, ".pre-commit-config.yaml")
+
+        if os.path.exists(os.path.join(self.path, "requirements.txt")):
+            with open(os.path.join(from_path, "requirements.txt"), "r") as file:
+                src_file_data = file.read()
+            with open(os.path.join(self.path, "requirements.txt"), "a") as file:
+                file.write(src_file_data)
+        else:
+            self.copy(from_path, self.path, "requirements.txt")
+
+        rprint(f"{self.name}: .pre-commit-config.yaml[green] created[/green]")
+
+    def add_workflows(self):
+        """Add github workflows (github pages ci for docs)."""
+        from_path = os.path.join(SRC_PATH, "github")
+        if not os.path.exists(from_path):
+            raise ValueError(f"{from_path} does not exist")
+
+        to_path = os.path.join(self.path, ".github")
+        if os.path.exists(to_path):
+            raise ValueError(f"{to_path} already exists")
+
+        shutil.copytree(
+            from_path,
+            to_path,
+            ignore=shutil.ignore_patterns(
+                *IGNORE_PATTERNS,
+            ),
+        )
+
+        rprint(f"{self.name}: github workflows[green] created[/green]")
+
+    def add_tests(self):
+        """Add project tests."""
+        from_path = os.path.join(SRC_PATH, "tests")
+        if not os.path.exists(from_path):
+            raise ValueError(f"{from_path} does not exist")
+
+        to_path = os.path.join(self.path, "tests")
+        if os.path.exists(to_path):
+            raise ValueError(f"{to_path} already exists")
+
+        shutil.copytree(
+            from_path,
+            to_path,
+            ignore=shutil.ignore_patterns(*IGNORE_PATTERNS, *("requirements.txt",)),
+        )
+
+        if os.path.exists(os.path.join(self.path, "requirements.txt")):
+            with open(os.path.join(from_path, "requirements.txt"), "r") as file:
+                src_file_data = file.read()
+            with open(os.path.join(self.path, "requirements.txt"), "a") as file:
+                file.write(src_file_data)
+        else:
+            self.copy(from_path, self.path, "requirements.txt")
+
+        rprint(f"{self.name}: tests[green] created[/green]")
+
     def init(self):
         """Handler for `init` CLI command."""
         rprint()
@@ -804,10 +922,24 @@ class Project:
         rprint(f"{json.dumps(self.settings, indent=JSON_INDENT)}")
         rprint()
 
-    def create(self):
+    def create(
+        self,
+        docs: bool = True,
+        hooks: bool = True,
+        workflows: bool = False,
+        tests: bool = True,
+    ):
         """Handler for `create` CLI command."""
         self.from_json()
         self.add_layout()
+        if docs:
+            self.add_docs()
+        if hooks:
+            self.add_hooks()
+        if workflows:
+            self.add_workflows()
+        if tests:
+            self.add_tests()
 
         layers = [
             Analytics(project=self),
@@ -890,7 +1022,7 @@ class Project:
         if not os.path.exists(to_path):
             raise ValueError(f"{to_path} does not exist")
 
-        scripts = ["build.sh", "setup.sh", "start.sh", "stop.sh", "requirements.txt"]
+        scripts = ["setup.sh", "requirements.txt"]
 
         for file_name in scripts:
             if file_name == "requirements.txt":
@@ -905,14 +1037,10 @@ class Project:
                 self.copy(from_path, to_path, file_name)
 
         self.copy(from_path, to_path, "docker-compose.yaml")
-
-        self.copy(from_path, self.path, "env.sh")
         self.copy(from_path, self.path, "expectations.py")
-        self.copy(from_path, self.path, "main.sh")
         self.copy(from_path, self.path, "README.md")
 
         self.replace(os.path.join(self.path, "README.md"), f"{PROJECT_NAME}", self.name)
-        self.replace(os.path.join(self.path, "main.sh"), f"{PROJECT_NAME}", self.name)
 
         for layer in COMPONENTS:
             if layer is Layer.DEVCONTAINERS:
@@ -944,11 +1072,20 @@ class Project:
                 if lines and layer == Layer.STORAGE and file_name == "setup.sh":
                     # put storage setup logic on top of setup.sh
                     with open(f"{to_path}/{file_name}", "r") as file:
-                        current_file_data = file.read()
-
+                        current_file_data = file.readlines()
+                        # keep shebang on top of the file
+                        header, current_file_data = (
+                            current_file_data[:3],
+                            current_file_data[3:],
+                        )
+                        header = [line for line in header if line != "\n"]
+                        header = "".join([*header])
+                        current_file_data = "".join([*current_file_data])
                     with open(f"{to_path}/{file_name}", "w") as file:
                         file.write(
-                            "\n".join([src_file_data, *lines, current_file_data])
+                            "\n".join(
+                                [header, src_file_data, *lines, current_file_data]
+                            )
                         )
 
                 else:
@@ -1398,6 +1535,113 @@ class Utility:
         """Create Utility layer instance."""
         self.project = project
 
+    def nginx(self):
+        """Configure Analytics `NGINX` component."""
+        from_path = os.path.join(SRC_PATH, Layer.UTILITY, Component.NGINX)
+        if not os.path.exists(from_path):
+            raise ValueError(f"{from_path} does not exist")
+
+        to_path = os.path.join(
+            self.project.path, PLATFORM_FOLDER, Layer.UTILITY, Component.NGINX
+        )
+        if os.path.exists(to_path):
+            raise ValueError(f"{to_path} already exists")
+
+        entities = self.project.settings.get("entities", {})
+
+        for plural_name, settings in entities.items():
+            components = settings["layers"].get(Layer.UTILITY, {})
+            if Component.NGINX not in components:
+                continue
+
+            if os.path.exists(to_path):
+                raise ValueError(f"{to_path} already exists")
+
+            shutil.copytree(
+                from_path,
+                to_path,
+                ignore=shutil.ignore_patterns(*IGNORE_PATTERNS),
+            )
+
+            break
+
+        if not os.path.exists(to_path):
+            return
+
+        Project.replace(
+            os.path.join(to_path, "docker-compose.yaml"),
+            PROJECT_NAME,
+            self.project.settings["project"],
+        )
+
+        ports = self.project.settings.get("ports", {})
+
+        Project.replace(
+            os.path.join(to_path, "docker-compose.yaml"),
+            f"{PORTS[Component.NGINX]}:",
+            f"{ports[Component.NGINX]}:",
+        )
+
+        host = "http://localhost"
+        target_path = os.path.join(to_path, "static", "index.html")
+
+        Project.replace(target_path, PROJECT_NAME, self.project.settings["project"])
+
+        with open(target_path, "r") as file:
+            filedata = file.read()
+
+        header = filedata.split("<tbody>")[0]
+        template, _ = filedata.split("<tbody>")[1].split("</tbody>")
+        footer = filedata.split("</tbody>")[1]
+
+        lines = [header]
+
+        for component, port in ports.items():
+            layer = None
+            for current_layer, components in COMPONENTS.items():
+                if component in components:
+                    layer = current_layer
+                    break
+            if not layer:
+                continue
+
+            tr = template.replace(
+                'class="badge badge-info gap-2">layer_name',
+                f'class="{BADGES[layer]}">{layer}',
+            )
+            tr = tr.replace("component_name", f"{component}")
+            tr = tr.replace("http://host:port/", f"{host}:{port}/")
+            lines.append(tr)
+
+        entities = self.project.settings.get("entities", {})
+
+        for plural_name, settings in entities.items():
+            layers = settings.get("layers", {})
+
+            if Component.NGINX not in layers.get(Layer.UTILITY, {}):
+                continue
+
+            for layer, components in layers.items():
+                for component in components:
+                    port = components[component].get("port")
+                    if not port:
+                        continue
+
+                    tr = template.replace(
+                        'class="badge badge-info gap-2">layer_name',
+                        f'class="{BADGES[layer]}">{layer}',
+                    )
+                    tr = tr.replace("component_name", f"{component} | {plural_name}")
+                    tr = tr.replace("http://host:port/", f"{host}:{port}/")
+                    lines.append(tr)
+
+        lines.append(footer)
+        filedata = "\n".join(lines)
+        with open(target_path, "w") as file:
+            file.write(filedata)
+
+        rprint(f"{to_path}[green] created[/green]")
+
     def texlive(self):
         """Configure Analytics `TEXLIVE` component."""
         from_path = os.path.join(SRC_PATH, Layer.UTILITY, Component.TEXLIVE)
@@ -1447,6 +1691,7 @@ class Utility:
 
     def __call__(self):
         """Call layer."""
+        self.nginx()
         self.texlive()
 
 
@@ -1507,15 +1752,228 @@ def init(
 
 
 @app.command()
-def create(project: str, path: str = ""):
+def create(
+    project: str,
+    path: str = "",
+    docs: bool = True,
+    hooks: bool = True,
+    workflows: bool = False,
+    tests: bool = True,
+):
     """Create PROJECT structure based on settings.json, optionally with a --path."""
     try:
         project = Project(name=project, path=path)
-        project.create()
+        project.create(docs=docs, hooks=hooks, workflows=workflows, tests=tests)
     except Exception:
         rprint(f"[bold red] {traceback.format_exc()} [/bold red]")
 
 
+@app.command()
+def install(project: str = "", path: str = ""):
+    """Install dependencies into .venv from requirements.txt under cwd/path."""
+    try:
+        if path and not os.path.exists(path):
+            raise ValueError(f"{path} does not exists")
+        elif not path:
+            path = os.getcwd()
+        path = os.path.join(path, project)
+        venv_path = os.path.join(path, ".venv")
+        if os.path.exists(venv_path):
+            raise ValueError(f"{venv_path} already exists")
+
+        requirements_path = os.path.join(path, "requirements.txt")
+        if not os.path.exists(requirements_path):
+            raise ValueError(f"{requirements_path} not exists")
+
+        venv.create(venv_path, with_pip=True)
+        subprocess.run(["bin/pip", "install", "-r", requirements_path], cwd=venv_path)
+    except Exception as e:
+        rprint(f"[bold red] {e} [/bold red]")
+
+
+@app.command()
+def build(project: str = "", path: str = ""):
+    """Run `docker compose --profile {layer} build`."""
+    try:
+        if path and not os.path.exists(path):
+            raise ValueError(f"{path} does not exists")
+        elif not path:
+            path = os.getcwd()
+        path = os.path.join(path, project, "platform")
+
+        compose_path = os.path.join(path, "docker-compose.yaml")
+        if not os.path.exists(compose_path):
+            raise ValueError(f"{compose_path} not exists")
+
+        for layer in COMPONENTS:
+            subprocess.run(
+                ["docker", "compose", "--profile", f"{layer}", "build"], cwd=path
+            )
+
+    except Exception as e:
+        rprint(f"[bold red] {e} [/bold red]")
+
+
+@app.command()
+def start(project: str = "", path: str = ""):
+    """Run `docker compose --profile {layer} up -d`."""
+    try:
+        if path and not os.path.exists(path):
+            raise ValueError(f"{path} does not exists")
+        elif not path:
+            path = os.getcwd()
+        path = os.path.join(path, project, "platform")
+
+        compose_path = os.path.join(path, "docker-compose.yaml")
+        if not os.path.exists(compose_path):
+            raise ValueError(f"{compose_path} not exists")
+
+        for layer in COMPONENTS:
+            subprocess.run(
+                ["docker", "compose", "--profile", f"{layer}", "up", "-d"], cwd=path
+            )
+
+    except Exception as e:
+        rprint(f"[bold red] {e} [/bold red]")
+
+
+@app.command()
+def stop(project: str = "", path: str = ""):
+    """Run `docker compose --profile {layer} stop`."""
+    try:
+        if path and not os.path.exists(path):
+            raise ValueError(f"{path} does not exists")
+        elif not path:
+            path = os.getcwd()
+        path = os.path.join(path, project, "platform")
+
+        compose_path = os.path.join(path, "docker-compose.yaml")
+        if not os.path.exists(compose_path):
+            raise ValueError(f"{compose_path} not exists")
+
+        for layer in COMPONENTS:
+            subprocess.run(
+                ["docker", "compose", "--profile", f"{layer}", "stop"], cwd=path
+            )
+
+    except Exception as e:
+        rprint(f"[bold red] {e} [/bold red]")
+
+
+@app.command()
+def status(project: str = "", path: str = ""):
+    """Run `docker compose ps`."""
+    try:
+        if path and not os.path.exists(path):
+            raise ValueError(f"{path} does not exists")
+        elif not path:
+            path = os.getcwd()
+        path = os.path.join(path, project, "platform")
+
+        compose_path = os.path.join(path, "docker-compose.yaml")
+        if not os.path.exists(compose_path):
+            raise ValueError(f"{compose_path} not exists")
+
+        subprocess.run(["docker", "compose", "ps", "--all"], cwd=path)
+
+    except Exception as e:
+        rprint(f"[bold red] {e} [/bold red]")
+
+
+@app.command()
+def setup(project: str = "", path: str = ""):
+    """Run `platform/setup.sh`."""
+    try:
+        if path and not os.path.exists(path):
+            raise ValueError(f"{path} does not exists")
+        elif not path:
+            path = os.getcwd()
+        path = os.path.join(path, project, "platform")
+
+        setup_path = os.path.join(path, "setup.sh")
+        if not os.path.exists(setup_path):
+            raise ValueError(f"{setup_path} not exists")
+
+        os.chmod(setup_path, os.stat(setup_path).st_mode | stat.S_IEXEC)
+        subprocess.run(["./setup.sh"], cwd=path)
+
+    except Exception as e:
+        rprint(f"[bold red] {e} [/bold red]")
+
+
+@app.command()
+def check(project: str = "", path: str = ""):
+    """Run `pre-commit run --all-files`."""
+    try:
+        if path and not os.path.exists(path):
+            raise ValueError(f"{path} does not exist")
+        elif not path:
+            path = os.getcwd()
+        path = os.path.join(path, project)
+        venv_path = os.path.join(path, ".venv")
+        if not os.path.exists(venv_path):
+            raise ValueError(f"{venv_path} does not exist")
+
+        pre_commit_path = os.path.join(path, ".pre-commit-config.yaml")
+        if not os.path.exists(pre_commit_path):
+            raise ValueError(f"{pre_commit_path} does not exist")
+
+        subprocess.run([".venv/bin/pre-commit", "run", "--all-files"], cwd=path)
+    except Exception as e:
+        rprint(f"[bold red] {e} [/bold red]")
+
+
+@app.command()
+def docs(project: str = "", path: str = ""):
+    """Run `mkdocs serve`."""
+    try:
+        if path and not os.path.exists(path):
+            raise ValueError(f"{path} does not exist")
+        elif not path:
+            path = os.getcwd()
+        path = os.path.join(path, project)
+        venv_path = os.path.join(path, ".venv")
+        if not os.path.exists(venv_path):
+            raise ValueError(f"{venv_path} does not exist")
+
+        docs_path = os.path.join(path, "mkdocs.yml")
+        if not os.path.exists(docs_path):
+            raise ValueError(f"{docs_path} does not exist")
+
+        subprocess.run([".venv/bin/mkdocs", "serve"], cwd=path)
+    except Exception as e:
+        rprint(f"[bold red] {e} [/bold red]")
+
+
+@app.command()
+def test(project: str = "", path: str = "", cov: str = ""):
+    """Run `pytest --cov={cov} tests`."""
+    try:
+        if path and not os.path.exists(path):
+            raise ValueError(f"{path} does not exist")
+        elif not path:
+            path = os.getcwd()
+        path = os.path.join(path, project)
+        venv_path = os.path.join(path, ".venv")
+        if not os.path.exists(venv_path):
+            raise ValueError(f"{venv_path} does not exist")
+
+        test_path = os.path.join(path, "tests")
+        if not os.path.exists(test_path):
+            raise ValueError(f"{test_path} does not exist")
+
+        if cov:
+            cov_path = os.path.join(path, cov)
+            if not os.path.exists(cov_path):
+                raise ValueError(f"{cov_path} does not exist")
+
+            subprocess.run([".venv/bin/pytest", f"--cov={cov}", "tests"], cwd=path)
+        else:
+            subprocess.run([".venv/bin/pytest", "tests"], cwd=path)
+    except Exception as e:
+        rprint(f"[bold red] {e} [/bold red]")
+
+
 def main():
     """Main function which starts the app."""
     rprint(colorized_logo())