zrb 0.0.45__py3-none-any.whl → 0.0.47__py3-none-any.whl

zrb/builtin/generator/fastapp_module/add.py

@@ -1,4 +1,5 @@
 from typing import Any, List
+from dotenv import dotenv_values
 from ..._group import project_add_group
 from ....task.task import Task
 from ....task.decorator import python_task
@@ -12,11 +13,15 @@ from .._common import (
 from ....helper import util
 from ....helper.codemod.add_import_module import add_import_module
 from ....helper.codemod.add_function_call import add_function_call
+from ....helper.docker_compose.file import add_services
+from ....helper.file.text import (
+    read_text_file_async, write_text_file_async, append_text_file_async
+)
 
+import asyncio
 import os
 import jsons
 
-
 current_dir = os.path.dirname(__file__)
 
 
@@ -66,120 +71,298 @@ copy_resource = ResourceMaker(
 
 
 @python_task(
-    name='register-module',
-    inputs=[project_dir_input, app_name_input, module_name_input],
-    upstreams=[copy_resource]
+    name='fastapp-module',
+    group=project_add_group,
+    upstreams=[copy_resource],
+    runner=runner
 )
-def register_module(*args: Any, **kwargs: Any):
+async def add_fastapp_module(*args: Any, **kwargs: Any):
     task: Task = kwargs.get('_task')
     project_dir = kwargs.get('project_dir', '.')
     app_name = kwargs.get('app_name')
     module_name = kwargs.get('module_name')
     kebab_app_name = util.to_kebab_case(app_name)
+    snake_app_name = util.to_snake_case(app_name)
+    kebab_module_name = util.to_kebab_case(module_name)
     snake_module_name = util.to_snake_case(module_name)
-    import_module_path = '.'.join([
-        'module', snake_module_name
+    upper_snake_module_name = snake_module_name.upper()
+    app_main_file = get_app_main_file(project_dir, kebab_app_name)
+    app_config_file = get_app_config_file(project_dir, kebab_app_name)
+    json_modules_file = get_json_modules_file(project_dir, snake_app_name)
+    enabled_env_file = get_all_enabled_env_file(project_dir, kebab_app_name)
+    disabled_env_file = get_all_disabled_env_file(project_dir, kebab_app_name)
+    docker_compose_file = get_docker_compose_file(project_dir, kebab_app_name)
+    results = await asyncio.gather(*[
+        asyncio.create_task(create_automation_json_config(
+            task, json_modules_file, snake_module_name
+        )),
+        asyncio.create_task(register_module(
+            task, app_main_file, snake_module_name
+        )),
+        asyncio.create_task(create_app_config(
+            task, app_config_file, snake_module_name, upper_snake_module_name
+        )),
+        asyncio.create_task(append_all_enabled_env(
+            task, enabled_env_file, upper_snake_module_name
+        )),
+        asyncio.create_task(append_all_disabled_env(
+            task, disabled_env_file, upper_snake_module_name
+        )),
     ])
-    function_name = f'register_{snake_module_name}'
-    main_file_path = os.path.join(
-        project_dir, 'src', kebab_app_name, 'src', 'main.py'
+    modules = results[0]  # return value of `create_automation_json_config`
+    module_port = 8080 + len(modules)
+    src_template_env_path = get_src_template_env_file(
+        project_dir, kebab_app_name
     )
-    task.print_out(f'Read code from: {main_file_path}')
-    with open(main_file_path, 'r') as f:
-        code = f.read()
-    task.print_out(
-        f'Add import "{function_name}" from "{import_module_path}" to the code'
+    src_template_env_map = dotenv_values(src_template_env_path)
+    app_port = int(src_template_env_map.get('APP_PORT', '8080'))
+    module_app_port = app_port + len(modules)
+    compose_env_path = get_compose_env_file(
+        project_dir, snake_app_name
     )
-    code = add_import_module(
-        code=code,
-        module_path=import_module_path,
-        resource=function_name
+    compose_env_map = dotenv_values(compose_env_path)
+    host_port = int(compose_env_map.get('APP_GATEWAY_HOST_PORT', '8080'))
+    module_host_port = host_port + len(modules)
+    await asyncio.gather(*[
+        asyncio.create_task(add_docker_compose_service(
+            task, module_port, docker_compose_file, kebab_app_name,
+            snake_app_name, kebab_module_name, snake_module_name,
+            upper_snake_module_name
+        )),
+        asyncio.create_task(append_src_template_env(
+            task, module_app_port, src_template_env_path,
+            upper_snake_module_name
+        )),
+        asyncio.create_task(append_compose_env(
+            task, module_host_port, compose_env_path,
+            upper_snake_module_name
+        ))
+    ])
+    # TODO: create runner for local microservices
+    # TODO: create deployment for local microservices
+    task.print_out('Success')
+
+
+async def add_docker_compose_service(
+    task: Task,
+    module_app_port: int,
+    docker_compose_file_path: str,
+    kebab_app_name: str,
+    snake_app_name: str,
+    kebab_module_name: str,
+    snake_module_name: str,
+    upper_snake_module_name: str
+):
+    module_app_port_str = str(module_app_port)
+    app_container_port_env_name = f'APP_{upper_snake_module_name}_MODULE_PORT'
+    app_container_port_env = '${' + app_container_port_env_name + ':-' + module_app_port_str + '}' # noqa
+    app_host_port_env_name = f'APP_{upper_snake_module_name}_HOST_MODULE_PORT'
+    app_host_port_env = '${' + app_host_port_env_name + ':-' + module_app_port_str + '}' # noqa
+    task.print_out(f'Add service at: {docker_compose_file_path}')
+    add_services(
+        file_name=docker_compose_file_path,
+        new_services={
+            f'{kebab_app_name}-{kebab_module_name}-module': {
+                'build': {
+                    'dockerfile': 'Dockerfile',
+                    'context': './src'
+                },
+                'image': '${IMAGE:-' + kebab_app_name + '}',
+                'container_name': f'{snake_app_name}_{snake_module_name}',
+                'hostname': f'snake_app_name_{snake_module_name}',
+                'env_file': [
+                    'src/template.env',
+                    'all-module-disabled.env'
+                ],
+                'environment': {
+                    'APP_NAME': '${APP_NAME:-' + kebab_app_name + '}-' + f'{kebab_module_name}-module', # noqa
+                    'APP_PORT': app_container_port_env,  # noqa
+                    'APP_RMQ_CONNECTION': '${APP_CONTAINER_RMQ_CONNECTION:-amqp://guest:guest@rabbitmq/}', # noqa
+                    'APP_KAFKA_BOOTSTRAP_SERVERS': '${APP_CONTAINER_KAFKA_BOOTSTRAP_SERVERS:-redpanda:9092}', # noqa
+                    'APP_ENABLE_MESSAGE_CONSUMER': 'true',
+                    'APP_ENABLE_RPC_SERVER': 'true',
+                    'APP_ENABLE_API': 'false',
+                    'APP_ENABLE_FRONTEND': 'false',
+                    f'APP_ENABLE_{upper_snake_module_name}_MODULE': 'true',
+                },
+                'ports': [
+                    f'{app_host_port_env}:{app_container_port_env}'
+                ],
+                'restart': 'on-failure',
+                'profiles': [
+                    'microservices'
+                ],
+                'healthcheck': {
+                    'test': [
+                        "CMD-SHELL",
+                        "curl --fail http://localhost:${APP_PORT:-8080}/ || exit 1" # noqa
+                    ],
+                    'interval': '5s',
+                    'timeout': '1s',
+                    'retries': 30
+                }
+            }
+        }
     )
-    task.print_out(f'Add "{function_name}" call to the code')
-    code = add_function_call(
-        code=code,
-        function_name=function_name,
-        parameters=[]
+
+
+async def append_compose_env(
+    task: Task,
+    module_app_port: int,
+    compose_template_env_path: str,
+    upper_snake_module_name: str
+):
+    new_env_str = '\n'.join([
+        f'APP_{upper_snake_module_name}_HOST_MODULE_PORT={module_app_port}',
+        f'APP_{upper_snake_module_name}_MODULE_PORT={module_app_port}',
+    ])
+    task.print_out(f'Add new environment to: {compose_template_env_path}')
+    await append_text_file_async(compose_template_env_path, new_env_str)
+
+
+async def append_src_template_env(
+    task: Task,
+    module_app_port: int,
+    src_template_env_path: str,
+    upper_snake_module_name: str
+):
+    new_env_str = '\n'.join([
+        f'APP_ENABLE_{upper_snake_module_name}_MODULE=true',
+    ])
+    task.print_out(f'Add new environment to: {src_template_env_path}')
+    await append_text_file_async(src_template_env_path, new_env_str)
+
+
+async def append_all_enabled_env(
+    task: Task,
+    all_enabled_env_path: str,
+    upper_snake_module_name: str
+):
+    task.print_out(f'Add new environment to: {all_enabled_env_path}')
+    await append_text_file_async(
+        all_enabled_env_path,
+        f'APP_ENABLE_{upper_snake_module_name}_MODULE=true'
     )
-    task.print_out(f'Write modified code to: {main_file_path}')
-    with open(main_file_path, 'w') as f:
-        f.write(code)
 
 
-@python_task(
-    name='create-app-config',
-    inputs=[project_dir_input, app_name_input, module_name_input],
-    upstreams=[copy_resource]
-)
-def create_app_config(*args: Any, **kwargs: Any):
-    task: Task = kwargs.get('_task')
-    project_dir = kwargs.get('project_dir', '.')
-    app_name = kwargs.get('app_name')
-    module_name = kwargs.get('module_name')
-    kebab_app_name = util.to_kebab_case(app_name)
-    snake_module_name = util.to_snake_case(module_name)
-    upper_snake_module_name = snake_module_name.upper()
-    config_file_path = os.path.join(
-        project_dir, 'src', kebab_app_name, 'src', 'config.py'
+async def append_all_disabled_env(
+    task: Task,
+    all_disabled_env_path: str,
+    upper_snake_module_name: str
+):
+    task.print_out(f'Add new environment to: {all_disabled_env_path}')
+    await append_text_file_async(
+        all_disabled_env_path,
+        f'APP_ENABLE_{upper_snake_module_name}_MODULE=false'
     )
+
+
+async def create_app_config(
+    task: Task,
+    config_file_path: str,
+    snake_module_name: str,
+    upper_snake_module_name: str
+):
     config_code = '\n'.join([
         f'app_enable_{snake_module_name}_module = str_to_boolean(os.environ.get(',  # noqa
         f"    'APP_{upper_snake_module_name}', 'true'"
         '))'
     ])
     task.print_out(f'Read config from: {config_file_path}')
-    with open(config_file_path, 'r') as f:
-        code = f.read()
+    code = await read_text_file_async(config_file_path)
     code += '\n' + config_code
     task.print_out(f'Write config to: {config_file_path}')
-    with open(config_file_path, 'w') as f:
-        f.write(code)
+    await write_text_file_async(config_file_path, code)
 
 
-@python_task(
-    name='create-automation-config',
-    inputs=[project_dir_input, app_name_input, module_name_input],
-    upstreams=[copy_resource]
-)
-def create_automation_config(*args: Any, **kwargs: Any):
-    task: Task = kwargs.get('_task')
-    project_dir = kwargs.get('project_dir', '.')
-    app_name = kwargs.get('app_name')
-    module_name = kwargs.get('module_name')
-    snake_app_name = util.to_snake_case(app_name)
-    snake_module_name = util.to_snake_case(module_name)
-    json_modules_file_path = os.path.join(
-        project_dir, '_automate', snake_app_name, 'config', 'modules.json'
-    )
+async def create_automation_json_config(
+    task: Task,
+    json_modules_file_path: str,
+    snake_module_name: str,
+):
     task.print_out(f'Read json config from: {json_modules_file_path}')
-    with open(json_modules_file_path, 'r') as f:
-        json_str = f.read()
+    json_str = await read_text_file_async(json_modules_file_path)
     task.print_out(f'Add "{snake_module_name}" to json config')
     modules: List[str] = jsons.loads(json_str)
     modules.append(snake_module_name)
     json_str = jsons.dumps(modules)
     task.print_out(f'Write new json config to: {json_modules_file_path}')
-    with open(json_modules_file_path, 'w') as f:
-        f.write(json_str)
+    await write_text_file_async(json_modules_file_path, json_str)
+    return modules
 
 
-# TODO: add env to template.env
-# TODO: add env to module_enabled.env (use automation path)
-# TODO: add env to module_disabled.env (use automation path)
-# TODO: create service in docker compose
-# TODO: create runner for local microservices
+async def register_module(
+    task: Task,
+    app_main_file_path: str,
+    snake_module_name: str
+):
+    import_module_path = '.'.join(['module', snake_module_name])
+    function_name = f'register_{snake_module_name}'
+    task.print_out(f'Read code from: {app_main_file_path}')
+    code = await read_text_file_async(app_main_file_path)
+    task.print_out(
+        f'Add import "{function_name}" from "{import_module_path}" to the code'
+    )
+    code = add_import_module(
+        code=code,
+        module_path=import_module_path,
+        resource=function_name
+    )
+    task.print_out(f'Add "{function_name}" call to the code')
+    code = add_function_call(
+        code=code,
+        function_name=function_name,
+        parameters=[]
+    )
+    task.print_out(f'Write modified code to: {app_main_file_path}')
+    await write_text_file_async(app_main_file_path, code)
 
 
-@python_task(
-    name='fastapp-module',
-    group=project_add_group,
-    upstreams=[
-        register_module,
-        create_app_config,
-        create_automation_config,
-    ],
-    runner=runner
-)
-async def add_fastapp_module(*args: Any, **kwargs: Any):
-    task: Task = kwargs.get('_task')
-    task.print_out('Success')
+def get_docker_compose_file(project_dir: str, kebab_app_name: str):
+    return os.path.join(
+        project_dir, 'src', kebab_app_name, 'docker-compose.yml'
+    )
+
+
+def get_app_main_file(project_dir: str, kebab_app_name: str):
+    return os.path.join(
+        project_dir, 'src', kebab_app_name, 'src', 'main.py'
+    )
+
+
+def get_app_config_file(project_dir: str, kebab_app_name: str):
+    return os.path.join(
+        project_dir, 'src', kebab_app_name, 'src', 'config.py'
+    )
+
+
+def get_json_modules_file(project_dir: str, snake_app_name: str):
+    return os.path.join(
+        project_dir, '_automate', snake_app_name, 'config', 'modules.json'
+    )
+
+
+def get_compose_env_file(project_dir: str, snake_app_name: str):
+    return os.path.join(
+        project_dir, '_automate', snake_app_name, 'config',
+        'docker-compose.env'
+    )
+
+
+def get_src_template_env_file(project_dir: str, kebab_app_name: str):
+    return os.path.join(
+        project_dir, 'src', kebab_app_name, 'src', 'template.env'
+    )
+
+
+def get_all_enabled_env_file(project_dir: str, kebab_app_name: str):
+    return os.path.join(
+        project_dir, 'src', kebab_app_name, 'all-module-enabled.env'
+    )
+
+
+def get_all_disabled_env_file(project_dir: str, kebab_app_name: str):
+    return os.path.join(
+        project_dir, 'src', kebab_app_name, 'all-module-disabled.env'
+    )
+

zrb/builtin/generator/project/template/README.md

@@ -33,8 +33,7 @@ pip freeze
 pip freeze | grep dbt
 ```
 
-
-# How to Start PascalProjectName
+# Activating PascalProjectName
 
 ```bash
 source ./project.sh
@@ -58,4 +57,21 @@ To reload your configurations, you can invoke the following command:
 reload
 ```
 
-Please note that PascalProjectName's virtual environment has to be activated first.
+Please note that PascalProjectName's virtual environment has to be activated first.
+
+# Getting task environments
+
+To get task environments, you can invoke the following command:
+
+```bash
+zrb project get-default-env
+```
+
+It is expected to use the command for informational purposes only.
+You should only override necessary environment variables to keep track of what you need to change.
+
+Being said so, you can also add the default task environments to `.env` by invoking the following command:
+
+```bash
+zrb project get-default-env >> .env
+```

zrb/builtin/generator/project_task/add.py

@@ -1,29 +1,28 @@
 from ....helper.file.copy_tree import copy_tree
 from ....helper.codemod.add_assert_resource import add_assert_resource
 from ....helper.codemod.add_import_module import add_import_module
+from ....helper.file.text import read_text_file_async, write_text_file_async
 
 import os
 
 current_dir = os.path.dirname(__file__)
 
 
-def add_project_automation(project_dir: str):
+async def add_project_automation(project_dir: str):
     if os.path.exists(os.path.join(project_dir, '_automate', '_project')):
         return
-    copy_tree(
+    await copy_tree(
         src=os.path.join(current_dir, 'template'),
         dst=project_dir
     )
     project_task_module_path = '_automate._project'
     zrb_init_path = os.path.join(project_dir, 'zrb_init.py')
-    with open(zrb_init_path, 'r') as f:
-        code = f.read()
-        import_alias = project_task_module_path.split('.')[-1]
-        code = add_import_module(
-            code=code,
-            module_path=project_task_module_path,
-            alias=import_alias
-        )
-        code = add_assert_resource(code, import_alias)
-    with open(zrb_init_path, 'w') as f:
-        f.write(code)
+    code = await read_text_file_async(zrb_init_path)
+    import_alias = project_task_module_path.split('.')[-1]
+    code = add_import_module(
+        code=code,
+        module_path=project_task_module_path,
+        alias=import_alias
+    )
+    code = add_assert_resource(code, import_alias)
+    await write_text_file_async(zrb_init_path, code)

zrb/builtin/generator/project_task/task_factory.py

@@ -5,6 +5,7 @@ from ....task.task import Task
 from ....helper import util
 from ....helper.codemod.add_import_module import add_import_module
 from ....helper.codemod.add_upstream_to_task import add_upstream_to_task
+from ....helper.file.text import read_text_file_async, write_text_file_async
 from .add import add_project_automation
 from .task_factory_helper import (
     _get_app_local_task_file, _get_app_container_task_file,
@@ -29,7 +30,7 @@ def create_add_project_automation(
         task: Task = kwargs.get('_task')
         project_dir = kwargs.get('project_dir', '.')
         task.print_out('Create project automation modules if not exist')
-        add_project_automation(project_dir)
+        await add_project_automation(project_dir)
     return _task
 
 
@@ -184,17 +185,14 @@ def _create_register_app_task(
         # turn into module path
         upstream_module_path = '.'.join(upstream_module_parts)
         task.print_out(f'Add {upstream_task_var} to project automation')
-        with open(project_automation_path, 'r') as f:
-            code = f.read()
-            code = add_import_module(
-                code=code,
-                module_path=upstream_module_path,
-                resource=upstream_task_var
-            )
-            code = add_upstream_to_task(
-                code, project_automation_task_name, upstream_task_var
-            )
-        with open(project_automation_path, 'w') as f:
-            f.write(code)
-
+        code = await read_text_file_async(project_automation_path)
+        code = add_import_module(
+            code=code,
+            module_path=upstream_module_path,
+            resource=upstream_task_var
+        )
+        code = add_upstream_to_task(
+            code, project_automation_task_name, upstream_task_var
+        )
+        await write_text_file_async(project_automation_path, code)
     return _task

zrb/builtin/generator/simple_python_app/add.py

@@ -66,15 +66,15 @@ register_local_app_module = create_register_local_app_module(
 )
 
 register_container_app_module = create_register_container_app_module(
-    upstreams=[copy_resource]
+    upstreams=[register_local_app_module]
 )
 
 register_image_app_module = create_register_image_app_module(
-    upstreams=[copy_resource]
+    upstreams=[register_container_app_module]
 )
 
 register_deployment_app_module = create_register_deployment_app_module(
-    upstreams=[copy_resource]
+    upstreams=[register_image_app_module]
 )
 
 add_project_task = create_add_project_automation(

zrb/builtin/generator/simple_python_app/template/src/kebab-app-name/docker-compose.yml

@@ -9,4 +9,10 @@ services:
     environment:
       APP_PORT: ${APP_PORT:-8080}
     ports:
-      - "${HOST_PORT:-8080}:${APP_PORT:-8080}"
+      - "${HOST_PORT:-8080}:${APP_PORT:-8080}"
+    restart: on-failure
+    healthcheck:
+      test: ["CMD-SHELL", "curl --fail http://localhost:${APP_PORT:-8080}/ || exit 1"]
+      interval: 5s
+      timeout: 1s
+      retries: 30

zrb/builtin/generator/simple_python_app/template/src/kebab-app-name/src/Dockerfile

@@ -1,5 +1,7 @@
 FROM python:3.10-slim
+ENV APP_MESSAGE "Hello, world!"
+ENV APP_PORT 8080
 WORKDIR /home
 COPY . .
 RUN pip install -r requirements.txt
-CMD python main.py
+CMD uvicorn main:app --host "0.0.0.0" --port "$APP_PORT"

zrb/builtin/generator/simple_python_app/template/src/kebab-app-name/src/main.py

@@ -1,4 +1,3 @@
-import uvicorn
 import os
 
 MESSAGE = os.getenv('APP_MESSAGE', 'Hello, world!')
@@ -19,7 +18,3 @@ async def app(scope, receive, send):
         'type': 'http.response.body',
         'body': bytes(MESSAGE, 'utf-8'),
     })
-
-
-if __name__ == "__main__":
-    uvicorn.run("main:app", host='0.0.0.0', port=PORT, log_level="info")

zrb/builtin/md5.py

@@ -4,6 +4,7 @@ from ..task.decorator import python_task
 from ..task_input.str_input import StrInput
 from ..runner import runner
 
+import aiofiles
 import hashlib
 
 # Common definitions
@@ -47,7 +48,7 @@ async def hash_text(*args: str, **kwargs: Any):
 )
 async def sum_file(*args: str, **kwargs: Any):
     file_path: str = kwargs.get('file', '')
-    with open(file_path, "rb") as f:
-        contents = f.read()
-        hashed_text = hashlib.md5(contents).hexdigest()
-        return hashed_text
+    async with aiofiles.open(file_path, mode='rb') as file:
+        contents = await file.read()
+    hashed_text = hashlib.md5(contents).hexdigest()
+    return hashed_text

zrb/builtin/project.py

@@ -0,0 +1,32 @@
+from typing import Any, Mapping
+from ._group import project_group
+from ..task.decorator import python_task
+from ..task_input.bool_input import BoolInput
+from ..runner import runner
+from ..helper.env_map.fetch import fetch_env_map_from_group
+
+
+@python_task(
+    name='get-default-env',
+    description='Get default values for project environments',
+    inputs=[
+        BoolInput(
+            name='export',
+            shortcut='e',
+            description='Whether add export statement or not',
+            default=True
+        )
+    ],
+    group=project_group,
+    runner=runner
+)
+async def get_default_env(*args: Any, **kwargs: Any) -> str:
+    env_map: Mapping[str, str] = {}
+    env_map = fetch_env_map_from_group(env_map, project_group)
+    env_keys = list(env_map.keys())
+    env_keys.sort()
+    should_export = kwargs.get('export', True)
+    export_prefix = 'export ' if should_export else ''
+    return '\n'.join([
+        f'{export_prefix}{key}={env_map[key]}' for key in env_keys
+    ])

zrb/helper/docker_compose/file.py

@@ -0,0 +1,22 @@
+from typing import Any, Mapping
+from ruamel.yaml import YAML, composer, CommentedMap
+
+
+def read_compose_file(file_name: str) -> Any:
+    yaml = YAML()
+    with open(file_name, 'r') as file:
+        data = yaml.load(file)
+    return data
+
+
+def add_services(file_name: str, new_services: Mapping[str, str]):
+    yaml = YAML()
+    with open(file_name, 'r') as f:
+        data = yaml.load(f)
+        # data = round_trip_load(f, preserve_quotes=True)
+    data = CommentedMap(data)
+    data['services'].update(CommentedMap(new_services))
+    # Write the modified data structure back to the file
+    with open(file_name, 'w') as f:
+        yaml.dump(data, f)
+        # round_trip_dump(data, f)