laniakea-api-agent 0.0.1__tar.gz

laniakea_api_agent-0.0.1/.gitignore

@@ -0,0 +1,9 @@
+.env
+*.key
+keys/
+*.crt
+__pycache__/
+*.pem
+*.log
+.terraform*
+terraform.tfstate*

laniakea_api_agent-0.0.1/PKG-INFO

@@ -0,0 +1,89 @@
+Metadata-Version: 2.4
+Name: laniakea-api-agent
+Version: 0.0.1
+Summary: Laniakea deployment agent orchestrates cloud deployments via Terraform and Ansible
+Project-URL: Homepage, https://github.com/riccardocaccia/laniakea-agent/tree/laniakea-agent-pkg
+Project-URL: Issues, https://github.com/riccardocaccia/laniakea-agent/issues
+Author: rcaccia
+License: MIT
+Keywords: ansible,aws,cloud,deployment,openstack,redis,terraform
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: System Administrators
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: System :: Systems Administration
+Requires-Python: >=3.10
+Requires-Dist: docker
+Requires-Dist: httpx
+Requires-Dist: hvac>=2.0.0
+Requires-Dist: keystoneauth1[openidconnect]
+Requires-Dist: openstacksdk
+Requires-Dist: pydantic>=2.0.0
+Requires-Dist: pyjwt
+Requires-Dist: python-dotenv
+Requires-Dist: python-keystoneclient
+Requires-Dist: pyyaml
+Requires-Dist: redis
+Requires-Dist: requests
+Requires-Dist: rq
+Description-Content-Type: text/markdown
+
+# laniakea-api-agent
+
+Cloud deployment worker agent for the Laniakea orchestration platform.
+Consumes jobs from Redis queues and orchestrates VM provisioning via Terraform and Ansible.
+
+## Installation
+
+```bash
+pip install laniakea-api-agent
+```
+
+## Configuration
+
+Create a `.env` file in your working directory:
+
+```bash
+# Redis
+REDIS_HOST=your-redis-host
+REDIS_PORT=6379
+REDIS_PASSWORD=your-redis-password
+
+# Laniakea API
+LANIAKEA_API_URL=https://your-api-host:8443/laniakea_core/v1.0
+AGENT_MASTER_PASSWORD=your-shared-secret
+AGENT_ID=laniakea-agent-1
+
+# Vault
+VAULT_ADDR=https://your-vault:8200
+VAULT_TOKEN=your-vault-token
+
+# Logs
+DEPLOYMENT_LOG_DIR=/var/log/laniakea-agent
+```
+
+## Usage
+
+```bash
+# listen on openstack queue (default)
+laniakea-agent
+
+# listen on a specific queue
+laniakea-agent --queue openstack
+
+# listen on multiple queues
+laniakea-agent --queue openstack --queue aws
+
+# use a custom .env file
+laniakea-agent --env /etc/laniakea/agent.env
+```
+
+## Requirements
+
+- Python 3.10+
+- Docker (for Terraform containers)
+- Ansible installed on the host
+

laniakea_api_agent-0.0.1/README.md

@@ -0,0 +1,56 @@
+# laniakea-api-agent
+
+Cloud deployment worker agent for the Laniakea orchestration platform.
+Consumes jobs from Redis queues and orchestrates VM provisioning via Terraform and Ansible.
+
+## Installation
+
+```bash
+pip install laniakea-api-agent
+```
+
+## Configuration
+
+Create a `.env` file in your working directory:
+
+```bash
+# Redis
+REDIS_HOST=your-redis-host
+REDIS_PORT=6379
+REDIS_PASSWORD=your-redis-password
+
+# Laniakea API
+LANIAKEA_API_URL=https://your-api-host:8443/laniakea_core/v1.0
+AGENT_MASTER_PASSWORD=your-shared-secret
+AGENT_ID=laniakea-agent-1
+
+# Vault
+VAULT_ADDR=https://your-vault:8200
+VAULT_TOKEN=your-vault-token
+
+# Logs
+DEPLOYMENT_LOG_DIR=/var/log/laniakea-agent
+```
+
+## Usage
+
+```bash
+# listen on openstack queue (default)
+laniakea-agent
+
+# listen on a specific queue
+laniakea-agent --queue openstack
+
+# listen on multiple queues
+laniakea-agent --queue openstack --queue aws
+
+# use a custom .env file
+laniakea-agent --env /etc/laniakea/agent.env
+```
+
+## Requirements
+
+- Python 3.10+
+- Docker (for Terraform containers)
+- Ansible installed on the host
+

laniakea_api_agent-0.0.1/laniakea-agent/__init__.py

@@ -0,0 +1,4 @@
+"""
+Consumes jobs from Redis queues and orchestrates Terraform + Ansible deployments.
+"""
+__version__ = "0.1.0"

laniakea_api_agent-0.0.1/laniakea-agent/ansible_agent.py

@@ -0,0 +1,45 @@
+import os
+import stat
+import logging
+from laniakea_agent.ansible_worker import AnsibleWorker
+
+# create a logging istance (for the ansible_agent -> __name__) used for the debugging
+logger = logging.getLogger(__name__)
+
+def run_ansible_step(job, playbook_url, requirements_url):
+    """
+    Passes the needed info to the ansible worker funcions in order to
+    configure the machine with the correct software.
+    """
+    uuid = job.deployment_uuid
+    # NOTE: PRIVATE KEY MANAGEMENT to be imporved
+    key_path = f"keys/{uuid}.pem"
+
+    worker = AnsibleWorker(playbook_url, requirements_url, uuid)
+
+    try:
+        success_prep, msg = worker.prepare_environment()
+
+        if not success_prep:
+            logger.error(f"[{uuid}] ERROR in Ansible preparation: {msg}")
+            return False
+
+        # bastion identification
+        bastion = "0.0.0.0"
+        if job.selected_provider.lower() == 'openstack':
+            os_data = job.cloud_providers.openstack
+            if os_data.inputs.network_type == 'private':
+                bastion = os_data.private_network_proxy_host or "0.0.0.0"
+
+        success= worker.execute_deployment(
+            target_ip=job.vm_ip,
+            ssh_key_path=key_path,
+            bastion_ip=bastion
+        )
+        worker.cleanup() # clean /tmp/uuid
+        return success
+
+    except Exception as e:
+        logger.error(f"[{uuid}] EXCEPTION in an Ansible step: {e}")
+        worker.cleanup()
+        return False

laniakea_api_agent-0.0.1/laniakea-agent/ansible_worker.py

@@ -0,0 +1,116 @@
+import os
+import subprocess
+import shutil
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+# NOTE: move from here, not interesting... also update from rcaccia to laniakea
+GROUP_VARS_URL = "https://raw.githubusercontent.com/riccardocaccia/laniakea-nebula/clean-main/terraform/ansible/group_vars/galaxy.yml"
+NGINX_TEMPLATE_URL = "https://raw.githubusercontent.com/riccardocaccia/laniakea-nebula/clean-main/terraform/ansible/templates/nginx/galaxy.j2"
+ANSIBLE_VENV = "/tmp/ansible-venv"
+
+class AnsibleWorker:
+    def __init__(self, playbook_url, requirements_url, uuid):
+        self.playbook_url = playbook_url
+        self.requirements_url = requirements_url
+        self.uuid = uuid
+        self.base_dir = f"/tmp/{uuid}"
+        self.playbook_path = os.path.join(self.base_dir, "deploy.yml")
+        self.requirements_path = os.path.join(self.base_dir, "requirements.yml")
+        self.group_vars_path = os.path.join(self.base_dir, "group_vars", "all.yml")
+        self.nginx_template_path = os.path.join(self.base_dir, "templates", "nginx", "galaxy.j2")
+
+    def prepare_environment(self):
+        try:
+            os.makedirs(self.base_dir, exist_ok=True)
+            os.makedirs(os.path.join(self.base_dir, "group_vars"), exist_ok=True)
+            os.makedirs(os.path.join(self.base_dir, "templates", "nginx"), exist_ok=True)
+
+            with open(self.playbook_path, "wb") as f:
+                f.write(requests.get(self.playbook_url).content)
+            with open(self.requirements_path, "wb") as f:
+                f.write(requests.get(self.requirements_url).content)
+            with open(self.group_vars_path, "wb") as f:
+                f.write(requests.get(GROUP_VARS_URL).content)
+            with open(self.nginx_template_path, "wb") as f:
+                f.write(requests.get(NGINX_TEMPLATE_URL).content)
+
+            return True, "OK"
+        except Exception as e:
+            return False, str(e)
+
+    def execute_deployment(self, target_ip, ssh_key_path, bastion_ip=None):
+        """
+        Run CLI command to copy & download playbooks.
+        Make a proxy jump if a bastion is specified.
+        """
+        ssh_base = (
+            f"ssh -i {ssh_key_path} "
+            f"-o StrictHostKeyChecking=no "
+            f"-o UserKnownHostsFile=/dev/null "
+        )
+        scp_base = (
+            f"scp -i {ssh_key_path} "
+            f"-o StrictHostKeyChecking=no "
+            f"-o UserKnownHostsFile=/dev/null "
+        )
+
+        if bastion_ip and bastion_ip != "0.0.0.0":
+            proxy = (
+                f"-o ProxyCommand='ssh -i {ssh_key_path} "
+                f"-o StrictHostKeyChecking=no -W %h:%p rocky@{bastion_ip}'"
+            )
+            ssh_base += proxy + " "
+            scp_base += proxy + " "
+
+        remote = f"rocky@{target_ip}"
+
+        steps = [
+            # 1. Create directory strucfure VM
+            f"{ssh_base} {remote} 'sudo mkdir -p /tmp/galaxy-deploy/group_vars /tmp/galaxy-deploy/templates/nginx && sudo chown -R rocky /tmp/galaxy-deploy'",
+
+            # 2. Copy all files
+            f"{scp_base} {self.playbook_path} {remote}:/tmp/galaxy-deploy/deploy.yml",
+            f"{scp_base} {self.requirements_path} {remote}:/tmp/galaxy-deploy/requirements.yml",
+            f"{scp_base} {self.group_vars_path} {remote}:/tmp/galaxy-deploy/group_vars/all.yml",
+            f"{scp_base} {self.nginx_template_path} {remote}:/tmp/galaxy-deploy/templates/nginx/galaxy.j2",
+
+            # 3. Create a virtualenv and install ansible
+            f"{ssh_base} {remote} 'sudo dnf install -y python3-pip git && sudo python3 -m venv {ANSIBLE_VENV} && sudo {ANSIBLE_VENV}/bin/pip install ansible \"virtualenv<20.22\"'",
+
+            # 4. Install ansible roles
+            f"{ssh_base} {remote} 'sudo {ANSIBLE_VENV}/bin/ansible-galaxy install -r /tmp/galaxy-deploy/requirements.yml -p /tmp/galaxy-deploy/roles'",
+
+            # 5. execute playbooks in local as root 
+            (
+                f"{ssh_base} {remote} "
+                f"'cd /tmp/galaxy-deploy && "
+                f"sudo ANSIBLE_ROLES_PATH=/tmp/galaxy-deploy/roles "
+                f"{ANSIBLE_VENV}/bin/ansible-playbook "
+                f"-i localhost, -c local "
+                f"-e \"target_hosts=localhost\" "   
+                f"/tmp/galaxy-deploy/deploy.yml "
+                f"2>&1 | sudo tee /tmp/galaxy-deploy/ansible.log; "
+                f"exit ${{PIPESTATUS[0]}}'"
+            ),
+        ]
+
+        # debug helper
+        for i, step in enumerate(steps):
+            logger.info(f"[{self.uuid}] Step {i+1}/{len(steps)}: {step[:80]}...")
+            res = subprocess.run(step, shell=True)
+            if res.returncode != 0:
+                logger.error(f"[{self.uuid}] Step {i+1} fallito.")
+                return False
+
+        return True
+
+    def cleanup(self):
+        """
+        Clean all the directories after the process
+        """
+        if os.path.exists(self.base_dir):
+            shutil.rmtree(self.base_dir)
+            logger.info(f"[{self.uuid}] Pulizia cartella temporanea completata.")

laniakea_api_agent-0.0.1/laniakea-agent/api_client.py

@@ -0,0 +1,156 @@
+"""
+Calls the Laniakea Queue API to update deployment status.
+Auth: JWT signed with AGENT_MASTER_PASSWORD (HMAC-SHA256).
+      No certificates needed just the shared master password.
+
+pool password model:
+  - one master password governs all agents
+  - to revoke ALL agents: change the password on API + all agents and restart
+"""
+
+import jwt
+import logging
+import os
+import time
+import uuid
+from typing import Optional
+import httpx
+
+logger = logging.getLogger(__name__)
+
+# Base URL of the Laniakea queue API (HTTPS default port)
+API_BASE_URL = os.getenv("LANIAKEA_API_URL", "https://.......:8443")
+AGENT_MASTER_PASSWORD = os.getenv("AGENT_MASTER_PASSWORD", "")
+AGENT_ID = os.getenv("AGENT_ID", "laniakea-agent")
+
+# NOTE: CA cert to verify the API server's TLS certificate
+AGENT_CA_CERT = os.getenv("AGENT_CA_CERT", "certs/ca.crt")
+
+# token lifespan
+TOKEN_TTL_SECONDS = 300 # short lived token
+
+# Token generation
+def _mint_token() -> str:
+    """
+    Generate a short-lived JWT signed with the master password.
+
+    Payload:
+      sub: agent identity (AGENT_ID from .env)
+      iat: issued at
+      exp: expires in TOKEN_TTL_SECONDS
+      jti:  unique token ID
+    """
+    if not AGENT_MASTER_PASSWORD:
+        raise RuntimeError("AGENT_MASTER_PASSWORD is not set.")
+
+    now = int(time.time())
+    payload = {
+        "sub": AGENT_ID,
+        "iat": now,
+        "exp": now + TOKEN_TTL_SECONDS,
+        "jti": str(uuid.uuid4()),
+    }
+    return jwt.encode(payload, AGENT_MASTER_PASSWORD, algorithm="HS256")
+
+
+# Internal helper
+def _make_client() -> httpx.Client:
+    """
+    Build an httpx Client with:
+      - Authorization: Bearer <JWT>  for agent authentication
+      - TLS server verification via CA cert
+    """
+    token = _mint_token()
+
+    # Use the CA cert if it exists, otherwise fall back to system bundle.
+    # The CA cert here verifies the API SERVER certificate — not client auth.
+    verify: str | bool = AGENT_CA_CERT if os.path.exists(AGENT_CA_CERT) else True
+
+    return httpx.Client(
+        base_url=API_BASE_URL,
+        headers={"Authorization": f"Bearer {token}"},
+        verify=verify,
+        timeout=30,
+    )
+
+
+# Public interface called by terraform_agent.py
+
+def update_deployment_status(
+    deployment_uuid: str, new_status: str, status_reason: Optional[str] = None, outputs: Optional[str] = None,)-> bool:
+    # NOTE: add email, not every time only the first
+    """
+    PATCH /internal/deployments/{uuid}/status on the queue API.
+
+    Parameters
+    ----------
+    deployment_uuid : str
+        UUID of the deployment to update.
+    new_status : str
+        One of: CREATE_IN_PROGRESS, CREATE_COMPLETE, CREATE_FAILED,
+                UPDATE_IN_PROGRESS, UPDATE_FAILED.
+    status_reason : str, optional
+        Human-readable reason (used on FAILED states).
+    outputs : str, optional
+        JSON string with deployment outputs (e.g. vm_ip).
+
+    Returns
+    -------
+    bool
+        True on success, False if the API rejected or was unreachable.
+        The caller decides whether to raise or continue.
+    """
+    payload = {"status": new_status.upper()}
+    if status_reason:
+        payload["status_reason"] = status_reason
+    if outputs:
+        payload["outputs"] = outputs
+
+    try:
+        with _make_client() as client:
+            response = client.patch(
+                f"/internal/deployments/{deployment_uuid}/status",
+                json=payload,
+            )
+
+        if response.status_code == 200:
+            logger.info("[%s] Status updated to %s via API.", deployment_uuid, new_status)
+            return True
+
+        # 409 = invalid transition (e.g. agent tried to go COMPLETE -> IN_PROGRESS)
+        # 404 = deployment not found in DB (API never saw the QUEUED write)
+        logger.error(
+            "[%s] API rejected status update to %s: HTTP %s — %s",
+            deployment_uuid,
+            new_status,
+            response.status_code,
+            response.text,
+        )
+        return False
+
+    except httpx.RequestError as exc:
+        logger.error(
+            "[%s] Could not reach API to update status: %s", deployment_uuid, exc
+        )
+        return False
+
+def push_log_line(deployment_uuid: str, level: str, message: str) -> None:
+    """
+    POST /internal/deployments/{uuid}/logs  on the Queue API.
+
+    Sends a single formatted log line so the API can accumulate it in
+    logs/orchestrator-{uuid}.log on the API VM. The dashboard reads that
+    file via GET /api/deployments/{uuid}/logs.
+
+    Failures are silently swallowed.
+    """
+    payload = {
+        "level":   level.upper(),
+        "message": message,
+        }
+    try:
+        with _make_client() as client:
+            client.post(f"/internal/deployments/{deployment_uuid}/logs", json=payload,)
+    except Exception:
+        pass  # never crash the agent because of a log push failure
+

laniakea_api_agent-0.0.1/laniakea-agent/auth_utils/aws_auth.py

@@ -0,0 +1,27 @@
+from vault_utils import get_secrets
+import logging
+
+logger = logging.getLogger(__name__)
+
+def get_aws_credentials(group: str):
+    """
+    AWS keys retrieving from Vault.
+    """
+    try:
+        # TODO: modify once the final vault is up
+        vault_path = f"SECRET/infrastructure/aws/{group}"
+        secrets = get_secrets(vault_path)
+        
+        if not secrets:
+            # NOTE : TEST!!! remove default implementation
+            logger.warning(f"No secret found for the group: {group}, TRYING DEFAULT...")
+            secrets = get_secrets("infrastructure/aws/default")
+
+        return {
+            "access": secrets.get("access_key"),
+            "secret": secrets.get("secret_key")
+        }
+
+    except Exception as e:
+        logger.error(f"Error in AWS credential retrieving from Vault: {e}")
+        return None

laniakea_api_agent-0.0.1/laniakea-agent/auth_utils/openstack_auth.py

@@ -0,0 +1,33 @@
+from keystoneauth1 import loading
+from keystoneauth1 import session
+import logging
+
+logger = logging.getLogger(__name__)
+
+def get_openstack_admin_creds():
+    ''' access to retrieve OpenStack credentials from Vault '''
+    # CHANGE THE VAULT PATH ONCE VAULT BUILD FINAL COMPLETED
+    return get_secrets("SECRET/infrastructure/openstack/admin")
+
+def get_keystone_token(aai_token, auth_url, project_id):
+    try:
+        loader = loading.get_plugin_loader('v3oidcaccesstoken')
+        
+        auth = loader.load_from_options(
+            auth_url=auth_url,
+            ### TODO: env variable? or input? ###
+            identity_provider='recas-bari', 
+            #####################################
+            protocol='openid',
+            access_token=aai_token,
+            project_id=project_id
+        )
+        
+        sess = session.Session(auth=auth, verify=True) 
+        token_os = sess.get_token()
+        logger.info("Token AAI exchange: success!")
+        return token_os
+
+    except Exception as e:
+        logger.error(f"ERROR [OpenStack Auth]: {e}")
+        return None

laniakea_api_agent-0.0.1/laniakea-agent/cli.py

@@ -0,0 +1,97 @@
+"""
+Entry point for laniakea-agent.
+
+After running `pip install laniakea-agent`, the user can launch it with:
+    laniakea-agent --queue openstack
+    laniakea-agent --queue aws
+    laniakea-agent --queue openstack --queue aws   # listens to both
+
+The process loads the .env file from the current directory, connects to Redis, 
+and spawns the RQ Worker listening on the specified queues.
+
+RQ imports 'worker_wrapper.run_from_dict' by its string name, this works 
+because laniakea_agent is an installed package and worker_wrapper is accessible 
+as laniakea_agent.worker_wrapper. However, a local worker_wrapper in the working 
+directory (if present) takes precedence, allowing for overrides without 
+reinstalling the package.
+"""
+
+import argparse
+import os
+import sys
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="laniakea-agent",
+        description="Laniakea deployment agent which consumes jobs from Redis queues.",
+    )
+    parser.add_argument(
+        "--queue", "-q",
+        action="append",
+        dest="queues",
+        default=None,
+        metavar="NAME",
+        help="Queue name to listen on (e.g. openstack, aws). Repeat for multiple queues.",
+    )
+    parser.add_argument(
+        "--env", "-e",
+        default=".env",
+        metavar="FILE",
+        help="Path to .env file (default: .env in current directory).",
+    )
+    parser.add_argument(
+        "--version", "-v",
+        action="store_true",
+        help="Print version and exit.",
+    )
+    args = parser.parse_args()
+
+    if args.version:
+        from laniakea_agent import __version__
+        print(f"laniakea-agent {__version__}")
+        sys.exit(0)
+
+    # load .env
+    env_path = os.path.abspath(args.env)
+    if os.path.exists(env_path):
+        from dotenv import load_dotenv
+        load_dotenv(env_path)
+        print(f"[config] loaded {env_path}")
+    else:
+        print(f"[config] .env not found at {env_path} — using environment variables")
+
+    # validate required env vars
+    missing = [v for v in ["REDIS_HOST", "REDIS_PASSWORD", "AGENT_MASTER_PASSWORD", "LANIAKEA_API_URL"]
+               if not os.getenv(v)]
+    if missing:
+        print(f"[error] missing required environment variables: {', '.join(missing)}")
+        print("         set them in .env or export them before running laniakea-agent")
+        sys.exit(1)
+
+    # default queue
+    queue_names = args.queues or ["openstack"]
+
+    # connect Redis
+    from redis import Redis
+    from rq import Worker, Queue
+
+    redis_conn = Redis(
+        host=os.getenv("REDIS_HOST"),
+        port=int(os.getenv("REDIS_PORT", "6379")),
+        password=os.getenv("REDIS_PASSWORD"),
+        decode_responses=False,
+    )
+
+    queues = [Queue(name, connection=redis_conn) for name in queue_names]
+
+    print(f"[agent] laniakea-agent listening on queues: {queue_names}")
+    print(f"[agent] redis: {os.getenv('REDIS_HOST')}:{os.getenv('REDIS_PORT', '6379')}")
+    print(f"[agent] api:   {os.getenv('LANIAKEA_API_URL')}")
+    print(f"[agent] id:    {os.getenv('AGENT_ID', 'laniakea-agent')}")
+
+    Worker(queues, connection=redis_conn).work()
+
+if __name__ == "__main__":
+    main()
+