cloudfall 0.2.0__py3-none-any.whl

cloudfall_engine/_bundled/ansible/roles/cloudfall_inspect/tasks/main.yml

@@ -0,0 +1,643 @@
+---
+- name: Validate inspection target and inputs
+  ansible.builtin.assert:
+    that:
+      - ansible_facts["distribution"] == "Debian"
+      - ansible_facts["service_mgr"] == "systemd"
+      - cloudfall_inspect_server_id | length > 0
+      - cloudfall_inspect_server_type.id | length > 0
+      - cloudfall_inspect_output_directory.startswith('/')
+    fail_msg: Cloudfall inspection requires a Debian systemd host and a server type
+
+- name: Collect installed Debian packages
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/dpkg-query
+      - --show
+      - >-
+        --showformat=${db:Status-Abbrev}{"name":"${Package}","version":"${Version}","source":"dpkg-query","arch":"${Architecture}"}\n
+  register: cloudfall_inspect_dpkg
+  changed_when: false
+
+- name: Normalize installed package evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_packages: >-
+      {{
+        cloudfall_inspect_dpkg.stdout_lines
+        | select('match', '^ii ')
+        | map('regex_replace', '^ii ', '')
+        | map('from_json')
+        | list
+      }}
+
+- name: Normalize Debian version evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_debian_version: >-
+      {{
+        cloudfall_inspect_packages
+        | selectattr('name', 'equalto', 'base-files')
+        | map(attribute='version')
+        | first
+      }}
+
+- name: Collect service facts
+  ansible.builtin.service_facts:
+
+- name: Collect block device topology
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/lsblk
+      - --json
+      - --bytes
+      - --output
+      - NAME,PATH,TYPE,SIZE,FSTYPE,MOUNTPOINT,PKNAME,MODEL,SERIAL,ROTA
+  register: cloudfall_inspect_lsblk
+  changed_when: false
+
+- name: Normalize block device evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_block_devices: >-
+      {{ (cloudfall_inspect_lsblk.stdout | from_json).blockdevices }}
+
+- name: Select NVMe namespaces for SMART inspection
+  ansible.builtin.set_fact:
+    cloudfall_inspect_nvme_devices: >-
+      {{
+        cloudfall_inspect_block_devices
+        | selectattr('type', 'equalto', 'disk')
+        | selectattr('path', 'match', '^/dev/nvme[0-9]+n[0-9]+$')
+        | list
+      }}
+
+- name: Check whether smartctl is installed
+  ansible.builtin.stat:
+    path: /usr/sbin/smartctl
+    get_checksum: false
+  register: cloudfall_inspect_smartctl_binary
+
+- name: Read smartctl version
+  ansible.builtin.command:
+    argv:
+      - /usr/sbin/smartctl
+      - --version
+  register: cloudfall_inspect_smartctl_version
+  changed_when: false
+  when: cloudfall_inspect_smartctl_binary.stat.exists
+
+- name: Initialize SMART device evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_smart_devices: []
+
+- name: Collect read-only NVMe SMART evidence
+  ansible.builtin.command:
+    argv:
+      - /usr/sbin/smartctl
+      - --xall
+      - "{{ item.path }}"
+  loop: "{{ cloudfall_inspect_nvme_devices }}"
+  loop_control:
+    label: "{{ item.path }}"
+  register: cloudfall_inspect_smartctl_devices
+  changed_when: false
+  failed_when: >-
+    cloudfall_inspect_smartctl_devices.rc < 0
+    or cloudfall_inspect_smartctl_devices.rc > 255
+  when: cloudfall_inspect_smartctl_binary.stat.exists
+
+- name: Validate NVMe SMART report fields
+  ansible.builtin.assert:
+    that:
+      - item.stdout is search('(?m)^Model Number:\s+.+$')
+      - item.stdout is search('(?m)^Serial Number:\s+.+$')
+      - item.stdout is search('(?m)^Firmware Version:\s+.+$')
+      - >-
+        item.stdout is search(
+          '(?m)^SMART overall-health self-assessment test result:\s+.+$'
+        )
+      - item.stdout is search('(?m)^Critical Warning:\s+0x[0-9A-Fa-f]+$')
+      - item.stdout is search('(?m)^Temperature:\s+-?[0-9]+ Celsius$')
+      - item.stdout is search('(?m)^Available Spare:\s+[0-9]+%$')
+      - >-
+        item.stdout is search(
+          '(?m)^Available Spare Threshold:\s+[0-9]+%$'
+        )
+      - item.stdout is search('(?m)^Percentage Used:\s+[0-9,]+%$')
+      - item.stdout is search('(?m)^Data Units Read:\s+[0-9,]+ .+$')
+      - item.stdout is search('(?m)^Data Units Written:\s+[0-9,]+ .+$')
+      - item.stdout is search('(?m)^Power Cycles:\s+[0-9,]+$')
+      - item.stdout is search('(?m)^Power On Hours:\s+[0-9,]+$')
+      - item.stdout is search('(?m)^Unsafe Shutdowns:\s+[0-9,]+$')
+      - >-
+        item.stdout is search(
+          '(?m)^Media and Data Integrity Errors:\s+[0-9,]+$'
+        )
+      - >-
+        item.stdout is search(
+          '(?m)^Error Information Log Entries:\s+[0-9,]+$'
+        )
+    fail_msg: >-
+      smartctl returned an unsupported NVMe report for {{ item.item.path }}
+  loop: "{{ cloudfall_inspect_smartctl_devices.results | default([]) }}"
+  loop_control:
+    label: "{{ item.item.path }}"
+  when:
+    - cloudfall_inspect_smartctl_binary.stat.exists
+    - not item.skipped | default(false)
+
+- name: Normalize NVMe SMART evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_smart_devices: >-
+      {{
+        cloudfall_inspect_smart_devices
+        + [
+          {
+            'path': item.item.path,
+            'model': (
+              item.stdout_lines
+              | select('match', '^Model Number:')
+              | first
+              | regex_replace('^Model Number:\s+', '')
+              | trim
+            ),
+            'serial': (
+              item.stdout_lines
+              | select('match', '^Serial Number:')
+              | first
+              | regex_replace('^Serial Number:\s+', '')
+              | trim
+            ),
+            'firmware': (
+              item.stdout_lines
+              | select('match', '^Firmware Version:')
+              | first
+              | regex_replace('^Firmware Version:\s+', '')
+              | trim
+            ),
+            'smartctlExitCode': item.rc | int,
+            'overallHealth': (
+              item.stdout_lines
+              | select('match', '^SMART overall-health')
+              | first
+              | regex_replace('^.*:\s+', '')
+              | regex_replace('!$', '')
+              | lower
+            ),
+            'criticalWarning': (
+              item.stdout_lines
+              | select('match', '^Critical Warning:')
+              | first
+              | regex_replace('^Critical Warning:\s+', '')
+            ),
+            'reliabilityDegraded': (
+              'reliability has been degraded' in item.stdout
+            ),
+            'temperatureCelsius': (
+              item.stdout_lines
+              | select('match', '^Temperature:')
+              | first
+              | regex_replace('^Temperature:\s+', '')
+              | regex_replace(' Celsius$', '')
+              | int
+            ),
+            'availableSparePercent': (
+              item.stdout_lines
+              | select('match', '^Available Spare:')
+              | first
+              | regex_replace('^Available Spare:\s+', '')
+              | regex_replace('%$', '')
+              | int
+            ),
+            'availableSpareThresholdPercent': (
+              item.stdout_lines
+              | select('match', '^Available Spare Threshold:')
+              | first
+              | regex_replace('^Available Spare Threshold:\s+', '')
+              | regex_replace('%$', '')
+              | int
+            ),
+            'percentageUsed': (
+              item.stdout_lines
+              | select('match', '^Percentage Used:')
+              | first
+              | regex_replace('^Percentage Used:\s+', '')
+              | regex_replace('%$', '')
+              | replace(',', '')
+              | int
+            ),
+            'dataUnitsRead': (
+              item.stdout_lines
+              | select('match', '^Data Units Read:')
+              | first
+              | regex_replace('^Data Units Read:\s+', '')
+              | regex_replace(' .+$', '')
+              | replace(',', '')
+              | int
+            ),
+            'dataUnitsWritten': (
+              item.stdout_lines
+              | select('match', '^Data Units Written:')
+              | first
+              | regex_replace('^Data Units Written:\s+', '')
+              | regex_replace(' .+$', '')
+              | replace(',', '')
+              | int
+            ),
+            'powerCycles': (
+              item.stdout_lines
+              | select('match', '^Power Cycles:')
+              | first
+              | regex_replace('^Power Cycles:\s+', '')
+              | replace(',', '')
+              | int
+            ),
+            'powerOnHours': (
+              item.stdout_lines
+              | select('match', '^Power On Hours:')
+              | first
+              | regex_replace('^Power On Hours:\s+', '')
+              | replace(',', '')
+              | int
+            ),
+            'unsafeShutdowns': (
+              item.stdout_lines
+              | select('match', '^Unsafe Shutdowns:')
+              | first
+              | regex_replace('^Unsafe Shutdowns:\s+', '')
+              | replace(',', '')
+              | int
+            ),
+            'mediaAndDataIntegrityErrors': (
+              item.stdout_lines
+              | select('match', '^Media and Data Integrity Errors:')
+              | first
+              | regex_replace('^Media and Data Integrity Errors:\s+', '')
+              | replace(',', '')
+              | int
+            ),
+            'errorInformationLogEntries': (
+              item.stdout_lines
+              | select('match', '^Error Information Log Entries:')
+              | first
+              | regex_replace('^Error Information Log Entries:\s+', '')
+              | replace(',', '')
+              | int
+            )
+          }
+        ]
+      }}
+  loop: "{{ cloudfall_inspect_smartctl_devices.results | default([]) }}"
+  loop_control:
+    label: "{{ item.item.path }}"
+  when:
+    - cloudfall_inspect_smartctl_binary.stat.exists
+    - not item.skipped | default(false)
+
+- name: Collect mounted filesystem sizes
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/findmnt
+      - --json
+      - --bytes
+      - --list
+      - --output
+      - TARGET,SOURCE,FSTYPE,OPTIONS,SIZE,USED,AVAIL
+  register: cloudfall_inspect_findmnt
+  changed_when: false
+
+- name: Check whether Linux software RAID status is available
+  ansible.builtin.stat:
+    path: /proc/mdstat
+    get_checksum: false
+  register: cloudfall_inspect_mdstat_file
+
+- name: Read Linux software RAID status
+  ansible.builtin.slurp:
+    src: /proc/mdstat
+  register: cloudfall_inspect_mdstat
+  when: cloudfall_inspect_mdstat_file.stat.exists
+
+- name: Check whether mdadm is installed
+  ansible.builtin.stat:
+    path: /usr/sbin/mdadm
+    get_checksum: false
+  register: cloudfall_inspect_mdadm_binary
+
+- name: Collect mdadm array definitions
+  ansible.builtin.command:
+    argv:
+      - /usr/sbin/mdadm
+      - --detail
+      - --scan
+      - --verbose
+  register: cloudfall_inspect_mdadm
+  changed_when: false
+  when: cloudfall_inspect_mdadm_binary.stat.exists
+
+- name: Collect allowlisted configuration metadata and hashes
+  ansible.builtin.stat:
+    path: "{{ item.path }}"
+    checksum_algorithm: sha256
+    get_checksum: "{{ item.capture == 'hash' }}"
+  loop: "{{ cloudfall_inspect_server_type.configuration.files }}"
+  loop_control:
+    label: "{{ item.path }}"
+  register: cloudfall_inspect_configuration_stats
+
+- name: Initialize configuration evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_configuration: []
+
+- name: Normalize configuration evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_configuration: >-
+      {{
+        cloudfall_inspect_configuration
+        + [
+          {
+            'path': item.item.path,
+            'capture': item.item.capture,
+            'exists': item.stat.exists,
+            'owner': item.stat.pw_name | default(none),
+            'group': item.stat.gr_name | default(none),
+            'mode': item.stat.mode | default(none),
+            'size': item.stat.size | default(none),
+            'sha256': item.stat.checksum | default(none)
+          }
+        ]
+      }}
+  loop: "{{ cloudfall_inspect_configuration_stats.results }}"
+  loop_control:
+    label: "{{ item.item.path }}"
+
+- name: Keep systemd service-unit observations only
+  ansible.builtin.set_fact:
+    cloudfall_inspect_services: >-
+      {{
+        ansible_facts["services"]
+        | dict2items
+        | selectattr('value.source', 'equalto', 'systemd')
+        | items2dict
+      }}
+
+- name: Collect systemd timer unit states
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/systemctl
+      - list-units
+      - --type=timer
+      - --all
+      - --output=json
+      - --no-pager
+  register: cloudfall_inspect_timer_units
+  changed_when: false
+
+- name: Collect systemd timer unit files
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/systemctl
+      - list-unit-files
+      - --type=timer
+      - --output=json
+      - --no-pager
+  register: cloudfall_inspect_timer_files
+  changed_when: false
+
+- name: Normalize systemd timer enablement evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_timer_statuses: >-
+      {{
+        cloudfall_inspect_timer_files.stdout
+        | from_json
+        | items2dict(key_name='unit_file', value_name='state')
+      }}
+
+- name: Initialize systemd timer evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_timers: {}
+
+- name: Normalize systemd timer evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_timers: >-
+      {{
+        cloudfall_inspect_timers
+        | combine({
+            item.unit: {
+              'name': item.unit,
+              'state': 'running' if item.active == 'active' else 'stopped',
+              'status': cloudfall_inspect_timer_statuses.get(
+                item.unit, 'unknown'
+              ),
+              'source': 'systemd'
+            }
+          })
+      }}
+  loop: "{{ cloudfall_inspect_timer_units.stdout | from_json }}"
+  loop_control:
+    label: "{{ item.unit }}"
+
+- name: Collect listening sockets
+  ansible.builtin.command:
+    argv:
+      - /usr/bin/ss
+      - -tulnH
+  register: cloudfall_inspect_listening_sockets
+  changed_when: false
+
+- name: Check whether nft is installed
+  ansible.builtin.stat:
+    path: /usr/sbin/nft
+    get_checksum: false
+  register: cloudfall_inspect_nft_binary
+
+- name: Read the managed nftables table
+  ansible.builtin.command:
+    argv:
+      - /usr/sbin/nft
+      - --json
+      - list
+      - table
+      - inet
+      - cloudfall
+  register: cloudfall_inspect_nft_table
+  changed_when: false
+  failed_when: false
+  when: cloudfall_inspect_nft_binary.stat.exists
+
+- name: Build normalized observation snapshot
+  ansible.builtin.set_fact:
+    cloudfall_inspect_snapshot:
+      apiVersion: cloudfall/v1
+      kind: ObservedServer
+      metadata:
+        id: "{{ cloudfall_inspect_server_id }}"
+        description: Read-only server evidence collected by Cloudfall
+      spec:
+        server: "{{ cloudfall_inspect_server_id }}"
+        serverType: "{{ cloudfall_inspect_server_type.id }}"
+        observedAt: "{{ ansible_facts['date_time']['iso8601'] }}"
+        collectorVersion: "{{ cloudfall_inspect_collector_version }}"
+        os:
+          distribution: "{{ ansible_facts['distribution'] }}"
+          version: "{{ cloudfall_inspect_debian_version }}"
+          majorVersion: "{{ cloudfall_inspect_debian_version.split('.')[0] }}"
+          release: "{{ ansible_facts['distribution_release'] }}"
+          kernel: "{{ ansible_facts['kernel'] }}"
+          architecture: "{{ ansible_facts['architecture'] }}"
+          serviceManager: "{{ ansible_facts['service_mgr'] }}"
+        hardware:
+          processorCount: "{{ ansible_facts['processor_count'] | int }}"
+          processorCores: "{{ ansible_facts['processor_cores'] | int }}"
+          processorVcpus: "{{ ansible_facts['processor_vcpus'] | int }}"
+          memoryBytes: >-
+            {{ (ansible_facts['memtotal_mb'] | int) * 1048576 }}
+        storage:
+          blockDevices: "{{ cloudfall_inspect_block_devices }}"
+          filesystems: >-
+            {{ (cloudfall_inspect_findmnt.stdout | from_json).filesystems }}
+          softwareRaid:
+            mdstat: >-
+              {{
+                (cloudfall_inspect_mdstat.content | b64decode)
+                if cloudfall_inspect_mdstat_file.stat.exists
+                else ''
+              }}
+            mdadmScan: "{{ cloudfall_inspect_mdadm.stdout | default('') }}"
+          smart:
+            available: "{{ cloudfall_inspect_smartctl_binary.stat.exists }}"
+            smartctlVersion: >-
+              {{
+                cloudfall_inspect_smartctl_version.stdout_lines[0].split()[1]
+                if cloudfall_inspect_smartctl_binary.stat.exists
+                else none
+              }}
+            devices: "{{ cloudfall_inspect_smart_devices }}"
+        packages: "{{ cloudfall_inspect_packages }}"
+        services: "{{ cloudfall_inspect_services }}"
+        timers: "{{ cloudfall_inspect_timers }}"
+        network:
+          listeningSockets: "{{ cloudfall_inspect_listening_sockets.stdout }}"
+        firewall:
+          nftAvailable: "{{ cloudfall_inspect_nft_binary.stat.exists }}"
+          managedTableJson: >-
+            {{
+              cloudfall_inspect_nft_table.stdout
+              if (
+                cloudfall_inspect_nft_binary.stat.exists
+                and cloudfall_inspect_nft_table.rc == 0
+              )
+              else none
+            }}
+        componentEnvironment: >-
+          {{ cloudfall_inspect_component_environment | default([]) }}
+        configuration: "{{ cloudfall_inspect_configuration }}"
+
+- name: Capture component environment file evidence
+  ansible.builtin.stat:
+    path: >-
+      {{ cloudfall_inspect_component_item.install_root }}/shared/env/{{
+        cloudfall_inspect_component_item.id
+      }}.env
+    checksum_algorithm: sha256
+  loop: "{{ cloudfall_inspect_components }}"
+  loop_control:
+    loop_var: cloudfall_inspect_component_item
+    label: "{{ cloudfall_inspect_component_item.id }}"
+  register: cloudfall_inspect_component_env
+
+- name: Normalize component environment evidence
+  ansible.builtin.set_fact:
+    cloudfall_inspect_component_environment: >-
+      {{
+        cloudfall_inspect_component_environment | default([]) + [{
+          'component':
+            cloudfall_inspect_env_result.cloudfall_inspect_component_item.id,
+          'path':
+            cloudfall_inspect_env_result.cloudfall_inspect_component_item
+              .install_root ~ '/shared/env/'
+            ~ cloudfall_inspect_env_result.cloudfall_inspect_component_item.id
+            ~ '.env',
+          'exists': cloudfall_inspect_env_result.stat.exists,
+          'sha256': cloudfall_inspect_env_result.stat.checksum | default(none),
+        }]
+      }}
+  loop: "{{ cloudfall_inspect_component_env.results | default([]) }}"
+  loop_control:
+    loop_var: cloudfall_inspect_env_result
+    label: >-
+      {{ cloudfall_inspect_env_result.cloudfall_inspect_component_item.id }}
+
+- name: Collect Prometheus alert rule evidence
+  ansible.builtin.uri:
+    url: >-
+      http://{{
+        cloudfall_inspect_logging_backend.metrics.backend.listenAddress
+      }}:{{
+        cloudfall_inspect_logging_backend.metrics.backend.port
+      }}/api/v1/rules?type=alert
+    return_content: true
+  register: cloudfall_inspect_prometheus_rules
+  failed_when: false
+  when: cloudfall_inspect_logging_backend | length > 0
+
+- name: Collect Prometheus active alert evidence
+  ansible.builtin.uri:
+    url: >-
+      http://{{
+        cloudfall_inspect_logging_backend.metrics.backend.listenAddress
+      }}:{{
+        cloudfall_inspect_logging_backend.metrics.backend.port
+      }}/api/v1/alerts
+    return_content: true
+  register: cloudfall_inspect_prometheus_alerts
+  failed_when: false
+  when: cloudfall_inspect_logging_backend | length > 0
+
+- name: Attach alerting evidence for the logging backend
+  ansible.builtin.set_fact:
+    cloudfall_inspect_snapshot: >-
+      {{
+        cloudfall_inspect_snapshot | combine(
+          {
+            'spec': {
+              'alerting': {
+                'prometheusAvailable': (
+                  cloudfall_inspect_prometheus_rules.status | default(0) == 200
+                  and cloudfall_inspect_prometheus_alerts.status
+                  | default(0) == 200
+                ),
+                'rulesJson': (
+                  cloudfall_inspect_prometheus_rules.content
+                  if cloudfall_inspect_prometheus_rules.status
+                  | default(0) == 200
+                  else none
+                ),
+                'alertsJson': (
+                  cloudfall_inspect_prometheus_alerts.content
+                  if cloudfall_inspect_prometheus_alerts.status
+                  | default(0) == 200
+                  else none
+                ),
+              }
+            }
+          },
+          recursive=true
+        )
+      }}
+  when: cloudfall_inspect_logging_backend | length > 0
+
+- name: Create controller observation directory
+  ansible.builtin.file:
+    path: "{{ cloudfall_inspect_output_directory }}"
+    state: directory
+    mode: "0700"
+  delegate_to: localhost
+  become: false
+
+- name: Write observation snapshot on controller
+  ansible.builtin.copy:
+    content: "{{ cloudfall_inspect_snapshot | to_nice_json }}\n"
+    dest: >-
+      {{ cloudfall_inspect_output_directory }}/{{ cloudfall_inspect_server_id }}.json
+    mode: "0600"
+  delegate_to: localhost
+  become: false

cloudfall_engine/_bundled/ansible/roles/cloudfall_logging_backend/defaults/main.yml

@@ -0,0 +1,6 @@
+---
+cloudfall_logging_backend_stack: {}
+cloudfall_logging_backend_grafana_repository_key_url: >-
+  https://apt.grafana.com/gpg-full.key
+cloudfall_logging_backend_grafana_repository_key_path: /etc/apt/keyrings/grafana.asc
+cloudfall_logging_backend_grafana_repository_url: https://apt.grafana.com

cloudfall_engine/_bundled/ansible/roles/cloudfall_logging_backend/handlers/main.yml

@@ -0,0 +1,29 @@
+---
+- name: Restart Loki
+  ansible.builtin.systemd_service:
+    name: loki.service
+    state: restarted
+    daemon_reload: true
+
+- name: Restart Prometheus
+  ansible.builtin.systemd_service:
+    name: prometheus.service
+    state: restarted
+    daemon_reload: true
+
+- name: Restart Alertmanager
+  ansible.builtin.systemd_service:
+    name: prometheus-alertmanager.service
+    state: restarted
+    daemon_reload: true
+
+- name: Restart Grafana
+  ansible.builtin.systemd_service:
+    name: grafana-server.service
+    state: restarted
+    daemon_reload: true
+
+- name: Reload Nginx
+  ansible.builtin.systemd_service:
+    name: nginx.service
+    state: reloaded