cmdbsyncer 4.4.0__tar.gz → 4.5.0.dev0__tar.gz

{cmdbsyncer-4.4.0 → cmdbsyncer-4.5.0.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: cmdbsyncer
-Version: 4.4.0
+Version: 4.5.0.dev0
 Summary: Rule-based modular synchronizer for Checkmk, Netbox, Ansible, I-DOIT, Jira CMDB and 15+ other CMDB/monitoring systems
 Author-email: Kuhn & Ruess GmbH <info@kuhn-ruess.de>
 Maintainer-email: Bastian Kuhn <mail@bastian-kuhn.de>
@@ -56,6 +56,7 @@ Requires-Dist: mysql-connector-python==9.1.0; extra == "extras"
 Requires-Dist: pyvim==3.0.3; extra == "extras"
 Requires-Dist: pyvmomi==8.0.3.0.1; extra == "extras"
 Requires-Dist: mcp==1.10.1; extra == "extras"
+Requires-Dist: fpdf2==2.8.9; extra == "extras"
 Provides-Extra: ansible
 Requires-Dist: ansible==10.5.0; extra == "ansible"
 Requires-Dist: cmdbsyncer-inventory>=0.2.0; extra == "ansible"

{cmdbsyncer-4.4.0 → cmdbsyncer-4.5.0.dev0}/application/__init__.py

@@ -467,6 +467,10 @@ if not CLI_MODE:
             insecure = False
         return {'show_https_warning': insecure}
 
+    # pylint: disable-next=import-outside-toplevel
+    from application.helpers.page_title import page_title
+    app.add_template_global(page_title)
+
     @app.template_filter('localtime')
     def _localtime(value, fmt='%Y-%m-%d %H:%M:%S'):
         """
@@ -764,6 +768,28 @@ def _register_web_layer():  # pylint: disable=too-many-locals,too-many-statement
 
     _register_all_plugin_admin_views(admin)
 
+    #   .-- Modules: shared data at the bottom
+    # Lookup Tables and the Secret Store serve every module, so they sit
+    # below the modules, after a divider. Registered after the plugins
+    # because Flask-Admin keeps the registration order.
+    from application.views.menu_links import AccessMenuLink
+    from application.models.lookup_table import LookupTable
+    from application.views.lookup_table import LookupTableView
+    from application.models.secret import Secret
+    from application.views.secret import SecretView
+    lookup_view = LookupTableView(LookupTable, name="Lookup Tables",
+                                  category="Modules",
+                                  menu_icon_type='fa', menu_icon_value='fa-table')
+    secret_view = SecretView(Secret, name="Secret Store", endpoint="secret",
+                             category="Modules",
+                             menu_icon_type='fa', menu_icon_value='fa-lock')
+    admin.add_link(AccessMenuLink(
+        name='', url='#', category='Modules', class_name='dropdown-divider',
+        access=lambda: lookup_view.is_accessible() or secret_view.is_accessible()))
+    admin.add_view(lookup_view)
+    admin.add_view(secret_view)
+    #.
+
     #   .-- Top bar grouping
     # The bar holds three groups: the working views (Objects, Modules,
     # Projects), the read-only tools (Log, Files) and the administration
@@ -811,7 +837,6 @@ def _register_web_layer():  # pylint: disable=too-many-locals,too-many-statement
     # compact. Each of the three topics gets its own sub menu, so Settings
     # itself stays a short list.
     from application.views.account import AccountModelView, ChildAccountModelView
-    from application.views.menu_links import AccessMenuLink
     from flask_login import current_user as _menu_user
 
     _add_sub_menu(admin, "Accounts", 'fa-user-circle')

{cmdbsyncer-4.4.0 → cmdbsyncer-4.5.0.dev0}/application/api/rules.py

@@ -235,9 +235,9 @@ class RulesExport(Resource):
                'passwords and role assignments — treat as secret. Skipped by '
                'default.')
     @API.param('include_passwords',
-               'Set to ``1`` to also export the Checkmk password store '
+               'Set to ``1`` to also export the Secret Store '
                '(``cmk_passwords``). The entries stay encrypted with this '
-               'instance key — treat as secret. Skipped by default.')
+               'instance key, treat as secret. Skipped by default.')
     @API.response(200, 'Every rule, grouped by ``rule_type``.',
                   RULES_EXPORT_RESPONSE)
     @API.response(401, 'Authentication failed', ERROR)

{cmdbsyncer-4.4.0 → cmdbsyncer-4.5.0.dev0}/application/api/syncer.py

@@ -72,6 +72,8 @@ CRON_STATUS = API.model('cron_status', {
     'is_running': fields.Boolean(description='True while a run is in flight'),
     'last_message': fields.String(description='Last progress / result line'),
     'has_error': fields.Boolean(description='True iff the last run failed'),
+    'enabled': fields.Boolean(description='False if the group is disabled '
+                                          'or no longer exists'),
 })
 
 CRON_LIST_RESPONSE = API.model('cron_list_response', {
@@ -247,6 +249,8 @@ class SyncerCronApi(Resource):
         """Return the current status of every CronGroup."""
         try:
             response = []
+            enabled = {group.name: bool(group.enabled)
+                       for group in CronGroup.objects.only('name', 'enabled')}
             for entry in CronStats.objects:
                 last_start = entry.last_start
                 if last_start:
@@ -261,6 +265,7 @@ class SyncerCronApi(Resource):
                     'is_running': entry.is_running,
                     'last_message': entry.last_message,
                     'has_error': entry.failure,
+                    'enabled': enabled.get(entry.group, False),
                 })
             return {
                 'result': response,

cmdbsyncer-4.5.0.dev0/application/buildinfo.txt

@@ -0,0 +1 @@
+BUILD_DATE=2026-10-29T12:50:13Z

{cmdbsyncer-4.4.0 → cmdbsyncer-4.5.0.dev0}/application/changelog/v4.3.md

@@ -17,6 +17,7 @@
 - FIX: An import source delivering a hostname that is not text now reports a readable error instead of a traceback
 - FIX: Inventorize runs only write a host to the database when an attribute really changed, which noticeably speeds up large Netbox inventories
 - FIX: A database problem while looking up CMDB templates is written to the log instead of silently looking like "no template matched"
+- FIX: A Checkmk "Move to Folder" or "Create Empty Folder" value wrapped in a Jinja condition, like `{% if var is defined %}/{{var}}|{'title': '{{var}}'}{% endif %}`, can be saved again instead of being rejected as invalid folder options
 
 ## Version 4.3.5
 - FIX: Inventorize applied `rewrite_hostname` twice for CSV, JSON, REST, YML, MySQL, ODBC, JDisc and LDAP accounts, so a template that appends something to the name, a domain for example, missed the host the import had created

{cmdbsyncer-4.4.0 → cmdbsyncer-4.5.0.dev0}/application/changelog/v4.4.md

@@ -1,5 +1,39 @@
 # Version 4.4
 
+## Unreleased
+- FIX: REST import: entries without the hostname field are skipped instead of aborting the whole import
+- FIX: Checkmk users: an empty row under Roles or Contact Groups is no longer sent to Checkmk, which rejected the user because of it
+- FIX: Files: creating a folder, uploading, renaming or editing a file works again when the syncer runs under a sub-path such as /cmdbsyncer — the dialog sent its form to the address without that path
+- FEAT: Accounts: a new import source can run as "Shadow Source" next to the current one — its data is stored beside the hosts without changing them, and the host details and Debug Host compare it with the live data and the rule outcomes; the shadow data can be cleared from the account and host list
+- FIX: Checkmk Site Pools: the member sites of a switched-off pool are readable again in the dark themes
+- FEAT: Checkmk DCD Rules: the text fields have the Jinja help, and {{ HOST_FOLDER }} gives the folder the host export puts the host in and {{ HOST_POOL_SITE }} the site its site pool assigned
+- FEAT: Checkmk Setup Rules: the Jinja help of the value shows how a password in the rule points to an entry of the Secret Store
+- FIX: Checkmk password export no longer fails for Secret Store entries without a comment, such as secrets written by an import — the entry's Checkmk ident is used as comment
+- FIX: Rules with a Pasted List: switching the rule source to the list no longer hides the Enabled and Last Match checkboxes
+- FIX: Host list: the page is about a fifth smaller for hosts with many labels and loads faster
+- FIX: Checkmk host export: a folder Checkmk refuses to create is reported in the terminal with Checkmk's reason — before, only the hosts in it failed later with "folder could not be found"
+- FIX: Exports to Checkmk, Netbox, i-doit, VMware and Jira write each host once per run instead of once per rule set and keep less in memory on large host counts
+- FIX: Checkmk rule export: CMDB templates are read once per run instead of once for every host
+- FIX: Ansible Checkmk agent playbook: a failed host creation, agent download, registration or discovery shows the error message instead of only "the output has been hidden", with the automation secret masked
+- FIX: Ansible Checkmk agent playbook: hosts without any agent step switched on are skipped before Ansible connects to them
+- FIX: Ansible inventory: several times faster and with considerably less memory on large host counts
+- FIX: Ansible projects no longer hand out each other's cached variables, each project computes its own
+- FIX: `ansible update_cache` rebuilds the whole Ansible cache instead of reusing the old values
+- FIX: Checkmk rule export: a rule with Keep Value is no longer deleted and created again on every run when its comment in Checkmk differs, so the value adjusted in Checkmk, including a password, is kept
+- FIX: Checkmk rule export: correcting the description of a rule no longer replaces a password in its value with the masked `******` Checkmk shows
+- FIX: Checkmk rule export: when several Setup Rules produce the very same Checkmk rule, it is created only once and its description no longer changes on every run
+- FIX: Checkmk rule export: when Checkmk holds an outdated copy of a rule next to the current one, the outdated copy is deleted instead of first being updated and the current one deleted as a duplicate
+- FIX: Checkmk rule export: an update of a rule that runs into the request timeout is no longer sent to Checkmk a second time, so a slow Checkmk costs one timeout per rule instead of two and the update is not applied twice
+- FIX: The host list shows the Search options button only once again; after a search it appeared a second time next to the filters
+- FIX: The Checkmk Settings offer the editions of Checkmk 2.5 (Community, Pro, Ultimate, Ultimate with Multi-Tenancy), so the Ansible site management can install and update 2.5 sites
+- FIX: The Ansible site update updates sites to Checkmk 2.5 without stopping at its version and edition questions, and stops on a failed pre-flight check instead of continuing with a broken site
+- FIX: The searchable Checkmk ruleset picker in the Setup rule form now also lists the rulesets when the Syncer was installed with pip, the ruleset lists were missing from the package
+- FIX: CMDB templates are listed alphabetically in the Templates view and in every template selection, like Set Template on the host list
+- FIX: Searching or filtering the host list by a label or inventory value now also finds numbers, e.g. `vlan:5` for a label stored as the number 5, including partial matches and regex
+- FIX: A Checkmk "Move to Folder" or "Create Empty Folder" value wrapped in a Jinja condition, like `{% if var is defined %}/{{var}}|{'title': '{{var}}'}{% endif %}`, can be saved again instead of being rejected as invalid folder options
+- FIX: Host Debug page now lists every condition that matched for an "ALL must match" rule instead of only one, and the hint for a matching rule without a new outcome no longer claims an earlier rule set the value when the outcome was simply empty
+- FIX: The host action "Redistribute Site Pool" now really moves the selected hosts: the next export assigns them the least-loaded site of their pool again instead of keeping the old one
+
 ## Version 4.4.0
 
 ### Setup

cmdbsyncer-4.5.0.dev0/application/changelog/v4.5.md

@@ -0,0 +1,29 @@
+# Version 4.5
+
+## Unreleased
+
+- FEAT: API: the cron status (`/api/v1/syncer/cron/`) tells whether a cron group is enabled, so the Checkmk plugin no longer monitors disabled or deleted cron groups
+- FEAT: Checkmk: new Manage LDAP Connections maintains the LDAP connections Checkmk logs its users in with, including contact groups and roles from directory groups; server and bind password come from an LDAP account, `checkmk export_ldap_connections` creates, updates or removes them
+- FEAT: `rules export_all_rules --zip` writes the export compressed into a zip archive, and `rules import_rules` reads such an archive back as it is
+- FEAT: Checkmk Settings: Fill in from checkmk.com loads the current Checkmk versions on click and fills in version, edition and the filename of the installation file for the chosen platform, including the Checkmk Appliance; a new Download Proxy is used for the download of the installation file and for this list
+- FEAT: Checkmk Setup Rules: the new option Per Template evaluates a rule once per CMDB template of the host, each time with only that template's labels, so a contact group and the services of one template no longer mix with those of another template on the same host; the host debug page shows these rules per template
+- FEAT: The account list is grouped by account type, each group under a heading with the type's name and its number of accounts that folds the group away like the Global Custom Attributes, with Expand all and Collapse all; sorting by a column shows the plain list
+- FEAT: Browser tab titles name the page first, e.g. "Edit web01 · Hosts · Objects" or "Filter · Rules · Ansible", so open tabs can be told apart
+- FEAT: Host list: new filter Checkmk Attribute Key:Value finds hosts by the attributes of their last Checkmk sync, the green label badges
+- FEAT: Host list: show up to 500 hosts per page
+- FEAT: Host list: new action Remove Template takes one CMDB template off the selected hosts and keeps their other templates
+- FEAT: Lookup Tables: maintain a long key/value list in the UI and read it in any Jinja field with `{{ lookup('table', key) }}`, e.g. to set an attribute from the value of another one
+- FEAT: Custom Attributes and Rewrite Attributes: a Jinja help next to the template fields lists the lookup tables, variables, functions and filters and inserts them with one click
+- FEAT: Global Custom Attributes: the rule list is grouped by the attribute each rule sets (a rule with several attributes in a group of that combination), shows every rule's place in the order the rules run, and the order can be changed by drag and drop or with the arrow keys, including moving a rule into another attribute's group. Each group folds away with a click on its heading
+- FEAT: Secret Store: the Checkmk password store is now Modules > Secret Store and holds passwords, tokens and certificates for every module, global or bound to a host. Rules reach a host's entry with `{{ secret('key') }}` (plaintext with `{{ secret_value('key') }}`), entries of an archived host are kept as orphaned and can be bound to another host, certificates are shown for copying, and the host overview lists the entries of the host with masked values. New roles: Secret Store, and Secret Store: show plaintext values. Users limited to accounts or templates only reach entries of their own hosts, never global ones. Lookup Tables and the Secret Store sit at the bottom of the Modules menu
+- FEAT: Host import: Import CSV / YAML also takes YAML files (several at once, several hosts per file as a list or as documents separated by ---, also YAML-like files whose certificate lines are out of indentation), and "Bulk add hosts" is now "Add hosts manually" with a format switch (auto, host list, CSV, YAML, key/value list)
+- FEAT: Host import preview: every import and every manual add opens a preview where the values of each host, added columns and a template per host can be edited, chosen columns go into the Secret Store instead of becoming labels, and the account of the imported hosts is picked. Users limited to accounts only get their own accounts and cannot overwrite hosts outside their scope
+- FEAT: Checkmk Setup Rules: the Jinja help sits next to the Value Template and the Folder, the new placeholder `{{ HOST_FOLDER }}` puts a rule into the folder its host lands in (rules per host only), and the Host label and Service label conditions are entered as compact key and value pairs with suggestions from the existing labels. The Host label condition now takes several labels as well, all of them must match
+- FEAT: Checkmk > Folders and Folder Options: the folders the host export would use, with the hosts landing in each and the folder options (contact groups, permissions, title, ...) sent to Checkmk, per Checkmk account. The debug of a folder shows which host and which Export Rule with which conditions gave which option and how the values merge. Calculated from the syncer alone without touching Checkmk or the pools, users limited to accounts or templates only see the folders and options of their own hosts
+- FEAT: Checkmk menu sorted into groups with dividers: the host export rules with the Pools first, then the rules for setup rules, groups, tags, users, downtimes, notifications, DCD and BI, then the Checkmk Server and the inventory from Checkmk, and at the bottom the views to check the result (Folders and Folder Options, Preview Generated Users, Data Quality Check, Cache). The menus are a little denser so the long ones fit the screen
+- FEAT: Checkmk > Data Quality > Configuration Checks: pick a Checkmk account and the checks to run and see the problems they find, split by topic with a summary of the counts on top, and download them as a PDF report in the syncer design (needs the optional package fpdf2 from the extras, the page works without it). The first check looks at the HTTP checks: services that are WARN, CRIT or UNKNOWN in the monitoring right now with their output and the rule behind them, the same URL checked twice, website and certificate checked by two separate checks that one check_httpv2 check can do, and old check_http rules that are not migrated or still run next to their migrated rule. The checks only read from Checkmk. The Data Quality Check moved into the same Data Quality submenu
+
+## Version 4.5.0
+
+(Initial 4.5 release — entries currently live in the Unreleased section
+above. They get folded down into this section when the release is cut.)

{cmdbsyncer-4.4.0 → cmdbsyncer-4.5.0.dev0}/application/docu_links.py

@@ -10,12 +10,17 @@ with the structure at https://docs.cmdbsyncer.de/.
 docu_links = {
     'rewrite' : 'https://docs.cmdbsyncer.de/basics/rewrite_attributes/',
     'accounts' : 'https://docs.cmdbsyncer.de/basics/accounts/',
+    'lookup_tables': 'https://docs.cmdbsyncer.de/basics/lookup_tables/',
+    'jinja_functions': 'https://docs.cmdbsyncer.de/advanced/jinja_functions/',
     'cmk_inventory_attributes': 'https://docs.cmdbsyncer.de/checkmk/inventorize/',
     'cmk_groups': 'https://docs.cmdbsyncer.de/checkmk/groups_management/',
     'cmk_user_generation': 'https://docs.cmdbsyncer.de/checkmk/user_generation/',
     'cmk_setup_rules': 'https://docs.cmdbsyncer.de/checkmk/rules_management/',
+    'cmk_sites': 'https://docs.cmdbsyncer.de/ansible/cmk_sites/',
     'cmk_hosttags': 'https://docs.cmdbsyncer.de/checkmk/create_hosttags/',
     'cmk_password_store': 'https://docs.cmdbsyncer.de/checkmk/password_store/',
+    'cmk_ldap_connections': 'https://docs.cmdbsyncer.de/checkmk/ldap_connections/',
+    'cmk_folders': 'https://docs.cmdbsyncer.de/checkmk/folders/',
 
     # Enterprise features — referenced from cmdbsyncer_enterprise views.
     'ent_overview': 'https://docs.cmdbsyncer.de/enterprise/',

cmdbsyncer-4.5.0.dev0/application/helpers/host_import_file.py

@@ -0,0 +1,229 @@
+"""
+YAML rows and Secret Store columns for the manual host import.
+
+The host list imports a CSV through a preview where the user picks the
+hostname column. YAML (an uploaded file or text pasted into Manual add) goes
+through the same preview: a mapping of ``key: value`` pairs is one host, a
+list of mappings or several documents separated by ``---`` are several
+hosts, and every key becomes a column. Columns the user marks as secret are not written as
+labels but into the Secret Store, bound to the imported host.
+"""
+import json
+import re
+
+import yaml
+
+from application.helpers.secret_access import can_bind
+from application.models.secret import store_host_secret
+
+YAML_SUFFIXES = ('.yml', '.yaml')
+
+# Column names preselected as secret on the preview.
+SECRET_HINTS = ('token', 'password', 'secret', 'certificate', 'cert')
+
+
+# A first line like ``cluster: name``, ``---`` or ``- key: value``. A
+# hostname never holds ``: ``, so a plain hostname list is never taken
+# for YAML.
+_YAML_START = re.compile(r'^(---|- |[A-Za-z0-9_.-]+:(\s|$))')
+
+
+def is_yaml_filename(filename):
+    """True for a file the import reads as YAML."""
+    return (filename or '').lower().endswith(YAML_SUFFIXES)
+
+
+def looks_like_yaml(text):
+    """True when pasted text is YAML rather than hostnames or CSV."""
+    for line in (text or '').splitlines():
+        stripped = line.strip()
+        if not stripped or stripped.startswith('#'):
+            continue
+        return bool(_YAML_START.match(stripped))
+    return False
+
+
+def _cell(value):
+    """A YAML value as the string a CSV cell would hold."""
+    if value is None:
+        return ''
+    if isinstance(value, (dict, list)):
+        return json.dumps(value)
+    return str(value)
+
+
+def _strict_entries(text):
+    """Mappings of ``text`` read as real YAML; ValueError when it is not."""
+    try:
+        documents = list(yaml.safe_load_all(text))
+    except yaml.YAMLError as exc:
+        raise ValueError(str(exc)) from exc
+    entries = []
+    for data in documents:
+        if data is None:
+            continue
+        for entry in data if isinstance(data, list) else [data]:
+            if not isinstance(entry, dict):
+                raise ValueError("expected 'key: value' lines")
+            entries.append(entry)
+    return entries
+
+
+# Top-level key of the YAML-like format: ``key: value`` or ``key:``.
+_FLAT_KEY = re.compile(r'^([A-Za-z_][\w.-]*):(?:\s+(.*))?$')
+_BLOCK_INDICATOR = re.compile(r'^[|>][-+]?$')
+
+
+def _block_value(indicator, lines):
+    """
+    Value of a ``key: |`` / ``key: >`` block in the YAML-like format.
+
+    Its lines may carry any indentation (certificate bodies often start at
+    column 0 between indented BEGIN/END lines), so every line is stripped
+    on the left; a PEM chain comes out as clean, unindented lines.
+    """
+    while lines and not lines[-1].strip():
+        lines = lines[:-1]
+    stripped = [line.strip() for line in lines]
+    if indicator.startswith('>'):
+        value = ' '.join(line for line in stripped if line)
+    else:
+        value = '\n'.join(stripped)
+    if not indicator.endswith('-'):
+        value += '\n'
+    return value
+
+
+def _plain_value(value):
+    """A plain ``key: value`` value, surrounding quotes removed."""
+    value = (value or '').strip()
+    if len(value) > 1 and value[0] == value[-1] and value[0] in ('"', "'"):
+        return value[1:-1]
+    return value
+
+
+# A ``key: value``, ``key=value`` or ``key<TAB>value`` line of a pasted
+# key/value list; split at the first separator, so URLs keep their port.
+_KV_LINE = re.compile(r'^\s*([A-Za-z_][\w .-]*?)\s*(?::|=|\t)\s*(.*)$')
+
+
+def _flat_entries(text, filename, key_value=False):
+    """
+    Mappings of the YAML-like format: flat ``key: value`` lines, a ``key: |``
+    block continues over every following line (whatever its indentation)
+    up to the next top-level key, ``---`` separates hosts. Raises
+    ValueError naming the first line that fits none of this.
+
+    With ``key_value`` it reads a pasted key/value list instead: lines may
+    also be ``key=value`` or ``key<TAB>value`` and an empty line separates
+    hosts as well. Inside a block only a ``key:`` line ends it, so base64
+    lines ending in ``=`` stay part of a certificate.
+    """
+    entries, current, block = [], {}, None
+
+    def close_block():
+        if block:
+            current[block[0]] = _block_value(block[1], block[2])
+
+    def close_host():
+        if current:
+            entries.append(dict(current))
+        current.clear()
+
+    for number, line in enumerate(text.splitlines(), start=1):
+        if line.strip() == '---' or (key_value and not line.strip()):
+            close_block()
+            block = None
+            close_host()
+            continue
+        match = _FLAT_KEY.match(line.rstrip())
+        if match is None and key_value and block is None:
+            match = _KV_LINE.match(line.rstrip())
+        if match:
+            close_block()
+            block = None
+            key, value = match.group(1).strip(), (match.group(2) or '').strip()
+            if _BLOCK_INDICATOR.match(value):
+                block = (key, value, [])
+            else:
+                current[key] = _plain_value(value)
+            continue
+        if block:
+            block[2].append(line)
+            continue
+        if not line.strip() or line.lstrip().startswith('#'):
+            continue
+        raise ValueError(f"{filename}, line {number}: not a 'key: value' line")
+    close_block()
+    close_host()
+    return entries
+
+
+def key_value_rows(text, filename='pasted text'):
+    """Rows of a pasted key/value list (see ``_flat_entries``)."""
+    return [{key: _cell(value) for key, value in entry.items()}
+            for entry in _flat_entries(text, filename, key_value=True)]
+
+
+def yaml_rows(files):
+    """
+    Rows of the given ``(filename, text)`` YAML files, one dict per host.
+    A file may hold several hosts: a list of mappings, or several
+    documents separated by ``---`` (each a mapping or a list of them).
+
+    Files that are only YAML-like (block lines out of indentation, as
+    certificate chains often come) are read by a tolerant parser for flat
+    ``key: value`` files when strict YAML fails. Raises ValueError with a
+    short message naming the file and line when both fail.
+    """
+    rows = []
+    for filename, text in files:
+        try:
+            entries = _strict_entries(text)
+        except ValueError:
+            entries = _flat_entries(text, filename)
+        for entry in entries:
+            rows.append({str(key): _cell(value) for key, value in entry.items()})
+    return rows
+
+
+def rows_columns(rows):
+    """``(columns, data_rows)`` of row dicts, columns in first-seen order."""
+    columns = []
+    for row in rows:
+        for key in row:
+            if key not in columns:
+                columns.append(key)
+    data_rows = [[row.get(col, '') for col in columns] for row in rows]
+    return columns, data_rows
+
+
+def suggested_secret_columns(columns, hostname_column):
+    """Columns whose name looks like it holds a secret."""
+    return [col for col in columns
+            if col != hostname_column
+            and any(hint in col.lower() for hint in SECRET_HINTS)]
+
+
+def split_secrets(row, secret_columns):
+    """Take the secret columns out of ``row``; returns them as a dict."""
+    secrets = {}
+    for col in secret_columns:
+        value = row.pop(col, None)
+        if value not in (None, ''):
+            secrets[col] = value
+    return secrets
+
+
+def store_row_secrets(user, host, secrets, source):
+    """
+    Write ``secrets`` into the Secret Store, bound to ``host``, when
+    ``user`` may bind entries to that host. Returns ``(stored, refused)``.
+    """
+    if not secrets:
+        return 0, 0
+    if not can_bind(user, host):
+        return 0, len(secrets)
+    for key, value in secrets.items():
+        store_host_secret(host, key, value, source)
+    return len(secrets), 0

{cmdbsyncer-4.4.0 → cmdbsyncer-4.5.0.dev0}/application/helpers/import_hostnames.py

@@ -102,7 +102,12 @@ def update_import_host(host_obj, labels, config, **kwargs):
     the importing account. Saved only when the account may own it, a
     host of another master account stays as it is. Returns whether it
     was saved. Extra keywords go to `set_account()`.
+
+    A shadow source account only stores its labels as shadow data of
+    the host, see `Host.update_shadow()`.
     """
+    if config.get('shadow_mode'):
+        return host_obj.update_shadow(labels, config)
     host_obj.update_host(labels)
     if host_obj.set_account(account_dict=config, **kwargs):
         host_obj.save()

cmdbsyncer-4.5.0.dev0/application/helpers/jinja_help.py

@@ -0,0 +1,202 @@
+"""
+Jinja help for template fields
+
+What the "Jinja help" panel next to a template field lists: the syncer's
+own functions and filters, a few Jinja basics, the variables a rule hands
+in and the existing lookup tables. Every entry has a snippet that a click
+inserts into the field.
+
+``tests/test_jinja_help.py`` checks that every function and filter
+registered in ``syncer_jinja`` has an entry here, so a new one cannot be
+added without its help.
+"""
+# pylint: disable=import-outside-toplevel
+from application.docu_links import docu_links
+
+# (name, snippet, description). The snippets are one line: the template
+# fields are single line inputs. `attribute` stands for the name of a host
+# attribute.
+FUNCTIONS = (
+    ('lookup',
+     "{{ lookup('table', attribute, default='unknown') }}",
+     "Value of a lookup table (Modules > Lookup Tables) for a key: the second "
+     "column, or the column named as third argument."),
+    ('get_list',
+     "{% for item in get_list(attribute) %}{{ item }},{% endfor %}",
+     "Turns a list, a list literal or a comma separated string into a list."),
+    ('merge_list_of_dicts',
+     "{{ merge_list_of_dicts(attribute)['key'] }}",
+     "Merges a list of dicts into one dict, empty values are dropped."),
+    ('eval',
+     "{{ eval(attribute) }}",
+     "Reads a string holding a Python literal (list, dict, number)."),
+    ('defined',
+     "{{ defined(attribute, 'fallback') }}",
+     "The value, or the fallback when it is empty, 'false' or 'none'."),
+    ('hash_value',
+     "{{ hash_value(attribute) }}",
+     "Short stable hash of a value, e.g. to make a label of a long list."),
+    ('cmk_cleanup_tag_id',
+     "{{ cmk_cleanup_tag_id(attribute) }}",
+     "Cleans a value up to a valid Checkmk host tag id."),
+    ('cmk_cleanup_hostname',
+     "{{ cmk_cleanup_hostname(attribute) }}",
+     "Cleans a value up to a valid Checkmk host name."),
+    ('cmk_password',
+     "{{ cmk_password('name') }}",
+     "Checkmk password store reference of a global Secret Store entry."),
+    ('secret',
+     "{{ secret('key') }}",
+     "Checkmk password store reference of the Secret Store entry of this host "
+     "(or the global entry of that name)."),
+    ('secret_value',
+     "{{ secret_value('key') }}",
+     "Plaintext of that Secret Store entry, e.g. a certificate."),
+    ('get_ip_network',
+     "{{ get_ip_network(attribute) }}",
+     "Network of an address with mask: 10.1.2.3/255.255.255.0 gives "
+     "10.1.2.0/255.255.255.0."),
+    ('get_ip_interface',
+     "{{ get_ip_interface(attribute) }}",
+     "Address with prefix length: 10.1.2.3/255.255.255.0 gives 10.1.2.3/24."),
+    ('get_ip4_interface',
+     "{{ get_ip4_interface(attribute) }}",
+     "Same as get_ip_interface."),
+    ('datetime',
+     "{{ datetime.datetime.now().strftime('%Y-%m-%d') }}",
+     "The Python datetime module, e.g. for the current date."),
+)
+
+FILTERS = (
+    ('hash',
+     "{{ attribute | hash }}",
+     "Filter form of hash_value."),
+)
+
+BASICS = (
+    ('if / else',
+     "{% if attribute == 'value' %}yes{% else %}no{% endif %}",
+     "Different values depending on a condition."),
+    ('default',
+     "{{ attribute | default('unknown') }}",
+     "Fallback for a host without the attribute (otherwise the whole value "
+     "is empty)."),
+    ('lower / upper',
+     "{{ attribute | lower }}",
+     "Lower case (upper for upper case)."),
+    ('replace',
+     "{{ attribute | replace('old', 'new') }}",
+     "Replaces text in the value."),
+    ('split',
+     "{{ attribute.split('.')[0] }}",
+     "Part of a value, e.g. the short name of an FQDN."),
+    ('ACCOUNT',
+     "{{ACCOUNT:account_name:field}}",
+     "A field of an account, e.g. its address or password."),
+)
+
+# The variables the host based rules (Custom Attributes, Rewrite
+# Attributes) hand in besides the host's attributes.
+RULE_VARIABLES = (
+    ('HOSTNAME', "{{ HOSTNAME }}", "Name of the host."),
+    ('FIRST_MATCHING_TAG', "{{ FIRST_MATCHING_TAG }}",
+     "Attribute name the first condition of the rule matched."),
+    ('FIST_MATCHING_VALUE', "{{ FIST_MATCHING_VALUE }}",
+     "Attribute value the first condition of the rule matched."),
+    ('any attribute', "{{ attribute }}",
+     "Every attribute of the host is a variable of its own name."),
+)
+
+# The variables of a Checkmk Setup Rule rendered per host.
+SETUP_RULE_VARIABLES = (
+    ('HOSTNAME', "{{ HOSTNAME }}", "Name of the host."),
+    ('HOST_FOLDER', "{{ HOST_FOLDER }}",
+     "Folder the host lands in (Checkmk > Export Rules: Move to Folder or "
+     "Pool Folder), e.g. as rule folder. Per host only: a rule from a list "
+     "or a Static Rule has no host."),
+    ('loop', "{{ loop }}",
+     "Current entry when Loop over List is set (loop_idx: its index)."),
+    ('any attribute', "{{ attribute }}",
+     "Every attribute of the host is a variable of its own name."),
+)
+
+
+# The variables of a Checkmk DCD Rule.
+DCD_RULE_VARIABLES = (
+    ('HOSTNAME', "{{ HOSTNAME }}", "Name of the host."),
+    ('HOST_FOLDER', "{{ HOST_FOLDER }}",
+     "Folder the host lands in (Checkmk > Export Rules: Move to Folder or "
+     "Pool Folder), e.g. as folder of a creation rule. Per host only: a rule "
+     "from a list or a Static Rule has no host."),
+    ('HOST_POOL_SITE', "{{ HOST_POOL_SITE }}",
+     "Site the site pool assigned to the host in the host export (empty "
+     "without one), e.g. as site of the connection. Per host only."),
+    ('cmk_site', "{{ cmk_site }}",
+     "Checkmk site of the account the export runs for."),
+    ('account', "{{ account.address }}",
+     "A field or custom field of that account (passwords are left out)."),
+    ('any attribute', "{{ attribute }}",
+     "Every attribute of the host is a variable of its own name, from a "
+     "pasted list its columns."),
+)
+
+# How a password in the value of a Checkmk Setup Rule points to the Secret
+# Store. Checkmk 2.3+ holds a password as such a tuple; a stored_password
+# only names the ident the password export writes, the secret itself stays
+# out of the rule.
+CHECKMK_RULE_PASSWORDS = (
+    ('host secret',
+     "('cmk_postprocessed', 'stored_password', ('{{ secret(\"key\") }}', ''))",
+     "Replaces the password tuple in the value with a reference to the "
+     "Secret Store entry of this host (or the global entry of that name), "
+     "e.g. one an import stored. Export the passwords to Checkmk first."),
+    ('global secret',
+     "('cmk_postprocessed', 'stored_password', ('{{ cmk_password(\"name\") }}', ''))",
+     "The same with a global Secret Store entry. An "
+     "('cmk_postprocessed', 'explicit_password', ...) tuple of a rule copied "
+     "from Checkmk is replaced like this; Checkmk shows its secret only as "
+     "******."),
+)
+
+
+def _entries(group, items):
+    return [{'group': group, 'name': name, 'snippet': snippet, 'text': text}
+            for name, snippet, text in items]
+
+
+def lookup_tables():
+    """
+    Name and columns of every lookup table. Only the header line is
+    parsed: a table can be long and the panel needs nothing else.
+    """
+    from application.models.lookup_table import LookupTable
+    from application.modules.rule.list_source import parse_list_source
+    tables = []
+    for entry in LookupTable.objects.only('name', 'entries').order_by('name'):
+        header = (entry.entries or '').strip('\r\n').split('\n', 1)[0]
+        columns = parse_list_source(header).columns
+        tables.append({'name': entry.name, 'key': columns[0] if columns else '',
+                       'columns': columns[1:]})
+    return tables
+
+
+def jinja_help(fields, note='', variables=RULE_VARIABLES, extra=()):
+    """
+    What the help panel of a form needs: the fields (outcome field names)
+    it is shown for, a note on when they are rendered, the entries (with
+    the variables the form's rule hands in, then the ``(group, items)``
+    pairs of ``extra``) and the lookup tables.
+    """
+    entries = _entries('Variables', variables)
+    for group, items in extra:
+        entries += _entries(group, items)
+    return {
+        'fields': list(fields),
+        'note': note,
+        'entries': (entries
+                    + _entries('Syncer functions', FUNCTIONS)
+                    + _entries('Syncer filters', FILTERS)
+                    + _entries('Jinja basics', BASICS)),
+        'tables': lookup_tables(),
+        'docs': docu_links['jinja_functions'],
+    }