pg-maker 0.1.0__py3-none-any.whl

pg_maker/builders/components.py

@@ -0,0 +1,359 @@
+"""Explicit Make/PGXS components and fresh-cluster acceptance checks."""
+
+import re
+import shutil
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+
+from pg_maker.builders.autotools import regression_evidence
+from pg_maker.documents import relative_output
+from pg_maker.errors import CommandError, MakerError, PreconditionError, ValidationError
+from pg_maker.storage import read_json, write_json
+from pg_maker.yamlio import load_yaml
+
+
+def integrate(branch, components):
+    for component in components:
+        spec, identifier = component["spec"], component["metadata"]["id"]
+        if spec["type"] != "source-tree":
+            continue
+        relative = spec.get("source_directory")
+        if not relative or not relative_output(relative):
+            raise ValidationError("source-tree component requires a relative integration directory")
+        destination = branch.source / relative
+        if destination.exists() or destination.is_symlink():
+            raise PreconditionError("source-tree component must not overwrite PostgreSQL files")
+        if any(p.is_symlink() for p in destination.parents):
+            raise ValidationError("component destination traverses a symlink")
+        shutil.copytree(
+            branch.inputs.tree(spec["source"]["source_hash"]), destination, symlinks=True
+        )
+        branch.report.setdefault("integrated_components", []).append(identifier)
+
+
+@contextmanager
+def cluster(branch, name, *, database="postgres", preload=()):
+    if not re.fullmatch(r"[A-Za-z0-9_.-]+", name):
+        raise ValidationError("invalid cluster identifier")
+    root = branch.root / "clusters" / name
+    root.mkdir(parents=True)
+    container = branch.container_root + "/clusters/" + name
+    data = container + "/data"
+    sockets = container + "/socket"
+    (root / "socket").mkdir()
+    prefix = branch.prefix
+    env = {
+        "PGHOST": sockets,
+        "PGPORT": "55432",
+        "PGUSER": "builder",
+        "PGDATABASE": database,
+        "PATH": prefix + "/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+        "LC_ALL": "C.UTF-8",
+    }
+    branch.command(
+        [prefix + "/bin/initdb", "-D", data, "-A", "trust", "--no-locale", "-U", "builder"]
+    )
+    settings = "listen_addresses = ''\nport = 55432\nunix_socket_directories = '" + sockets + "'\n"
+    if preload:
+        if any(not re.fullmatch(r"[A-Za-z0-9_/-]+", item) for item in preload):
+            raise ValidationError("invalid preload library")
+        settings += "shared_preload_libraries = '" + ",".join(preload) + "'\n"
+    with (root / "data/postgresql.conf").open("a") as stream:
+        stream.write(settings)
+    # pg_ctl readiness is bounded; all PostgreSQL file logs stay under PGDATA.
+    branch.command(
+        [prefix + "/bin/pg_ctl", "-D", data, "-l", data + "/server.log", "-w", "-t", "60", "start"]
+    )
+    try:
+        if database != "postgres":
+            branch.command(
+                [prefix + "/bin/createdb", database], env={**env, "PGDATABASE": "postgres"}
+            )
+        yield env, root / "data/server.log"
+    finally:
+        # Cancellation stops the entire owned container; no unrelated host process is touched.
+        if not branch.context.token.requested():
+            original_error = sys.exc_info()[0] is not None
+            try:
+                branch.command(
+                    [
+                        prefix + "/bin/pg_ctl",
+                        "-D",
+                        data,
+                        "-w",
+                        "-t",
+                        "60",
+                        "-m",
+                        "immediate",
+                        "stop",
+                    ]
+                )
+            except MakerError:
+                # A timed-out command may already have stopped the container. Preserve its
+                # failure; the outer engine still releases/verifies the entire owned resource.
+                if not original_error:
+                    raise
+
+
+def build_components(branch, components, *, run_tests=True):
+    reports = []
+    for component in components:
+        spec, identifier = component["spec"], component["metadata"]["id"]
+        kind = spec["type"]
+        if kind in {"contrib", "source-tree"}:
+            relative = spec.get("source_directory", "contrib/" + identifier)
+            if not (branch.source / relative / "Makefile").is_file():
+                raise PreconditionError("component source directory is missing")
+            cwd = branch.container_root + "/obj/" + relative
+            (branch.obj / relative).mkdir(parents=True, exist_ok=True)
+            component_source = branch.source / relative
+            container_source = branch.container_root + "/source/" + relative
+        else:
+            path = branch.root / "components" / identifier
+            if not path.exists():
+                shutil.copytree(
+                    branch.inputs.tree(spec["source"]["source_hash"]), path, symlinks=True
+                )
+            cwd = branch.container_root + "/components/" + identifier
+            if spec.get("source_directory"):
+                cwd += "/" + spec["source_directory"]
+                path /= spec["source_directory"]
+            component_source, container_source = path, cwd
+            if spec.get("configure_args") or (
+                kind in {"utility", "library"} and (path / "configure").is_file()
+            ):
+                branch.command(
+                    ["./configure", "--prefix=" + branch.prefix, *spec.get("configure_args", [])],
+                    cwd=cwd,
+                )
+        make = ["make", "--output-sync=target", "-j", str(branch.variant["build"]["jobs"])]
+        if kind in {"contrib", "source-tree"}:
+            make += ["-f", container_source + "/Makefile"]
+        if kind == "pgxs":
+            make += ["USE_PGXS=1", "PG_CONFIG=" + branch.prefix + "/bin/pg_config"]
+        elif kind in {"utility", "library"}:
+            make += ["prefix=" + branch.prefix, "PG_CONFIG=" + branch.prefix + "/bin/pg_config"]
+        for targets in (spec["make_targets"], spec["install_targets"]):
+            if targets:
+                branch.command([*make, *targets], cwd=cwd)
+        if not run_tests:
+            reports.append(
+                {"id": identifier, "built": True, "native_targets": [], "tests": "skipped"}
+            )
+            continue
+        expected = None
+        if kind in {"pgxs", "contrib", "source-tree"} and spec["test_targets"]:
+            result = branch.command(
+                [
+                    *make,
+                    *(["-f", "Makefile"] if kind == "pgxs" else []),
+                    "-f",
+                    "/work/source/inventory.mk",
+                    "maker-inventory",
+                ],
+                cwd=cwd,
+            )
+            values = dict(
+                re.findall(r"^PG_MAKER_([A-Z_]+)=(.*)$", Path(result.stdout_log).read_text(), re.M)
+            )
+            expected = {
+                "regression": values["REGRESS"].split(),
+                "isolation": values["ISOLATION"].split(),
+                "tap": [
+                    p.relative_to(component_source).as_posix()
+                    for p in sorted((component_source / "t").glob("*.pl"))
+                ]
+                if values["TAP"] == "1"
+                else [],
+            }
+            if not any(expected.values()):
+                raise PreconditionError("component has no declared native test inventory")
+            write_json(branch.output / "components" / identifier / "expected.json", expected)
+        with cluster(branch, "component-" + identifier, preload=spec.get("preload", [])) as (
+            env,
+            _,
+        ):
+            if spec.get("shared_library"):
+                library = spec["shared_library"].replace("'", "''")
+                branch.command(
+                    [
+                        branch.prefix + "/bin/psql",
+                        "-X",
+                        "-v",
+                        "ON_ERROR_STOP=1",
+                        "-c",
+                        "LOAD '" + library + "'",
+                    ],
+                    env=env,
+                )
+            if spec.get("sql_extension"):
+                extension = spec["sql_extension"].replace('"', '""')
+                branch.command(
+                    [
+                        branch.prefix + "/bin/psql",
+                        "-X",
+                        "-v",
+                        "ON_ERROR_STOP=1",
+                        "-c",
+                        'CREATE EXTENSION "' + extension + '" CASCADE',
+                    ],
+                    env=env,
+                )
+            for target in spec["test_targets"]:
+                tap = branch.output / "components" / identifier / target / "tap"
+                tap.mkdir(parents=True)
+                test_env = {
+                    **env,
+                    "PERL5LIB": "/work/source",
+                    "PG_MAKER_SOURCE": container_source,
+                    "PG_MAKER_TAP": "/work/output/"
+                    + branch.id
+                    + "/components/"
+                    + identifier
+                    + "/"
+                    + target
+                    + "/tap",
+                }
+                result = branch.command(
+                    [*make, target, "PROVE_FLAGS=--formatter=PgMaker::Formatter"],
+                    cwd=cwd,
+                    env=test_env,
+                )
+                if expected is not None:
+                    verify_component_tests(expected, Path(result.stdout_log).read_text(), tap)
+        reports.append(
+            {
+                "id": identifier,
+                "built": True,
+                "native_targets": spec["test_targets"],
+                "tests": "passed",
+                "passed": True,
+            }
+        )
+    branch.report.setdefault("components", []).extend(reports)
+
+
+def verify_component_tests(expected, stdout, tap):
+    evidence = regression_evidence(stdout)
+    for kind in ("regression", "isolation"):
+        values = [v for (k, _), v in evidence.items() if k == kind]
+        actual = [name for v in values for name in v["tests"]]
+        if sorted(expected[kind]) != sorted(actual) or any(
+            v["failed"] or v["plans"] != [len(v["tests"])] for v in values
+        ):
+            raise CommandError("component regression inventory mismatch", details={"kind": kind})
+    values = [read_json(p) for p in tap.glob("*.json")]
+    if sorted(v["path"] for v in values) != sorted(expected["tap"]) or any(
+        v["tests"] <= len(v["skipped"])
+        or v["tests"] != v["planned"]
+        or v["failed"]
+        or v["errors"]
+        or v["skip_all"]
+        or v["exit"]
+        or v["wait"]
+        for v in values
+    ):
+        raise CommandError("component TAP inventory mismatch")
+
+
+def acceptance(branch, changes):
+    reports, seen = [], set()
+    for change in changes:
+        for test in change["spec"]["acceptance"]:
+            if branch.branch not in test["expectations"]:
+                continue
+            if test["id"] in seen:
+                continue
+            seen.add(test["id"])
+            blob = branch.inputs.blob(test["script"]["sha256"])
+            script = branch.helper / ("acceptance-" + test["id"])
+            shutil.copyfile(blob, script)
+            script_path = "/work/source/" + script.name
+            with cluster(
+                branch, "acceptance-" + test["id"], database=test["setup"]["database"]
+            ) as (env, log):
+                if test["kind"] == "command":
+                    document = load_yaml(blob)
+                    if (
+                        not isinstance(document, dict)
+                        or set(document) != {"argv"}
+                        or not isinstance(document["argv"], list)
+                        or not document["argv"]
+                        or not all(isinstance(v, str) and "\x00" not in v for v in document["argv"])
+                    ):
+                        raise ValidationError(
+                            "command acceptance script must be YAML {argv: [program, arguments...]}"
+                        )
+                    argv = [arg.replace("{prefix}", branch.prefix) for arg in document["argv"]]
+                else:
+                    argv = [
+                        branch.prefix + "/bin/psql",
+                        "-X",
+                        "-A",
+                        "-t",
+                        "-v",
+                        "ON_ERROR_STOP=1",
+                        "-v",
+                        "VERBOSITY=verbose",
+                        "-f",
+                        script_path,
+                    ]
+                sqlstate_assertion = test["kind"] != "command" and any(
+                    a["source"] == "sqlstate" for a in test["assertions"]
+                )
+                result = branch.command(
+                    argv,
+                    env=env,
+                    timeout=test["timeout_seconds"],
+                    accepted_exit_codes=(0, 3) if sqlstate_assertion else (0,),
+                )
+                stdout, stderr = (
+                    Path(result.stdout_log).read_text(),
+                    Path(result.stderr_log).read_text(),
+                )
+                outputs = {
+                    "stdout": stdout,
+                    "stderr": stderr,
+                    "sql": stdout,
+                    "server-log": log.read_text(),
+                    "sqlstate": sqlstate(result.returncode, stderr),
+                }
+                failures = []
+                for assertion in test["assertions"]:
+                    output = outputs[assertion["source"]]
+                    if "contains" in assertion:
+                        passed = assertion["contains"] in output
+                    else:
+                        expected = assertion["equals"]
+                        if isinstance(expected, bool):
+                            expected = "t" if expected else "f"
+                        passed = output.strip() == str(expected)
+                    if not passed:
+                        failures.append(assertion["id"])
+                expectation = test["expectations"][branch.branch]
+                passed = (
+                    not failures
+                    if expectation["outcome"] == "pass"
+                    else failures == [expectation.get("assertion_id")]
+                )
+                report = {
+                    "id": test["id"],
+                    "expectation": expectation,
+                    "failed_assertions": failures,
+                    "passed": passed,
+                }
+                reports.append(report)
+                write_json(branch.output / "acceptance.json", reports)
+                if not passed:
+                    raise CommandError("acceptance expectation mismatch", details=report)
+    branch.report["acceptance"] = reports
+
+
+def sqlstate(returncode, stderr):
+    if returncode == 0:
+        return "00000"
+    match = re.search(r"ERROR:\s+([0-9A-Z]{5}):", stderr)
+    if returncode != 3 or not match:
+        raise CommandError("SQL failure has no valid SQLSTATE evidence")
+    return match[1]

pg_maker/bundled/PROJECT.md

@@ -0,0 +1,57 @@
+# pg_maker project
+
+The initial target is PostgreSQL 18.1 / Debian 13 / x86_64 with feature my-feature.
+Edit vendor Test, paths, build modes and resources in config/. Defaults enable the
+common socket patch. Configuration and file logs live in PGDATA; headers, PGXS and
+ecpg are not shipped.
+
+The feature selects pg_stat_statements, pg_stat_kcache 2.3.2 and pg_wait_sampling 1.1.11
+(SQL version 1.1). Editable component recipes pin external sources by commit SHA;
+pg_stat_statements uses PostgreSQL contrib. Set components: [] for no extra extensions.
+Do not remove pg_stat_statements while pg_stat_kcache requires it.
+
+```bash
+pg-maker validate build.yaml
+pg-maker config show build.yaml --resolved
+pg-maker package names build.yaml
+pg-maker env prepare build.yaml --out environments.json
+pg-maker resolve build.yaml --environments environments.json --out lock.json --input-store inputs
+pg-maker plan lock.json --out plan.json --goal build
+pg-maker build lock.json --plan plan.json --detach
+```
+
+Environment preparation installs dependencies into separate Docker images. Resolve
+freezes sources/changes; offline resolve reuses stored inputs without networking.
+Plan verifies hashes and describes work. Database/component tests, acceptance and
+baseline are disabled by default for fast development; results remain unverified.
+Use build --run-db-tests for full testing, or create a matching saved plan first.
+The default is config/defaults.yaml spec.validation.run_db_tests. Utility tests are
+unaffected. Build --goal package produces three DEBs and manifests; verified requests
+long source/installed suites. The systemctl emulator does not qualify native systemd.
+The build user must match the non-root host workspace owner.
+
+Enable documentation in config/documentation.yaml before environment preparation and
+resolve. Docs build produces selected HTML/man/PDF; prepare-translations exports bundles
+for an external translator. Register completed bundles under spec.translations and
+resolve again. Rendering is offline. With docs, a fourth docs package is added and
+server only Suggests it. Selected external-component guides join navigation/search/PDF;
+Russian builds require corresponding translation bundles.
+
+Package installation does not activate extensions in existing clusters. Configure
+shared_preload_libraries in order pg_stat_statements,pg_stat_kcache,pg_wait_sampling,
+restart and create extensions in the needed databases, following their manuals.
+
+Long operations support --detach and --request-id. Status/events/wait/cancel accept
+operations/OPERATION_ID. Do not edit active input/state/journal files. Logs live under
+attempts/N/logs, progress under attempts/N/progress.log and worker output in worker.log.
+Repeated init retains identical files and reports conflicts before any remaining writes.
+
+Resume build with the original plan hash; preparation requires its input hash. Changed
+code belongs in a new resolve/build or a dev session. Dev open --from-run creates an
+owned Git workspace for intermediate commits, patch replay and selected component builds.
+
+Export preserves packages/reports/evidence; --include-inputs adds source trees/blobs,
+not Docker images. Shared cache defaults to operations/.cache. No-cache disables shared
+reuse but preserves private checkpoints. Preview cache prune before applying it.
+Monitor df/du/docker system df. Export needed results before explicit scratch cleanup;
+cleaned runs cannot resume. Global Docker prune is never automatic.

pg_maker/bundled/build.yaml

@@ -0,0 +1,29 @@
+api_version: pg_maker/v1
+kind: PostgreSQLBuild
+metadata:
+  name: test-18-1
+spec:
+  config_dir: ./config
+  source:
+    kind: git
+    repository: https://github.com/postgres/postgres.git
+    ref: REL_18_1
+  postgresql:
+    version: '18.1'
+  product:
+    release: 0
+    package_revision: 1
+  features:
+  - my-feature
+  targets:
+  - id: debian13-x64
+    platform: debian-13
+    architecture: x86_64
+    cpu_target: x86-64
+    artifacts:
+    - deb
+  documentation:
+    profile: ./config/documentation.yaml
+  validation:
+    profile: full
+    baseline: auto

pg_maker/bundled/components/pg-stat-kcache.yaml

@@ -0,0 +1,30 @@
+api_version: pg_maker/component/v1
+kind: ComponentRecipe
+metadata:
+  id: pg-stat-kcache
+  revision: 1
+spec:
+  type: pgxs
+  source:
+    kind: git
+    repository: https://github.com/powa-team/pg_stat_kcache.git
+    # Release REL2_3_2 (2.3.2).
+    ref: 1e39814453ab747a3817cfdf4f6f6bd5c0fb9507
+  compatible_postgresql: '>=14,<20'
+  requires: [pg-stat-statements]
+  conflicts: []
+  build_dependencies: []
+  runtime_dependencies: []
+  make_targets: [all]
+  test_targets: [installcheck]
+  install_targets: [install]
+  required_tests: []
+  sql_extension: pg_stat_kcache
+  shared_library: pg_stat_kcache
+  preload: [pg_stat_statements, pg_stat_kcache]
+  runtime_files:
+  - lib/postgresql/pg_stat_kcache.so
+  - share/postgresql/extension/pg_stat_kcache.control
+  - share/postgresql/extension/pg_stat_kcache--*.sql
+  documentation:
+    rules: [../documentation/pg-stat-kcache.yaml]

pg_maker/bundled/components/pg-stat-statements.yaml

@@ -0,0 +1,24 @@
+api_version: pg_maker/component/v1
+kind: ComponentRecipe
+metadata:
+  id: pg-stat-statements
+  revision: 1
+spec:
+  type: contrib
+  source_directory: contrib/pg_stat_statements
+  compatible_postgresql: '>=14,<20'
+  requires: []
+  conflicts: []
+  build_dependencies: []
+  runtime_dependencies: []
+  make_targets: [all]
+  test_targets: [check]
+  install_targets: [install]
+  required_tests: []
+  sql_extension: pg_stat_statements
+  shared_library: pg_stat_statements
+  preload: [pg_stat_statements]
+  runtime_files:
+  - lib/postgresql/pg_stat_statements.so
+  - share/postgresql/extension/pg_stat_statements.control
+  - share/postgresql/extension/pg_stat_statements--*.sql

pg_maker/bundled/components/pg-wait-sampling.yaml

@@ -0,0 +1,30 @@
+api_version: pg_maker/component/v1
+kind: ComponentRecipe
+metadata:
+  id: pg-wait-sampling
+  revision: 1
+spec:
+  type: pgxs
+  source:
+    kind: git
+    repository: https://github.com/postgrespro/pg_wait_sampling.git
+    # Release v1.1.11; the SQL extension version is 1.1.
+    ref: 7e3a5fc937393596c0a51c5881dd63d7208408d1
+  compatible_postgresql: '>=14,<20'
+  requires: []
+  conflicts: []
+  build_dependencies: []
+  runtime_dependencies: []
+  make_targets: [all]
+  test_targets: [installcheck]
+  install_targets: [install]
+  required_tests: []
+  sql_extension: pg_wait_sampling
+  shared_library: pg_wait_sampling
+  preload: [pg_wait_sampling]
+  runtime_files:
+  - lib/postgresql/pg_wait_sampling.so
+  - share/postgresql/extension/pg_wait_sampling.control
+  - share/postgresql/extension/pg_wait_sampling--*.sql
+  documentation:
+    rules: [../documentation/pg-wait-sampling.yaml]

pg_maker/bundled/config/defaults.yaml

@@ -0,0 +1,80 @@
+api_version: pg_maker/defaults/v1
+kind: MakerDefaults
+metadata:
+  id: default
+spec:
+  paths:
+    runtime_prefix: /opt/{vendor.slug}/{feature.id}/{postgresql.major}{build.mode_suffix}
+    data_directory: /var/lib/{vendor.slug}/{feature.id}/{postgresql.major}{build.mode_suffix}/data
+    documentation_directory: /opt/{vendor.slug}/{feature.id}/{postgresql.major}{build.mode_suffix}/share/doc/{product.version}
+  service:
+    user: postgres
+    group: postgres
+    home: /var/lib/postgresql
+    port: 5432
+  naming:
+    package: '{vendor.slug}-{feature.id}-{role}{role.abi_suffix}-{postgresql.major}{build.mode_suffix}'
+    service: '{vendor.slug}-{feature.id}-server-{postgresql.major}{build.mode_suffix}'
+    deb_version: '{product.version}-{package_revision}.{platform.package_suffix}'
+    rpm_version: '{product.version}'
+    rpm_release: '{package_revision}.{platform.package_suffix}'
+  common_changes:
+  - builtin: unix-socket-directory
+  build:
+    backend: autotools
+    mode: release
+    options:
+      llvm: true
+      openssl: true
+      icu: true
+    nls_languages:
+    - en
+    - ru
+    message_locale: C.UTF-8
+    jobs: 4
+    flags:
+      cppflags:
+      - -Wno-missing-braces
+      - -Wformat
+      - -Werror=format-security
+      - -fstack-protector-strong
+      ldflags:
+      - -Wl,-z,relro
+      - -Wl,-z,now
+      lto: false
+  build_modes:
+    release:
+      naming_suffix: ''
+      configure_options: []
+      cflags:
+      - -pipe
+      - -O2
+      cppflags:
+      - -D_FORTIFY_SOURCE=2
+    debug:
+      naming_suffix: -debug
+      configure_options:
+      - --enable-debug
+      - --enable-cassert
+      cflags:
+      - -pipe
+      - -O0
+      - -ggdb3
+      cppflags: []
+  validation:
+    profile: full
+    # Temporary development default; enable explicitly for source verification.
+    run_db_tests: false
+    baseline: auto
+    database_locales:
+    - en_US.UTF-8
+    - ru_RU.UTF-8
+  execution:
+    runner: docker
+    max_parallel_targets: 1
+    build_user: 1000:1000
+    limits:
+      memory_bytes: 8589934592
+      min_free_disk_bytes: 21474836480
+      stage_timeout_seconds: 7200
+    keep_failed_environment: false

pg_maker/bundled/config/documentation.yaml

@@ -0,0 +1,24 @@
+api_version: pg_maker/documentation/v1
+kind: DocumentationProfile
+metadata:
+  id: default
+spec:
+  enabled: false
+  source: postgres-tree
+  source_language: en
+  languages:
+  - en
+  language_aliases: {en: en, en-US: en, ru: ru, ru-RU: ru}
+  formats:
+  - html
+  - man
+  html_backend: sphinx
+  branding: vendor-metadata
+  rules: []
+  translations: []
+  missing_translation: error
+  require_reviewed: false
+  glossary: {revision: '1', terms: {}}
+  install:
+    html: docs-package
+    man: owning-package

pg_maker/bundled/config/features/my-feature.yaml

@@ -0,0 +1,17 @@
+api_version: pg_maker/feature/v1
+kind: FeatureProfile
+metadata:
+  id: my-feature
+  revision: 2
+spec:
+  description: Test
+  numeric_id: null
+  changes: []
+  components:
+  - ../../components/pg-stat-statements.yaml
+  - ../../components/pg-stat-kcache.yaml
+  - ../../components/pg-wait-sampling.yaml
+  build_options: {}
+  documentation:
+    rules: []
+  required_tests: []

pg_maker/bundled/config/vendor.yaml

@@ -0,0 +1,13 @@
+api_version: pg_maker/vendor/v1
+kind: VendorConfig
+metadata:
+  id: test
+spec:
+  company: Test
+  brand: Test
+  product_name: Test
+  description: Test
+  slug: test
+  homepage: null
+  documentation_url: null
+  contact_email: test@example.invalid