conan-cci-build 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- conan_cci_build-0.1.0/LICENSE +21 -0
- conan_cci_build-0.1.0/PKG-INFO +10 -0
- conan_cci_build-0.1.0/README.md +8 -0
- conan_cci_build-0.1.0/extensions/commands/cmd_cci_build.py +19 -0
- conan_cci_build-0.1.0/setup.cfg +4 -0
- conan_cci_build-0.1.0/setup.py +30 -0
- conan_cci_build-0.1.0/src/cci_build/__init__.py +0 -0
- conan_cci_build-0.1.0/src/cci_build/conan_centre_index.py +108 -0
- conan_cci_build-0.1.0/src/cci_build/conan_client.py +59 -0
- conan_cci_build-0.1.0/src/cci_build/installer.py +54 -0
- conan_cci_build-0.1.0/src/cci_build/main.py +39 -0
- conan_cci_build-0.1.0/src/cci_build/model/__init__.py +0 -0
- conan_cci_build-0.1.0/src/cci_build/model/context.py +29 -0
- conan_cci_build-0.1.0/src/cci_build/model/settings/__init__.py +0 -0
- conan_cci_build-0.1.0/src/cci_build/model/settings/types.py +28 -0
- conan_cci_build-0.1.0/src/cci_build/package_parser.py +91 -0
- conan_cci_build-0.1.0/src/cci_build/profile_matcher.py +33 -0
- conan_cci_build-0.1.0/src/cci_build/workflow.py +201 -0
- conan_cci_build-0.1.0/src/conan_cci_build.egg-info/PKG-INFO +10 -0
- conan_cci_build-0.1.0/src/conan_cci_build.egg-info/SOURCES.txt +22 -0
- conan_cci_build-0.1.0/src/conan_cci_build.egg-info/dependency_links.txt +1 -0
- conan_cci_build-0.1.0/src/conan_cci_build.egg-info/entry_points.txt +2 -0
- conan_cci_build-0.1.0/src/conan_cci_build.egg-info/requires.txt +2 -0
- conan_cci_build-0.1.0/src/conan_cci_build.egg-info/top_level.txt +2 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 conan-py
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"""
|
|
2
|
+
A conan extension command entry point that thunks through to the cc-build module.
|
|
3
|
+
"""
|
|
4
|
+
import argparse
|
|
5
|
+
|
|
6
|
+
from conan.api.conan_api import ConanAPI
|
|
7
|
+
from conan.cli.command import conan_command
|
|
8
|
+
|
|
9
|
+
@conan_command(group="Custom commands")
|
|
10
|
+
def cci_build(conan_api: ConanAPI, parser: argparse.ArgumentParser, *args):
|
|
11
|
+
"""
|
|
12
|
+
Build Conan Center Index packages using a custom utility pipeline.
|
|
13
|
+
|
|
14
|
+
Run the following to install/use this command:
|
|
15
|
+
> conan config install .
|
|
16
|
+
> conan cci-build --help
|
|
17
|
+
"""
|
|
18
|
+
from cci_build.main import cci_build_command
|
|
19
|
+
return cci_build_command(conan_api, parser, *args)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Trivial setup.py to package for publishing the module to pypi.
|
|
3
|
+
"""
|
|
4
|
+
from setuptools import setup, find_packages
|
|
5
|
+
|
|
6
|
+
setup(
|
|
7
|
+
name="conan-cci-build",
|
|
8
|
+
version="0.1.0",
|
|
9
|
+
package_dir={
|
|
10
|
+
"": "src",
|
|
11
|
+
"cci_build_extensions": "extensions",
|
|
12
|
+
},
|
|
13
|
+
packages=find_packages(where="src") + [
|
|
14
|
+
"cci_build_extensions",
|
|
15
|
+
"cci_build_extensions.commands",
|
|
16
|
+
],
|
|
17
|
+
python_requires=">=3.13",
|
|
18
|
+
install_requires=[
|
|
19
|
+
"conan>=2.25,<3.0",
|
|
20
|
+
"packaging>=26.2",
|
|
21
|
+
],
|
|
22
|
+
package_data={
|
|
23
|
+
"cci_build_extensions.commands": ["cmd_cci_build.py"],
|
|
24
|
+
},
|
|
25
|
+
entry_points={
|
|
26
|
+
"console_scripts": [
|
|
27
|
+
"conan-cci-build-install = cci_build.installer:main",
|
|
28
|
+
],
|
|
29
|
+
},
|
|
30
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CCI Utilities
|
|
3
|
+
"""
|
|
4
|
+
import yaml
|
|
5
|
+
from packaging.version import parse as parse_version
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Optional, Tuple
|
|
8
|
+
|
|
9
|
+
from cci_build.error.exception import RecipeNotFoundError
|
|
10
|
+
from cci_build.model.cci.conandata.types import ConanDataLayout
|
|
11
|
+
from cci_build.model.cci.config.types import Config, VersionInfo
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def make_cci_recipie_package_config_filename(recipe_dir: Path) -> Path:
|
|
15
|
+
"""
|
|
16
|
+
Make the path to the 'config.yml' for a package in a CCI
|
|
17
|
+
"""
|
|
18
|
+
return recipe_dir / "config.yml"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def make_cci_recipie_package_conanfile_filename(conanfile_folder_name, recipe_dir: Path) -> Path:
|
|
22
|
+
"""
|
|
23
|
+
Make the path to the 'conanfile.py' for a package in a CCI
|
|
24
|
+
"""
|
|
25
|
+
return recipe_dir / conanfile_folder_name / "conanfile.py"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def make_cci_recipie_package_conandata_filename(conanfile_folder_name, recipe_dir: Path) -> Path:
|
|
29
|
+
"""
|
|
30
|
+
Make the path to the 'conandata.yml' for a package in a CCI
|
|
31
|
+
"""
|
|
32
|
+
return recipe_dir / conanfile_folder_name / "conandata.yml"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def make_cci_recipe_package_path(cci_root: Path, package_name: str) -> Path:
|
|
36
|
+
return Path(cci_root) / "recipes" / package_name
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def read_package_config_yaml(config_path: Path) -> Config:
|
|
40
|
+
with open(config_path, "r", encoding="utf-8") as f:
|
|
41
|
+
config_data = yaml.safe_load(f)
|
|
42
|
+
return config_data
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def read_conandata_yaml(config_path: Path) -> ConanDataLayout:
|
|
46
|
+
with open(config_path, "r", encoding="utf-8") as f:
|
|
47
|
+
config_data = yaml.safe_load(f)
|
|
48
|
+
if config_data.get("sources"):
|
|
49
|
+
return config_data
|
|
50
|
+
return None
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def find_cci_conanfile(cci_root: Path, package_name: str, version: Optional[str]) -> Tuple[str, Path]:
|
|
54
|
+
"""
|
|
55
|
+
Finds the exact conanfile.py path for a given package version
|
|
56
|
+
based on standard Conan Center Index (CCI) layout specifications.
|
|
57
|
+
|
|
58
|
+
:argument cci_root: The root directory of the local copy of a CCI.
|
|
59
|
+
:argument package_name: The name of the package to find.
|
|
60
|
+
:argument version: An optional version of the package to find, otherwise None will be translated to latest.
|
|
61
|
+
|
|
62
|
+
Return the conanfile.py path and the exact version to be built.
|
|
63
|
+
"""
|
|
64
|
+
recipe_dir = make_cci_recipe_package_path(cci_root, package_name)
|
|
65
|
+
config_path = make_cci_recipie_package_config_filename(recipe_dir)
|
|
66
|
+
|
|
67
|
+
if config_path.exists():
|
|
68
|
+
config_data = read_package_config_yaml(config_path)
|
|
69
|
+
|
|
70
|
+
# Standard CCI layout groups everything under the 'versions' node
|
|
71
|
+
v, info = find_config_version(config_data.get("versions", {}), version)
|
|
72
|
+
if v and info and "folder" in info:
|
|
73
|
+
|
|
74
|
+
# Pull the targeted folder (frequently 'all', or specific folders like '1.x.x')
|
|
75
|
+
conanfile_folder_name = info.get("folder")
|
|
76
|
+
conanfile_path = make_cci_recipie_package_conanfile_filename(conanfile_folder_name, recipe_dir)
|
|
77
|
+
if conanfile_path.exists():
|
|
78
|
+
|
|
79
|
+
conandata_path = make_cci_recipie_package_conandata_filename(conanfile_folder_name, recipe_dir)
|
|
80
|
+
if conandata_path.exists():
|
|
81
|
+
|
|
82
|
+
conandata = read_conandata_yaml(conandata_path)
|
|
83
|
+
if conandata:
|
|
84
|
+
return v, conanfile_path
|
|
85
|
+
raise FileNotFoundError(f"Package '{package_name}/{v}' not found in conandata")
|
|
86
|
+
raise FileNotFoundError(f"Expected conandata.yml missing at: {conandata_path}")
|
|
87
|
+
raise FileNotFoundError(f"Expected conanfile.py missing at: {conanfile_path}")
|
|
88
|
+
raise RecipeNotFoundError(f"Recipe '{package_name}/{version if not None else 'latest'}' not found config.yml.")
|
|
89
|
+
raise RecipeNotFoundError(f"Package '{package_name}' or its config.yml not found in CCI path.")
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def find_config_version(
|
|
93
|
+
versions: dict[str, VersionInfo],
|
|
94
|
+
version: Optional[str]) -> Tuple[Optional[str], Optional[VersionInfo]]:
|
|
95
|
+
"""
|
|
96
|
+
Given a `config.yml` set of versions available, select the version information
|
|
97
|
+
for that version. If no version is specified, then select the latest/newest version.
|
|
98
|
+
|
|
99
|
+
Use standard PEP 440 version parsing rules to compare version numbers.
|
|
100
|
+
|
|
101
|
+
return A valid ConanCentreVersionInfo or None if there is no matching version
|
|
102
|
+
"""
|
|
103
|
+
if version:
|
|
104
|
+
specific_version = versions.get(version)
|
|
105
|
+
return (version, specific_version) if specific_version else (None, None,)
|
|
106
|
+
else:
|
|
107
|
+
max_version = max(versions.keys(), key=parse_version)
|
|
108
|
+
return (max_version, versions.get(max_version),) if max_version else (None, None,)
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Wrap the ConanAPI
|
|
3
|
+
"""
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Iterable
|
|
6
|
+
|
|
7
|
+
from conan.api.conan_api import ConanAPI
|
|
8
|
+
from conan.api.model import Remote, ListPattern
|
|
9
|
+
from conan.internal.errors import NotFoundException
|
|
10
|
+
|
|
11
|
+
from cci_build.error.exception import ConanAdapterError
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ConanClient:
|
|
15
|
+
"""
|
|
16
|
+
Thin wrapper over Conan 2 public API.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(self, api: ConanAPI) -> None:
|
|
20
|
+
self.api = api
|
|
21
|
+
|
|
22
|
+
def binary_exists(self, ref: ListPattern, remote: Remote) -> bool:
|
|
23
|
+
"""
|
|
24
|
+
Uses Conan list against remote.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
try:
|
|
28
|
+
result = self.api.list.select(ref, remote=remote)
|
|
29
|
+
return bool(result)
|
|
30
|
+
except NotFoundException:
|
|
31
|
+
return False
|
|
32
|
+
|
|
33
|
+
def create(self, recipe: Path, host: str, build: str) -> list[str]:
|
|
34
|
+
"""
|
|
35
|
+
Executes in-process create using Conan API.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
try:
|
|
39
|
+
graph = self.api.graph.load_graph(
|
|
40
|
+
path=str(recipe),
|
|
41
|
+
name=None,
|
|
42
|
+
version=None,
|
|
43
|
+
args=[f"-pr:h={host}", f"-pr:b={build}", "--build=missing"],
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
self.api.graph.analyze_binaries(graph)
|
|
47
|
+
self.api.graph.build(graph)
|
|
48
|
+
|
|
49
|
+
return [str(node.ref) for node in graph.nodes.values() if node.binary_package]
|
|
50
|
+
|
|
51
|
+
except Exception as e:
|
|
52
|
+
raise ConanAdapterError(f"Create failed: {e}")
|
|
53
|
+
|
|
54
|
+
def upload(self, refs: Iterable[str], remote: str) -> None:
|
|
55
|
+
try:
|
|
56
|
+
for ref in refs:
|
|
57
|
+
self.api.upload.upload_full(ref, remote=remote, only_recipe=False, only_package=True)
|
|
58
|
+
except Exception as e:
|
|
59
|
+
raise ConanAdapterError(f"Upload failed: {e}")
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""
|
|
2
|
+
A script to install the cci-build command into Conan.
|
|
3
|
+
"""
|
|
4
|
+
import logging
|
|
5
|
+
import os
|
|
6
|
+
import shutil
|
|
7
|
+
from importlib.resources import files, as_file
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
log = logging.getLogger(__name__)
|
|
11
|
+
|
|
12
|
+
def get_conan_home() -> Path:
|
|
13
|
+
"""
|
|
14
|
+
Resolve Conan 2 home directory.
|
|
15
|
+
|
|
16
|
+
This is the root directory where Conan stores:
|
|
17
|
+
- cache
|
|
18
|
+
- extensions
|
|
19
|
+
- configuration
|
|
20
|
+
- remotes
|
|
21
|
+
|
|
22
|
+
Resolution order:
|
|
23
|
+
1. CONAN_HOME environment variable (if set)
|
|
24
|
+
2. Default platform-specific location (~/.conan2)
|
|
25
|
+
|
|
26
|
+
Returns:
|
|
27
|
+
Path to Conan home directory.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
# 1. Explicit override (highest priority)
|
|
31
|
+
env_home = os.environ.get("CONAN_HOME")
|
|
32
|
+
if env_home:
|
|
33
|
+
return Path(env_home).expanduser().resolve()
|
|
34
|
+
|
|
35
|
+
# 2. Default Conan 2 home directory
|
|
36
|
+
# Conan 2 standard default is ~/.conan2 unless overridden
|
|
37
|
+
return Path.home() / ".conan2"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def main():
|
|
42
|
+
"""
|
|
43
|
+
Entry
|
|
44
|
+
"""
|
|
45
|
+
conan_home = get_conan_home()
|
|
46
|
+
target_dir = conan_home / "extensions" / "commands"
|
|
47
|
+
target_dir.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
destination = target_dir / "cmd_cci_build.py"
|
|
49
|
+
|
|
50
|
+
resource = files("cci_build_extensions.commands").joinpath("cmd_cci_build.py")
|
|
51
|
+
with as_file(resource) as src:
|
|
52
|
+
shutil.copyfile(src, destination)
|
|
53
|
+
|
|
54
|
+
log.info(f"Installed Conan extension to: {destination}")
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Entrypoint for the cci-build (called from the conan extension command)
|
|
3
|
+
"""
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
from conan.api.conan_api import ConanAPI
|
|
7
|
+
from conan.api.output import ConanOutput
|
|
8
|
+
|
|
9
|
+
from cci_build.model.context import Context
|
|
10
|
+
from cci_build.workflow import Workflow
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def cci_build_command(_conan_api: ConanAPI, parser, *args):
|
|
14
|
+
"""
|
|
15
|
+
Build Conan Center Index packages using a custom utility pipeline.
|
|
16
|
+
"""
|
|
17
|
+
# Conan provides its own pre-configured argparse instance via 'parser'
|
|
18
|
+
parser.add_argument("--cci-root", required=True, help="Path to the CCI root directory.")
|
|
19
|
+
parser.add_argument("--remote", required=True, help="Conan remote target.")
|
|
20
|
+
parser.add_argument("--host-profile", required=True, help="Host profile name or path.")
|
|
21
|
+
parser.add_argument("--build-profile", required=True, help="Build profile name or path.")
|
|
22
|
+
parser.add_argument("file", help="The packages file to process.")
|
|
23
|
+
|
|
24
|
+
# Parse the arguments forwarded from Conan's command line interface
|
|
25
|
+
parsed_args = parser.parse_args(*args)
|
|
26
|
+
|
|
27
|
+
# Initialize your existing Context object
|
|
28
|
+
config = Context(
|
|
29
|
+
cci_root=Path(parsed_args.cci_root),
|
|
30
|
+
remote=parsed_args.remote,
|
|
31
|
+
host_profile=parsed_args.host_profile,
|
|
32
|
+
build_profile=parsed_args.build_profile,
|
|
33
|
+
packages_filename=parsed_args.file,
|
|
34
|
+
channel=None,
|
|
35
|
+
user=None
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
workflow = Workflow(ConanAPI(), ConanOutput())
|
|
39
|
+
workflow.run(config)
|
|
File without changes
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""
|
|
2
|
+
An immutable POCO to store program configuration
|
|
3
|
+
"""
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass(frozen=True)
|
|
10
|
+
class Context:
|
|
11
|
+
"""
|
|
12
|
+
A top level state to hold command line arguments and program state.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
cci_root: Path
|
|
16
|
+
""" A filesystem Path to the root of the CCI project """
|
|
17
|
+
|
|
18
|
+
remote: str
|
|
19
|
+
""" The name of a configured (and authenticated) of a jFrog Artifactory package repository to use"""
|
|
20
|
+
|
|
21
|
+
build_profile: str
|
|
22
|
+
""" The profile used for building the package """
|
|
23
|
+
|
|
24
|
+
host_profile: str
|
|
25
|
+
|
|
26
|
+
packages_filename: str
|
|
27
|
+
|
|
28
|
+
channel: Optional[str]
|
|
29
|
+
user: Optional[str]
|
|
File without changes
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
from typing import Optional, List
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@dataclass(frozen=True)
|
|
6
|
+
class PackageRef:
|
|
7
|
+
"""
|
|
8
|
+
A reference to a package to be built.
|
|
9
|
+
"""
|
|
10
|
+
name: str
|
|
11
|
+
""" the name of the package to be built"""
|
|
12
|
+
version: Optional[str] = None
|
|
13
|
+
""" the version of the package to build """
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
@dataclass(frozen=True)
|
|
17
|
+
class ProfileRule:
|
|
18
|
+
""" A rule to determine whether a profile should be included in a build """
|
|
19
|
+
include: bool
|
|
20
|
+
pattern: str
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass(frozen=True)
|
|
24
|
+
class PackageEntry(PackageRef):
|
|
25
|
+
""" A description of a single package to build, that will be uploaded to the Conan jFrog Artifactory repository """
|
|
26
|
+
|
|
27
|
+
profiles: List[ProfileRule] = field(default_factory=list)
|
|
28
|
+
""" The list of profiles that this package should be built for """
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Parse 'package.txt' files which have a list of packages and optionally a version.
|
|
3
|
+
"""
|
|
4
|
+
import re
|
|
5
|
+
from io import TextIOWrapper
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import List, Optional
|
|
8
|
+
|
|
9
|
+
from .error.exception import PackageFileError
|
|
10
|
+
from cci_build.model.settings.types import PackageEntry, ProfileRule
|
|
11
|
+
|
|
12
|
+
# LINE = re.compile(r"^\s*([a-zA-Z0-9_.+-]+/[a-zA-Z0-9_.+-]+)(?:\s*\[(.+)\])?\s*$")
|
|
13
|
+
LINE_REGEX = re.compile(
|
|
14
|
+
r"""
|
|
15
|
+
^
|
|
16
|
+
|
|
17
|
+
(?:
|
|
18
|
+
\s* # Ignore leading whitespace
|
|
19
|
+
|
|
20
|
+
(?P<name> [a-zA-Z0-9_.+-]+ )
|
|
21
|
+
|
|
22
|
+
# An optional version
|
|
23
|
+
(?: \s* / \s* (?P<version>[\w.+-]+ ) )?
|
|
24
|
+
|
|
25
|
+
\s*
|
|
26
|
+
(?:
|
|
27
|
+
(?: p | profiles ) =
|
|
28
|
+
(?P<profiles> !? [\w.+*-]+ (?: , \s* !? [\w.+*-]+ )* )
|
|
29
|
+
)?
|
|
30
|
+
|
|
31
|
+
)?
|
|
32
|
+
\s* # allow trailing whitespace
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
(?: \# .* )? # ignore comments
|
|
36
|
+
|
|
37
|
+
$
|
|
38
|
+
""",
|
|
39
|
+
re.VERBOSE,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def parse_line(line: str) -> Optional[PackageEntry]:
|
|
46
|
+
"""
|
|
47
|
+
"""
|
|
48
|
+
m = LINE_REGEX.match(line)
|
|
49
|
+
if not m:
|
|
50
|
+
raise PackageFileError(f"Invalid line '{line}'")
|
|
51
|
+
|
|
52
|
+
name = m.group("name")
|
|
53
|
+
if name is not None:
|
|
54
|
+
version = m.group("version")
|
|
55
|
+
profiles = m.group("profiles").split(",") if m.group("profiles") else []
|
|
56
|
+
|
|
57
|
+
profile_rules = []
|
|
58
|
+
for r in [x.strip() for x in profiles]:
|
|
59
|
+
profile_rules.append(parse_profile_rule(r))
|
|
60
|
+
|
|
61
|
+
return PackageEntry(name=name, version=version, profiles=profile_rules)
|
|
62
|
+
return None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def parse_profile_rule(rule_str: str) -> ProfileRule:
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
"""
|
|
69
|
+
if not rule_str.startswith("!"):
|
|
70
|
+
return ProfileRule(include=True, pattern=rule_str)
|
|
71
|
+
else:
|
|
72
|
+
return ProfileRule(include=False, pattern=rule_str[1:])
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def parse_lines(stream: TextIOWrapper) -> List[PackageEntry]:
|
|
76
|
+
"""
|
|
77
|
+
Parse a 'package.txt' from a stream.
|
|
78
|
+
"""
|
|
79
|
+
return [
|
|
80
|
+
entry
|
|
81
|
+
for a_line in stream
|
|
82
|
+
if (entry := parse_line(a_line.strip())) is not None
|
|
83
|
+
]
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def load_package_file(path: Path) -> List[PackageEntry]:
|
|
87
|
+
"""
|
|
88
|
+
Parse a 'package.txt' from a filesystem based file.
|
|
89
|
+
"""
|
|
90
|
+
with open(path) as f:
|
|
91
|
+
return parse_lines(f)
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from fnmatch import fnmatch
|
|
2
|
+
from typing import List
|
|
3
|
+
|
|
4
|
+
from cci_build.model.settings.types import ProfileRule, PackageEntry
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def include( pkg: PackageEntry, profile: str) -> bool:
|
|
8
|
+
return include_rules(pkg.profiles, profile)
|
|
9
|
+
|
|
10
|
+
def include_rules(rules : List[ProfileRule], profile: str) -> bool:
|
|
11
|
+
"""
|
|
12
|
+
Given a host profile, check if a given package should be built.
|
|
13
|
+
|
|
14
|
+
Rules:
|
|
15
|
+
- no listed profiles, means match all profiles
|
|
16
|
+
- an exclamation prefix on a rule excludes those profiles
|
|
17
|
+
"""
|
|
18
|
+
if not rules:
|
|
19
|
+
# if no filter rules are provided, then the package is built for
|
|
20
|
+
# all profiles, so it is a match.
|
|
21
|
+
return True
|
|
22
|
+
|
|
23
|
+
# Find the first matching rule, or default to do not build
|
|
24
|
+
for rule in rules:
|
|
25
|
+
# check if the rule is a negative check (i.e. do not built this package).
|
|
26
|
+
#
|
|
27
|
+
if not rule.include:
|
|
28
|
+
if fnmatch(profile, rule.pattern):
|
|
29
|
+
return False # a positive match means that the package is not built
|
|
30
|
+
elif fnmatch(profile, rule.pattern):
|
|
31
|
+
return True # the rule positively matches the profile, so the package is built
|
|
32
|
+
|
|
33
|
+
return False
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Main workflow for building the packages.
|
|
3
|
+
"""
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Tuple, List, Optional, Any
|
|
6
|
+
|
|
7
|
+
from conan.api.conan_api import ConanAPI
|
|
8
|
+
from conan.api.model import RecipeReference, Remote, ListPattern, PackagesList
|
|
9
|
+
from conan.api.output import ConanOutput
|
|
10
|
+
from conan.internal.errors import NotFoundException
|
|
11
|
+
from conan.internal.graph.graph import DepsGraph, BINARY_BUILD
|
|
12
|
+
from conan.internal.model.profile import Profile
|
|
13
|
+
|
|
14
|
+
from cci_build.conan_centre_index import find_cci_conanfile
|
|
15
|
+
from cci_build.conan_client import ConanClient
|
|
16
|
+
from cci_build.error.exception import PackageFileError
|
|
17
|
+
from cci_build.model.context import Context
|
|
18
|
+
from cci_build.model.settings.types import PackageEntry
|
|
19
|
+
from cci_build.package_parser import load_package_file
|
|
20
|
+
from cci_build.profile_matcher import include_rules
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def make_revision(ref: RecipeReference) -> str:
|
|
24
|
+
"""
|
|
25
|
+
Make a conan search revision based on a recipe reference.
|
|
26
|
+
"""
|
|
27
|
+
pattern_str = f"{ref.name}/{ref.version if ref.version else '*'}"
|
|
28
|
+
return pattern_str + (f"@{ref.user}/{ref.channel}" if ref.user and ref.channel else "#*")
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def list_select(conan: ConanClient, search_pattern: ListPattern, remote: Optional[Remote]) -> bool:
|
|
32
|
+
"""
|
|
33
|
+
Select a list of conan recipes.
|
|
34
|
+
"""
|
|
35
|
+
try:
|
|
36
|
+
return conan.api.list.select(search_pattern, remote=remote)
|
|
37
|
+
except NotFoundException:
|
|
38
|
+
# Eat the exception (the specific error is not reported)
|
|
39
|
+
return False
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class Workflow:
|
|
43
|
+
"""
|
|
44
|
+
Run a conan workflow to build all missing packages in a jFrog Artifactory repository.
|
|
45
|
+
|
|
46
|
+
see:
|
|
47
|
+
- https://docs.conan.io/2/reference/commands/graph/info.html
|
|
48
|
+
- https://docs.conan.io/2/reference/extensions/python_api/GraphAPI.html
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(self, api: ConanAPI, logger: ConanOutput):
|
|
52
|
+
self.profile_build: Optional[Profile] = None
|
|
53
|
+
self.profile_host: Optional[Profile] = None
|
|
54
|
+
self.conan = ConanClient(api)
|
|
55
|
+
self.api = api
|
|
56
|
+
self.log = logger
|
|
57
|
+
|
|
58
|
+
def run(self, ctx: Context) -> None:
|
|
59
|
+
"""
|
|
60
|
+
Perform a minimal build of one host (target) profile. The inputs
|
|
61
|
+
to the process are:
|
|
62
|
+
- the local package list configuration file
|
|
63
|
+
- a local filesystem copy of a CCI
|
|
64
|
+
|
|
65
|
+
The state is help in:
|
|
66
|
+
- a jFrog Artifactory repository
|
|
67
|
+
|
|
68
|
+
The pre-requisites are:
|
|
69
|
+
- this script
|
|
70
|
+
- conan installed and configured with profiles
|
|
71
|
+
- a conan remote configured and authenticated to the Artifactory repository
|
|
72
|
+
|
|
73
|
+
"""
|
|
74
|
+
self.profile_host = self.api.profiles.get_profile([ctx.host_profile])
|
|
75
|
+
if self.profile_host is None:
|
|
76
|
+
raise ModuleNotFoundError(f"No profile found for '{ctx.host_profile}'")
|
|
77
|
+
|
|
78
|
+
self.profile_build = self.api.profiles.get_profile(
|
|
79
|
+
[ctx.build_profile]) if ctx.build_profile else self.api.profiles.get_default_build()
|
|
80
|
+
if self.profile_build is None:
|
|
81
|
+
raise ModuleNotFoundError(f"No profile found for '{ctx.build_profile}'")
|
|
82
|
+
|
|
83
|
+
# Load the configuration file which describes packages in the local CCI that need to be built
|
|
84
|
+
pkgs = load_package_file(Path(ctx.packages_filename))
|
|
85
|
+
self.log.info(f"Found {len(pkgs)} packages")
|
|
86
|
+
for pkg in pkgs:
|
|
87
|
+
self.log.info(f"Processing package '{pkg.name}:{pkg.version if pkg.version else "*"}'")
|
|
88
|
+
|
|
89
|
+
if ctx.remote:
|
|
90
|
+
remote = self.api.remotes.get(ctx.remote)
|
|
91
|
+
remotes = [remote] if remote else []
|
|
92
|
+
|
|
93
|
+
#
|
|
94
|
+
# Using the local CCI, map the desired packages to the exact recipe and version to be built
|
|
95
|
+
# and filter which packages should be included based on the profile filter criteria.
|
|
96
|
+
#
|
|
97
|
+
refs = self.filter_package_names(ctx, pkgs)
|
|
98
|
+
|
|
99
|
+
self.create_remote_recipe(refs, ctx, remote)
|
|
100
|
+
graph = self.build_package_graph(refs, ctx, remotes)
|
|
101
|
+
self.install_and_upload_missing(graph, remote, remotes)
|
|
102
|
+
else:
|
|
103
|
+
raise NotFoundException("No remote configured")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def filter_package_names(self, ctx: Context, pkgs: list[PackageEntry]) -> List[Tuple[RecipeReference, Path]]:
|
|
107
|
+
"""
|
|
108
|
+
Given the list of packages to build, filter them based on the host profile.
|
|
109
|
+
"""
|
|
110
|
+
refs = []
|
|
111
|
+
for pkg in pkgs:
|
|
112
|
+
|
|
113
|
+
# Check if the profile rules in the configuration file allow this 'host' profile
|
|
114
|
+
if include_rules(pkg.profiles, ctx.host_profile):
|
|
115
|
+
|
|
116
|
+
# Given the package/profile/version needs to be present, resolve the package
|
|
117
|
+
# and the version in the CCI.
|
|
118
|
+
version, conanfile = find_cci_conanfile(ctx.cci_root, pkg.name, pkg.version)
|
|
119
|
+
if version:
|
|
120
|
+
ref = RecipeReference(name=pkg.name, version=version, user=ctx.user, channel=ctx.channel)
|
|
121
|
+
refs.append((ref, conanfile,))
|
|
122
|
+
self.log.info(f"Found CCI reference '{ref.name}:{ref.version}'")
|
|
123
|
+
else:
|
|
124
|
+
self.log.error("Can not determine version for package '%s'", pkg.name)
|
|
125
|
+
raise PackageFileError()
|
|
126
|
+
else:
|
|
127
|
+
self.log.info("Skipping package '%s' as it does not match the host profile", pkg.name)
|
|
128
|
+
return refs
|
|
129
|
+
|
|
130
|
+
def build_package_graph(
|
|
131
|
+
self, packages: List[Tuple[RecipeReference, Path]], context: Context, remotes: List[Remote]) -> DepsGraph:
|
|
132
|
+
"""
|
|
133
|
+
For each of the packages, add it to the package graph.
|
|
134
|
+
"""
|
|
135
|
+
requires = [ref for ref, _ in packages]
|
|
136
|
+
deps_graph = self.api.graph.load_graph_requires(
|
|
137
|
+
requires,
|
|
138
|
+
tool_requires=None,
|
|
139
|
+
profile_host=self.profile_host, profile_build=self.profile_build,
|
|
140
|
+
lockfile=None,
|
|
141
|
+
remotes=remotes,
|
|
142
|
+
update=True)
|
|
143
|
+
# Stop now if there was an error
|
|
144
|
+
deps_graph.report_graph_error()
|
|
145
|
+
|
|
146
|
+
# mark only remote-missing binaries for build (replaces make_build_graph/make_one)
|
|
147
|
+
self.api.graph.analyze_binaries(deps_graph, build_mode=["missing"], remotes=remotes, update=True)
|
|
148
|
+
|
|
149
|
+
return deps_graph
|
|
150
|
+
|
|
151
|
+
def create_remote_recipe(self, refs: List[Tuple[RecipeReference, Path]], context: Context, remote: Remote):
|
|
152
|
+
"""
|
|
153
|
+
Upload all recipes that don't exist at the remote.
|
|
154
|
+
|
|
155
|
+
Note: this creates a recipe in the remote, but none of the binaries.
|
|
156
|
+
"""
|
|
157
|
+
for ref, conanfile_path in refs:
|
|
158
|
+
search_pattern = ListPattern(make_revision(ref))
|
|
159
|
+
self.log.info(f'Searching for recipe "{make_revision(ref)}"')
|
|
160
|
+
remote_check = list_select(self.conan, search_pattern, remote=remote)
|
|
161
|
+
if not remote_check:
|
|
162
|
+
|
|
163
|
+
local_recipes = list_select(self.conan, search_pattern, remote=None)
|
|
164
|
+
if not local_recipes:
|
|
165
|
+
self.log.info(f'Exporting recipe "{str(ref)}"')
|
|
166
|
+
clean_conanfile_path = str(Path(conanfile_path).resolve().absolute())
|
|
167
|
+
|
|
168
|
+
exported_ref, conanfile_obj = self.conan.api.export.export(
|
|
169
|
+
path=clean_conanfile_path,
|
|
170
|
+
name=ref.name,
|
|
171
|
+
version=ref.version,
|
|
172
|
+
user=ref.user,
|
|
173
|
+
channel=ref.channel)
|
|
174
|
+
|
|
175
|
+
self.log.info(f'Upload recipe "{str(ref)}"')
|
|
176
|
+
self.conan.api.upload.upload_full(
|
|
177
|
+
package_list=local_recipes,
|
|
178
|
+
remote=remote,
|
|
179
|
+
enabled_remotes=[remote],
|
|
180
|
+
dry_run=False)
|
|
181
|
+
else:
|
|
182
|
+
self.log.info(f'Recipe "{str(ref)}" present at remote "{remote.name}"')
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def install_and_upload_missing(self, graph: DepsGraph, remote: Remote, remotes: list[Remote] | list[Any]):
|
|
186
|
+
"""
|
|
187
|
+
Locally build those binaries (for the given profile) that are missing at the remote.
|
|
188
|
+
"""
|
|
189
|
+
install_error = self.api.install.install_binaries(deps_graph=graph, remotes=remotes, return_install_error=True)
|
|
190
|
+
|
|
191
|
+
built = PackagesList()
|
|
192
|
+
for node in graph.nodes:
|
|
193
|
+
# a completed build has a package revision; a failed one does not
|
|
194
|
+
if node.binary == BINARY_BUILD and node.pref.revision:
|
|
195
|
+
built.add_ref(node.ref)
|
|
196
|
+
built.add_pref(node.pref)
|
|
197
|
+
if built:
|
|
198
|
+
self.api.upload.upload_full(
|
|
199
|
+
built, remote, enabled_remotes=remotes, check_integrity=True, dry_run=False)
|
|
200
|
+
if install_error is not None:
|
|
201
|
+
raise install_error
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
LICENSE
|
|
2
|
+
README.md
|
|
3
|
+
setup.py
|
|
4
|
+
extensions/commands/cmd_cci_build.py
|
|
5
|
+
src/cci_build/__init__.py
|
|
6
|
+
src/cci_build/conan_centre_index.py
|
|
7
|
+
src/cci_build/conan_client.py
|
|
8
|
+
src/cci_build/installer.py
|
|
9
|
+
src/cci_build/main.py
|
|
10
|
+
src/cci_build/package_parser.py
|
|
11
|
+
src/cci_build/profile_matcher.py
|
|
12
|
+
src/cci_build/workflow.py
|
|
13
|
+
src/cci_build/model/__init__.py
|
|
14
|
+
src/cci_build/model/context.py
|
|
15
|
+
src/cci_build/model/settings/__init__.py
|
|
16
|
+
src/cci_build/model/settings/types.py
|
|
17
|
+
src/conan_cci_build.egg-info/PKG-INFO
|
|
18
|
+
src/conan_cci_build.egg-info/SOURCES.txt
|
|
19
|
+
src/conan_cci_build.egg-info/dependency_links.txt
|
|
20
|
+
src/conan_cci_build.egg-info/entry_points.txt
|
|
21
|
+
src/conan_cci_build.egg-info/requires.txt
|
|
22
|
+
src/conan_cci_build.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|