sdg-hub 0.1.0a2__py3-none-any.whl → 0.1.0a3__py3-none-any.whl

sdg_hub/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.1.0a2'
+__version__ = version = '0.1.0a3'
 __version_tuple__ = version_tuple = (0, 1, 0)

sdg_hub/flow.py

@@ -38,10 +38,37 @@ class Flow(ABC):
         self.base_path = str(resources.files(__package__))
         self.registered_blocks = BlockRegistry.get_registry()
 
+    def _getFilePath(self, dirs, filename):
+        """
+        Find a named configuration file.
+
+        Files are checked in the following order
+            - absulute path is always used
+            - checked relative to the directories in "dirs"
+            - relative the the current directory
+
+        Args:
+            dirs (list): Directories in which to search for "config_path"
+            config_path (str): The path to the configuration file.
+
+        Returns:
+            Selected file path
+        """
+        if os.path.isabs(filename):
+            return filename
+        for d in dirs:
+            full_file_path = os.path.join(d, filename)
+            if os.path.isfile(full_file_path):
+                return full_file_path
+        # If not found above then return the path unchanged i.e.
+        # assume the path is relative to the current directory
+        return filename
+
     def get_flow_from_file(self, yaml_path: str) -> list:
         yaml_path_relative_to_base = os.path.join(self.base_path, yaml_path)
         if os.path.isfile(yaml_path_relative_to_base):
             yaml_path = yaml_path_relative_to_base
+        yaml_dir = os.path.dirname(yaml_path)
 
         try:
             with open(yaml_path, "r", encoding="utf-8") as yaml_file:
@@ -86,33 +113,23 @@ class Flow(ABC):
 
             # update config path to absolute path
             if "config_path" in block["block_config"]:
-                config_path_relative_to_base = os.path.join(
-                    self.base_path, block["block_config"]["config_path"]
+                block["block_config"]["config_path"] = self._getFilePath(
+                    [yaml_dir, self.base_path], block["block_config"]["config_path"]
                 )
-                if os.path.isfile(config_path_relative_to_base):
-                    block["block_config"]["config_path"] = config_path_relative_to_base
 
             # update config paths to absolute paths - this might be a list or a dict
             if "config_paths" in block["block_config"]:
                 if isinstance(block["block_config"]["config_paths"], dict):
                     for key, path in block["block_config"]["config_paths"].items():
-                        config_path_relative_to_base = os.path.join(
-                            self.base_path, path
+                        block["block_config"]["config_paths"][key] = self._getFilePath(
+                            [yaml_dir, self.base_path], path
                         )
-                        if os.path.isfile(config_path_relative_to_base):
-                            block["block_config"]["config_paths"][key] = (
-                                config_path_relative_to_base
-                            )
 
-                if isinstance(block["block_config"]["config_paths"], list):
+                elif isinstance(block["block_config"]["config_paths"], list):
                     for i, path in enumerate(block["block_config"]["config_paths"]):
-                        config_path_relative_to_base = os.path.join(
-                            self.base_path, path
+                        block["block_config"]["config_paths"][i] = self._getFilePath(
+                            [yaml_dir, self.base_path], path
                         )
-                        if os.path.isfile(config_path_relative_to_base):
-                            block["block_config"]["config_paths"][i] = (
-                                config_path_relative_to_base
-                            )
 
             if "operation" in block["block_config"]:
                 block["block_config"]["operation"] = OPERATOR_MAP[

{sdg_hub-0.1.0a2.dist-info → sdg_hub-0.1.0a3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sdg_hub
-Version: 0.1.0a2
+Version: 0.1.0a3
 Summary: Synthetic Data Generation
 Author-email: Red Hat AI Innovation <abhandwa@redhat.com>
 License: Apache-2.0

{sdg_hub-0.1.0a2.dist-info → sdg_hub-0.1.0a3.dist-info}/RECORD

@@ -1,6 +1,6 @@
 sdg_hub/__init__.py,sha256=5Wa6onDndPvG4iwnjq2jK747t3-7XKdQn2WfHfq1sFc,67
-sdg_hub/_version.py,sha256=ypYjg1lOKm-KjhEwqwSB8XPrSuAfnfK8hSDDWtD6VoA,513
-sdg_hub/flow.py,sha256=t03SrX2AHHhzt5ebOAg2TwuZasNerrucanjb5_6FnOk,5025
+sdg_hub/_version.py,sha256=wrhrM1UZdxROWn7XOHbbPZa5jOBzV8tlSBMw233huBg,513
+sdg_hub/flow.py,sha256=3b97fMei1rWuQWeNfv-xyHKUbcMaf-d_b9Xms9J3BCQ,5425
 sdg_hub/logger_config.py,sha256=7uHEJVRfym1c4n95DOKHelLXqAus8uHsZYmzLsEjqpo,422
 sdg_hub/pipeline.py,sha256=u24ccryfy_nOSvsrWiynNmq1rOmOOkw1L5-TqJvuRSo,2339
 sdg_hub/prompts.py,sha256=dOiC9CsNbMt5Km9PnwyuW0v9zUs3cVXE5jZYwtXZTwc,1957
@@ -80,15 +80,11 @@ sdg_hub/flows/generation/skills/synth_grounded_skills.yaml,sha256=91Dm--agpmbm02
 sdg_hub/flows/generation/skills/synth_skills.yaml,sha256=PhUP2iBo4RkeFafSW-qxh4WmX_ZTfGi0UAmwN_XSTqs,1504
 sdg_hub/utils/__init__.py,sha256=UEo-9qPt5iVKBIRvgZhOI0SoIBO6zeBxOuLvUQXaM3g,185
 sdg_hub/utils/chunking.py,sha256=VSPQ8dSFI5LF4sefcI0tzWG0Vc1rM_FSMTO6xg_iFzA,2556
-sdg_hub/utils/datamixing.py,sha256=nkjyRY3AnOYYJAU4cyVk17XQwr9hUC6DicU_tuK8O8I,4099
 sdg_hub/utils/datautils.py,sha256=0t_SZ_UXBKl8uL6rVp3SUh8YKRbzKlh2oO5gr2cKyEw,389
 sdg_hub/utils/docprocessor.py,sha256=Z4J2DfLhRxMCeIeMKttwi-FdivmPqI-hjEwq6-Ub35c,12485
-sdg_hub/utils/json.py,sha256=Ub6OzSYu8PqUHki9RXxuwLkW9RSqY_DBWA5qiyLE_PA,1378
-sdg_hub/utils/models.py,sha256=mp6J1cPcEUYvr6AvRAflsnY0E5moLcmHj3MgnR0FPE4,894
 sdg_hub/utils/parse_and_convert.py,sha256=I27FdS-H2mSoZ07SsKZmNYM2F_Cg7GHTBXD7YNgASNw,13443
-sdg_hub/utils/taxonomy.py,sha256=WmO4oR4O1o1I9Yr7urfMNP80nS-p2z5bhgvDt0cg4mE,16947
-sdg_hub-0.1.0a2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-sdg_hub-0.1.0a2.dist-info/METADATA,sha256=ol0u_UumQKw1VVdatPV0OW0-J7bPx_EKdqn1O7scJoo,5847
-sdg_hub-0.1.0a2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-sdg_hub-0.1.0a2.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
-sdg_hub-0.1.0a2.dist-info/RECORD,,
+sdg_hub-0.1.0a3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+sdg_hub-0.1.0a3.dist-info/METADATA,sha256=vUusH0jLACOcoxvTL-e5dAPfhoTV--zgs_MJ-6IYQfQ,5847
+sdg_hub-0.1.0a3.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+sdg_hub-0.1.0a3.dist-info/top_level.txt,sha256=TqI7d-HE1n6zkXFkU0nF3A1Ct0P0pBaqI675uFokhx4,8
+sdg_hub-0.1.0a3.dist-info/RECORD,,

sdg_hub/utils/datamixing.py

@@ -1,123 +0,0 @@
-# Standard
-import json
-import os
-
-# Third Party
-from datasets import Dataset, load_dataset
-import yaml
-
-# First Party
-from sdg_hub.logger_config import setup_logger
-from .datautils import safe_concatenate_datasets
-
-
-LOGGER = setup_logger(__name__)
-ALLOWED_COLS = ["id", "messages", "metadata"]
-
-
-def adjust_train_sample_size(ds: Dataset, num_samples: int):
-    LOGGER.info(f"Rebalancing dataset to have {num_samples} samples ...")
-    df = ds.to_pandas()
-    df = df.sample(n=num_samples, random_state=42, replace=True).reset_index(drop=True)
-    return Dataset.from_pandas(df)
-
-
-def load_ds(path, sampling_size):
-    if path.endswith(".jsonl"):
-        LOGGER.info(f"Loading dataset from {path} ...")
-        dataset = load_dataset("json", data_files=path, split="train")
-    else: 
-        LOGGER.info(f"Loading dataset from HF {path} ...")
-        dataset = load_dataset(path, split="train")
-    LOGGER.info(f"Dataset columns: {dataset.column_names}")
-    LOGGER.info(f"Dataset loaded with {len(dataset)} samples")
-
-    if sampling_size != 1.0:
-        if isinstance(sampling_size, int):
-            num_samples = sampling_size
-        else:
-            num_samples = int(len(dataset) * sampling_size)
-        dataset = adjust_train_sample_size(dataset, num_samples)
-
-    # move any column that is not in ALLOWED_COLS to metadata
-    def move_unallowed_cols_to_metadata(example):
-        metadata = example.get("metadata", {})
-        if isinstance(metadata, str):
-            metadata = json.loads(metadata)
-        for col in dataset.column_names:
-            if col not in ALLOWED_COLS:
-                metadata[col] = example[col]
-                example.pop(col)
-        example["metadata"] = json.dumps(metadata)
-        return example
-
-    dataset = dataset.map(move_unallowed_cols_to_metadata, num_proc=8)
-
-    # check if metadata column is string if not convert it using json.dumps
-    if not isinstance(dataset["metadata"][0], str):
-        dataset = dataset.map(
-            lambda x: {"metadata": json.dumps(x["metadata"])}, num_proc=8
-        )
-
-    return dataset
-
-
-def add_system_message(sample: dict, sys_prompt: str) -> dict:
-    # check if the messages have role system
-    has_system = False
-    for msg in sample["messages"]:
-        if msg["role"] == "system":
-            has_system = True
-            msg["content"] = sys_prompt
-
-    if not has_system:
-        sample["messages"].insert(0, {"role": "system", "content": sys_prompt})
-
-    return sample
-
-
-class Recipe:
-    def __init__(self, recipe_path):
-        self.recipe_path = recipe_path
-        self.recipe = self._load_recipe()
-        self.sys_prompt = self.recipe.get("sys_prompt", "")
-        self.dataset_added = False
-
-    def _load_recipe(self):
-        with open(self.recipe_path, encoding="utf-8") as fp:
-            return yaml.safe_load(fp)
-
-    def add_dataset(self, path, sampling_size=1.0):
-        self.dataset_added = True
-        self.recipe["datasets"].append({"path": path, "sampling_size": sampling_size})
-
-    def save_recipe(self, output_path):
-        # check if directory exists
-        output_dir = os.path.dirname(output_path)
-        if not os.path.exists(output_dir):
-            os.makedirs(output_dir)
-
-        with open(output_path, "w", encoding="utf-8") as fp:
-            yaml.dump(self.recipe, fp)
-
-    def save_mixed_dataset(self, output_path):
-        if not self.dataset_added:
-            LOGGER.error("No dataset added to the recipe")
-
-        mixed_ds = [
-            load_ds(dataset["path"], dataset["sampling_size"])
-            for dataset in self.recipe["datasets"]
-        ]
-
-        mixed_ds = safe_concatenate_datasets(mixed_ds)
-        mixed_ds = mixed_ds.map(
-            add_system_message, fn_kwargs={"sys_prompt": self.sys_prompt}, num_proc=8
-        )
-
-        # assert that the dataset only has the allowed columns
-        assert set(mixed_ds.column_names) == set(
-            ALLOWED_COLS
-        ), "Dataset has invalid columns"
-
-        mixed_ds.to_json(output_path, orient="records", lines=True)
-        LOGGER.info(f"Mixed Dataset saved to {output_path}")

sdg_hub/utils/json.py

@@ -1,48 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-# Standard
-import io
-import json
-import os
-
-
-def _make_w_io_base(f, mode: str):
-    # pylint: disable=consider-using-with
-    if not isinstance(f, io.IOBase):
-        f_dirname = os.path.dirname(f)
-        if f_dirname != "":
-            os.makedirs(f_dirname, exist_ok=True)
-        f = open(f, mode=mode, encoding="utf-8")
-    return f
-
-
-def _make_r_io_base(f, mode: str):
-    # pylint: disable=consider-using-with
-    if not isinstance(f, io.IOBase):
-        f = open(f, mode=mode, encoding="utf-8")
-    return f
-
-
-def jdump(obj, f, mode="w", indent=4, default=str):
-    """Dump a str or dictionary to a file in json format.
-
-    Args:
-        obj: An object to be written.
-        f: A string path to the location on disk.
-        mode: Mode for opening the file.
-        indent: Indent for storing json dictionaries.
-        default: A function to handle non-serializable entries; defaults to `str`.
-    """
-    with _make_w_io_base(f, mode) as f_:
-        if isinstance(obj, (dict, list)):
-            json.dump(obj, f_, indent=indent, default=default)
-        elif isinstance(obj, str):
-            f_.write(obj)
-        else:
-            raise ValueError(f"Unexpected type: {type(obj)}")
-
-
-def jload(f, mode="r"):
-    """Load a .json file into a dictionary."""
-    with _make_r_io_base(f, mode) as f_:
-        return json.load(f_)

sdg_hub/utils/models.py

@@ -1,31 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-# Standard
-import os
-import re
-
-# First Party
-from sdg_hub.utils import GenerateException
-
-# When otherwise unknown, ilab uses this as the default family
-DEFAULT_MODEL_FAMILY = "merlinite"
-
-# Model families understood by ilab
-MODEL_FAMILIES = set(("merlinite", "mixtral"))
-
-# Map model names to their family
-MODEL_FAMILY_MAPPINGS = {
-    "granite": "merlinite",
-}
-
-
-def get_model_family(forced, model_path):
-    forced = MODEL_FAMILY_MAPPINGS.get(forced, forced)
-    if forced and forced.lower() not in MODEL_FAMILIES:
-        raise GenerateException("Unknown model family: %s" % forced)
-
-    # Try to guess the model family based on the model's filename
-    guess = re.match(r"^\w*", os.path.basename(model_path)).group(0).lower()
-    guess = MODEL_FAMILY_MAPPINGS.get(guess, guess)
-
-    return guess if guess in MODEL_FAMILIES else DEFAULT_MODEL_FAMILY

sdg_hub/utils/taxonomy.py

@@ -1,489 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-
-# Standard
-from functools import cache
-from pathlib import Path
-from typing import Any, Dict, List, Mapping, Optional, Union
-import glob
-import json
-import logging
-import os
-import re
-import subprocess
-import tempfile
-
-# Third Party
-import git
-import gitdb
-import yaml
-
-# First Party
-from sdg_hub import utils
-from sdg_hub.utils import chunking
-
-logger = logging.getLogger(__name__)
-
-DEFAULT_YAML_RULES = """\
-extends: relaxed
-
-rules:
-  line-length:
-    max: 120
-"""
-
-
-class TaxonomyReadingException(Exception):
-    """An exception raised during reading of the taxonomy."""
-
-
-TAXONOMY_FOLDERS: List[str] = ["compositional_skills", "knowledge"]
-"""Taxonomy folders which are also the schema names"""
-
-
-def _istaxonomyfile(fn):
-    path = Path(fn)
-    if path.suffix == ".yaml" and path.parts[0] in TAXONOMY_FOLDERS:
-        return True
-    return False
-
-
-def _get_taxonomy_diff(repo="taxonomy", base="origin/main"):
-    repo = git.Repo(repo)
-    untracked_files = [u for u in repo.untracked_files if _istaxonomyfile(u)]
-
-    branches = [b.name for b in repo.branches]
-
-    head_commit = None
-    if "/" in base:
-        re_git_branch = re.compile(f"remotes/{base}$", re.MULTILINE)
-    elif base in branches:
-        re_git_branch = re.compile(f"{base}$", re.MULTILINE)
-    else:
-        try:
-            head_commit = repo.commit(base)
-        except gitdb.exc.BadName as e:
-            raise SystemExit(
-                yaml.YAMLError(
-                    f'Couldn\'t find the taxonomy git ref "{base}" from the current HEAD'
-                )
-            ) from e
-
-    # Move backwards from HEAD until we find the first commit that is part of base
-    # then we can take our diff from there
-    current_commit = repo.commit("HEAD")
-    while not head_commit:
-        branches = repo.git.branch("-a", "--contains", current_commit.hexsha)
-        if re_git_branch.findall(branches):
-            head_commit = current_commit
-            break
-        try:
-            current_commit = current_commit.parents[0]
-        except IndexError as e:
-            raise SystemExit(
-                yaml.YAMLError(
-                    f'Couldn\'t find the taxonomy base branch "{base}" from the current HEAD'
-                )
-            ) from e
-
-    modified_files = [
-        d.b_path
-        for d in head_commit.diff(None)
-        if not d.deleted_file and _istaxonomyfile(d.b_path)
-    ]
-
-    updated_taxonomy_files = list(set(untracked_files + modified_files))
-    return updated_taxonomy_files
-
-
-def _get_documents(
-    source: Dict[str, Union[str, List[str]]],
-    skip_checkout: bool = False,
-) -> List[str]:
-    """
-    Retrieve the content of files from a Git repository.
-
-    Args:
-        source (dict): Source info containing repository URL, commit hash, and list of file patterns.
-
-    Returns:
-         List[str]: List of document contents.
-    """ ""
-    repo_url = source.get("repo")
-    commit_hash = source.get("commit")
-    file_patterns = source.get("patterns", [])
-    with tempfile.TemporaryDirectory() as temp_dir:
-        try:
-            repo = git.Repo.clone_from(repo_url, temp_dir)
-            if not skip_checkout:
-                repo.git.checkout(commit_hash)
-
-            file_contents = []
-
-            logger.debug("Processing files...")
-            for pattern in file_patterns:
-                for file_path in glob.glob(os.path.join(repo.working_dir, pattern)):
-                    if os.path.isfile(file_path) and file_path.endswith(".md"):
-                        with open(file_path, "r", encoding="utf-8") as file:
-                            file_contents.append(file.read())
-
-            if file_contents:
-                return file_contents
-            raise SystemExit("Couldn't find knowledge documents")
-        except (OSError, git.exc.GitCommandError, FileNotFoundError) as e:
-            raise e
-
-
-@cache
-def _load_schema(path: "importlib.resources.abc.Traversable") -> "referencing.Resource":
-    """Load the schema from the path into a Resource object.
-
-    Args:
-        path (Traversable): Path to the schema to be loaded.
-
-    Raises:
-        NoSuchResource: If the resource cannot be loaded.
-
-    Returns:
-        Resource: A Resource containing the requested schema.
-    """
-    # pylint: disable=C0415
-    # Third Party
-    from referencing import Resource
-    from referencing.exceptions import NoSuchResource
-    from referencing.jsonschema import DRAFT202012
-
-    try:
-        contents = json.loads(path.read_text(encoding="utf-8"))
-        resource = Resource.from_contents(
-            contents=contents, default_specification=DRAFT202012
-        )
-    except Exception as e:
-        raise NoSuchResource(ref=str(path)) from e
-    return resource
-
-
-def _validate_yaml(contents: Mapping[str, Any], taxonomy_path: Path) -> int:
-    """Validate the parsed yaml document using the taxonomy path to
-    determine the proper schema.
-
-    Args:
-        contents (Mapping): The parsed yaml document to validate against the schema.
-        taxonomy_path (Path): Relative path of the taxonomy yaml document where the
-        first element is the schema to use.
-
-    Returns:
-        int: The number of errors found during validation.
-        Messages for each error have been logged.
-    """
-    # pylint: disable=C0415
-    # Standard
-    from importlib import resources
-
-    # Third Party
-    from jsonschema.protocols import Validator
-    from jsonschema.validators import validator_for
-    from referencing import Registry, Resource
-    from referencing.exceptions import NoSuchResource
-    from referencing.typing import URI
-
-    errors = 0
-    version = _get_version(contents)
-    schemas_path = resources.files("instructlab.schema").joinpath(f"v{version}")
-
-    def retrieve(uri: URI) -> Resource:
-        path = schemas_path.joinpath(uri)
-        return _load_schema(path)
-
-    schema_name = taxonomy_path.parts[0]
-    if schema_name not in TAXONOMY_FOLDERS:
-        schema_name = "knowledge" if "document" in contents else "compositional_skills"
-        logger.info(
-            f"Cannot determine schema name from path {taxonomy_path}. Using {schema_name} schema."
-        )
-
-    try:
-        schema_resource = retrieve(f"{schema_name}.json")
-        schema = schema_resource.contents
-        validator_cls = validator_for(schema)
-        validator: Validator = validator_cls(
-            schema, registry=Registry(retrieve=retrieve)
-        )
-
-        for validation_error in validator.iter_errors(contents):
-            errors += 1
-            yaml_path = validation_error.json_path[1:]
-            if not yaml_path:
-                yaml_path = "."
-            if validation_error.validator == "minItems":
-                # Special handling for minItems which can have a long message for seed_examples
-                message = (
-                    f"Value must have at least {validation_error.validator_value} items"
-                )
-            else:
-                message = validation_error.message[-200:]
-            logger.error(
-                f"Validation error in {taxonomy_path}: [{yaml_path}] {message}"
-            )
-    except NoSuchResource as e:
-        cause = e.__cause__ if e.__cause__ is not None else e
-        errors += 1
-        logger.error(f"Cannot load schema file {e.ref}. {cause}")
-
-    return errors
-
-
-def _get_version(contents: Mapping) -> int:
-    version = contents.get("version", 1)
-    if not isinstance(version, int):
-        # schema validation will complain about the type
-        try:
-            version = int(version)
-        except ValueError:
-            version = 1  # fallback to version 1
-    return version
-
-
-# pylint: disable=broad-exception-caught
-def _read_taxonomy_file(file_path: str, yaml_rules: Optional[str] = None):
-    seed_instruction_data = []
-    warnings = 0
-    errors = 0
-    file_path = Path(file_path).resolve()
-    # file should end with ".yaml" explicitly
-    if file_path.suffix != ".yaml":
-        logger.warning(
-            f"Skipping {file_path}! Use lowercase '.yaml' extension instead."
-        )
-        warnings += 1
-        return None, warnings, errors
-    for i in range(len(file_path.parts) - 1, -1, -1):
-        if file_path.parts[i] in TAXONOMY_FOLDERS:
-            taxonomy_path = Path(*file_path.parts[i:])
-            break
-    else:
-        taxonomy_path = file_path
-    # read file if extension is correct
-    try:
-        with open(file_path, "r", encoding="utf-8") as file:
-            contents = yaml.safe_load(file)
-        if not contents:
-            logger.warning(f"Skipping {file_path} because it is empty!")
-            warnings += 1
-            return None, warnings, errors
-        if not isinstance(contents, Mapping):
-            logger.error(
-                f"{file_path} is not valid. The top-level element is not an object with key-value pairs."
-            )
-            errors += 1
-            return None, warnings, errors
-
-        # do general YAML linting if specified
-        version = _get_version(contents)
-        if version > 1:  # no linting for version 1 yaml
-            if yaml_rules is not None:
-                is_file = os.path.isfile(yaml_rules)
-                if is_file:
-                    logger.debug(f"Using YAML rules from {yaml_rules}")
-                    yamllint_cmd = [
-                        "yamllint",
-                        "-f",
-                        "parsable",
-                        "-c",
-                        yaml_rules,
-                        file_path,
-                        "-s",
-                    ]
-                else:
-                    logger.debug(f"Cannot find {yaml_rules}. Using default rules.")
-                    yamllint_cmd = [
-                        "yamllint",
-                        "-f",
-                        "parsable",
-                        "-d",
-                        DEFAULT_YAML_RULES,
-                        file_path,
-                        "-s",
-                    ]
-            else:
-                yamllint_cmd = [
-                    "yamllint",
-                    "-f",
-                    "parsable",
-                    "-d",
-                    DEFAULT_YAML_RULES,
-                    file_path,
-                    "-s",
-                ]
-            try:
-                subprocess.check_output(yamllint_cmd, text=True)
-            except subprocess.SubprocessError as e:
-                lint_messages = [f"Problems found in file {file_path}"]
-                parsed_output = e.output.splitlines()
-                for p in parsed_output:
-                    errors += 1
-                    delim = str(file_path) + ":"
-                    parsed_p = p.split(delim)[1]
-                    lint_messages.append(parsed_p)
-                logger.error("\n".join(lint_messages))
-                return None, warnings, errors
-
-        # validation_errors = _validate_yaml(contents, taxonomy_path)
-        # if validation_errors:
-        #     errors += validation_errors
-        #     return None, warnings, errors
-
-        # get seed instruction data
-        tax_path = "->".join(taxonomy_path.parent.parts)
-        task_description = contents.get("task_description", None)
-        domain = contents.get("domain")
-        documents = contents.get("document")
-        if documents:
-            documents = _get_documents(source=documents)
-            logger.debug("Content from git repo fetched")
-
-        for seed_example in contents.get("seed_examples"):
-            context = seed_example.get("context", "")
-            if 'questions_and_answers' in seed_example:
-                question_answer_list = seed_example.get("questions_and_answers")
-                seed_instruction_data.append(
-                    {
-                        "questions_and_answers": question_answer_list,
-                        "input": context,
-                        "taxonomy_path": tax_path,
-                        "document": documents,
-                        "domain": domain,
-                        "document_outline": contents.get("document_outline")
-                    }
-                )
-            else:
-                question = seed_example.get("question")
-                answer = seed_example.get("answer")
-           
-                seed_instruction_data.append(
-                    {
-                        "instruction": question,
-                        "input": context,
-                        "output": answer,
-                        "taxonomy_path": tax_path,
-                        "task_description": task_description,
-                        "document": documents,
-                        "domain": domain,
-                    }
-                )
-    except Exception as e:
-        errors += 1
-        raise TaxonomyReadingException(f"Exception {e} raised in {file_path}") from e
-
-    return seed_instruction_data, warnings, errors
-
-
-def read_taxonomy(taxonomy, taxonomy_base, yaml_rules):
-    seed_instruction_data = []
-    is_file = os.path.isfile(taxonomy)
-    if is_file:  # taxonomy is file
-        seed_instruction_data, warnings, errors = _read_taxonomy_file(
-            taxonomy, yaml_rules
-        )
-        if warnings:
-            logger.warning(
-                f"{warnings} warnings (see above) due to taxonomy file not (fully) usable."
-            )
-        if errors:
-            raise SystemExit(yaml.YAMLError("Taxonomy file with errors! Exiting."))
-    else:  # taxonomy is dir
-        # Gather the new or changed YAMLs using git diff
-        updated_taxonomy_files = _get_taxonomy_diff(taxonomy, taxonomy_base)
-        total_errors = 0
-        total_warnings = 0
-        if updated_taxonomy_files:
-            logger.debug("Found new taxonomy files:")
-            for e in updated_taxonomy_files:
-                logger.debug(f"* {e}")
-        for f in updated_taxonomy_files:
-            file_path = os.path.join(taxonomy, f)
-            data, warnings, errors = _read_taxonomy_file(file_path, yaml_rules)
-            total_warnings += warnings
-            total_errors += errors
-            if data:
-                seed_instruction_data.extend(data)
-        if total_warnings:
-            logger.warning(
-                f"{total_warnings} warnings (see above) due to taxonomy files that were not (fully) usable."
-            )
-        if total_errors:
-            raise SystemExit(
-                yaml.YAMLError(f"{total_errors} taxonomy files with errors! Exiting.")
-            )
-    return seed_instruction_data
-
-
-def read_taxonomy_leaf_nodes(taxonomy, taxonomy_base, yaml_rules):
-    seed_instruction_data = read_taxonomy(taxonomy, taxonomy_base, yaml_rules)
-
-    # Transform into a more convenient format to feed into our updated SDG library
-    leaf_nodes = {}
-    for seed in seed_instruction_data:
-        node = leaf_nodes.setdefault(seed["taxonomy_path"], [])
-        node.append(seed)
-        leaf_nodes[seed["taxonomy_path"]] = node
-
-    return leaf_nodes
-
-
-def _knowledge_leaf_node_to_samples(leaf_node, server_ctx_size, chunk_word_count):
-    samples = []
-    # document is the same for the whole leaf node
-    chunks = (
-        chunking.chunk_document(
-            documents=leaf_node[0]["document"],
-            server_ctx_size=server_ctx_size,
-            chunk_word_count=chunk_word_count,
-        )
-        if leaf_node[0].get("document")
-        else []
-    )
-
-    # domain is the same for the whole leaf node
-    domain = leaf_node[0].get("domain")
-
-    for chunk in chunks:
-        # pylint: disable=consider-using-enumerate
-        for icl_ in leaf_node:
-            icl_query = {f"icl_query_{idx+1}": val["question"] for idx, val in enumerate(icl_["questions_and_answers"])}
-            icl_resp = {f"icl_response_{idx+1}": val["answer"] for idx, val in enumerate(icl_["questions_and_answers"])}
-            samples_row = {
-                "icl_document": icl_["input"],
-                "document": chunk,
-                "document_outline": icl_["document_outline"],
-                "domain": domain
-            }
-            samples_row.update(icl_query)
-            samples_row.update(icl_resp)
-            samples.append(samples_row)
-
-    return samples
-
-
-def _skill_leaf_node_to_samples(leaf_node):
-    samples = []
-
-    # pylint: disable=consider-using-enumerate
-    for i in range(len(leaf_node)):
-        samples.append({})
-        samples[-1]["task_description"] = leaf_node[i]["task_description"]
-        if leaf_node[i].get("input"):
-            samples[-1]["seed_context"] = leaf_node[i]["input"]
-        samples[-1]["seed_question"] = leaf_node[i]["instruction"]
-        samples[-1]["seed_response"] = leaf_node[i]["output"]
-
-    return samples
-
-
-def leaf_node_to_samples(leaf_node, server_ctx_size, chunk_word_count):
-    if not leaf_node:
-        return []
-    if leaf_node[0].get("document"):
-        return _knowledge_leaf_node_to_samples(
-            leaf_node, server_ctx_size, chunk_word_count
-        )
-    return _skill_leaf_node_to_samples(leaf_node)