flwr-nightly 1.11.0.dev20240824__py3-none-any.whl → 1.11.0.dev20240827__py3-none-any.whl

flwr/cli/app.py

@@ -18,7 +18,6 @@ import typer
 from typer.main import get_command
 
 from .build import build
-from .example import example
 from .install import install
 from .new import new
 from .run import run
@@ -33,7 +32,6 @@ app = typer.Typer(
 )
 
 app.command()(new)
-app.command()(example)
 app.command()(run)
 app.command()(build)
 app.command()(install)

flwr/cli/new/new.py

@@ -187,24 +187,20 @@ def new(
             "pyproject.toml": {"template": f"app/pyproject.{framework_str}.toml.tpl"},
             "README.md": {"template": f"app/README.{framework_str}.md.tpl"},
             f"{import_name}/__init__.py": {"template": "app/code/__init__.py.tpl"},
-            f"{import_name}/server.py": {
-                "template": "app/code/flwr_tune/server.py.tpl"
+            f"{import_name}/server_app.py": {
+                "template": "app/code/flwr_tune/server_app.py.tpl"
             },
-            f"{import_name}/client.py": {
-                "template": "app/code/flwr_tune/client.py.tpl"
+            f"{import_name}/client_app.py": {
+                "template": "app/code/flwr_tune/client_app.py.tpl"
             },
-            f"{import_name}/app.py": {"template": "app/code/flwr_tune/app.py.tpl"},
             f"{import_name}/models.py": {
                 "template": "app/code/flwr_tune/models.py.tpl"
             },
             f"{import_name}/dataset.py": {
                 "template": "app/code/flwr_tune/dataset.py.tpl"
             },
-            f"{import_name}/conf/config.yaml": {
-                "template": "app/code/flwr_tune/config.yaml.tpl"
-            },
-            f"{import_name}/conf/static_config.yaml": {
-                "template": "app/code/flwr_tune/static_config.yaml.tpl"
+            f"{import_name}/strategy.py": {
+                "template": "app/code/flwr_tune/strategy.py.tpl"
             },
         }
 

flwr/cli/new/templates/app/README.flowertune.md.tpl

@@ -23,10 +23,12 @@ pip install -e .
 
 ## Experimental setup
 
-The dataset is partitioned into $num_clients shards with IID fashion serving as clients.
-We randomly sample $fraction_fit clients to be available for each round,
-and the federated fine-tuning lasts for `200` rounds.
-All settings are defined in `$project_name/conf/static_config.yaml`, which is not allowed to be modified for fair competition if you plan to participated in the [LLM leaderboard](https://flower.ai/benchmarks/llm-leaderboard).
+The dataset is divided into $num_clients partitions in an IID fashion, a partition is assigned to each ClientApp.
+We randomly sample a fraction ($fraction_fit) of the total nodes to participate in each round, for a total of `200` rounds.
+All settings are defined in `pyproject.toml`.
+
+> [!IMPORTANT]
+> Please note that `[tool.flwr.app.config.static]` and `options.num-supernodes` under `[tool.flwr.federations.local-simulation]` are not allowed to be modified for fair competition if you plan to participated in the [LLM leaderboard](https://flower.ai/benchmarks/llm-leaderboard).
 
 
 ## Running the challenge
@@ -39,7 +41,7 @@ huggingface-cli login
 ```
 
 Run the challenge with default config values.
-The configs are in `$project_name/conf/config.yaml` and `$project_name/conf/static_config.yaml`, and are loaded automatically.
+The configs are defined in `[tool.flwr.app.config]` entry of `pyproject.toml`, and are loaded automatically.
 
 ```bash
 flwr run
@@ -53,4 +55,12 @@ We use Mistral-7B model with 4-bit quantization as default. The estimated VRAM c
 | :--------: | :--------: | :--------: | :--------: | :--------: |
 |    VRAM    | ~25.50 GB  | ~17.30 GB  | ~22.80 GB  | ~17.40 GB  |
 
-You can adjust the CPU/GPU resources you assign to each of the clients based on your device, which is specified with `flower.engine.simulation` in `pyproject.toml`.
+You can adjust the CPU/GPU resources you assign to each of the clients based on your device, which are specified with `options.backend.clientapp-cpus` and `options.backend.clientapp-gpus` under `[tool.flwr.federations.local-simulation]` entry in `pyproject.toml`.
+
+
+## Model saving
+
+The global PEFT model checkpoints are saved every 5 rounds after aggregation on the sever side as default, which can be specified with `train.save-every-round` under [tool.flwr.app.config] entry in `pyproject.toml`.
+
+> [!NOTE]
+> Please provide the last PEFT checkpoint if you plan to participated in the [LLM leaderboard](https://flower.ai/benchmarks/llm-leaderboard).

flwr/cli/new/templates/app/code/flwr_tune/{client.py.tpl → client_app.py.tpl}

@@ -1,20 +1,32 @@
 """$project_name: A Flower / FlowerTune app."""
 
+import os
+import warnings
 from collections import OrderedDict
-from typing import Callable, Dict, Tuple
+from typing import Dict, Tuple
 
 import torch
+from flwr.client import ClientApp, NumPyClient
+from flwr.common import Context
+from flwr.common.config import unflatten_dict
+from flwr.common.typing import NDArrays, Scalar
 from omegaconf import DictConfig
 from peft import get_peft_model_state_dict, set_peft_model_state_dict
 from transformers import TrainingArguments
 from trl import SFTTrainer
 
-from flwr.client import NumPyClient
-from flwr.common import Context
-from flwr.common.typing import NDArrays, Scalar
-from $import_name.dataset import reformat
+from $import_name.dataset import (
+    get_tokenizer_and_data_collator_and_propt_formatting,
+    load_data,
+    replace_keys,
+)
 from $import_name.models import cosine_annealing, get_model
 
+# Avoid warnings
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
+os.environ["RAY_DISABLE_DOCKER_CPU_WARNING"] = "1"
+warnings.filterwarnings("ignore", category=UserWarning)
+
 
 # pylint: disable=too-many-arguments
 # pylint: disable=too-many-instance-attributes
@@ -29,7 +41,7 @@ class FlowerClient(NumPyClient):
         tokenizer,
         formatting_prompts_func,
         data_collator,
-        save_path,
+        num_rounds,
     ):  # pylint: disable=too-many-arguments
         self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
         self.train_cfg = train_cfg
@@ -37,13 +49,12 @@ class FlowerClient(NumPyClient):
         self.tokenizer = tokenizer
         self.formatting_prompts_func = formatting_prompts_func
         self.data_collator = data_collator
-        self.save_path = save_path
+        self.num_rounds = num_rounds
+        self.trainset = trainset
 
         # instantiate model
         self.model = get_model(model_cfg)
 
-        self.trainset = trainset
-
     def fit(
         self, parameters: NDArrays, config: Dict[str, Scalar]
     ) -> Tuple[NDArrays, int, Dict]:
@@ -52,13 +63,13 @@ class FlowerClient(NumPyClient):
 
         new_lr = cosine_annealing(
             int(config["current_round"]),
-            self.train_cfg.num_rounds,
+            self.num_rounds,
             self.train_cfg.learning_rate_max,
             self.train_cfg.learning_rate_min,
         )
 
         self.training_argumnets.learning_rate = new_lr
-        self.training_argumnets.output_dir = self.save_path
+        self.training_argumnets.output_dir = config["save_path"]
 
         # Construct trainer
         trainer = SFTTrainer(
@@ -95,32 +106,31 @@ def get_parameters(model) -> NDArrays:
     return [val.cpu().numpy() for _, val in state_dict.items()]
 
 
-def gen_client_fn(
-    fds,
-    tokenizer,
-    formatting_prompts_func,
-    data_collator,
-    model_cfg: DictConfig,
-    train_cfg: DictConfig,
-    save_path: str,
-) -> Callable[[Context], FlowerClient]:  # pylint: disable=too-many-arguments
-    """Generate the client function that creates the Flower Clients."""
-
-    def client_fn(context: Context) -> FlowerClient:
-        """Create a Flower client representing a single organization."""
-        # Let's get the partition corresponding to the i-th client
-        partition_id = context.node_config["partition-id"]
-        client_trainset = fds.load_partition(partition_id, "train")
-        client_trainset = reformat(client_trainset, llm_task="$llm_challenge_str")
-
-        return FlowerClient(
-            model_cfg,
-            train_cfg,
-            client_trainset,
-            tokenizer,
-            formatting_prompts_func,
-            data_collator,
-            save_path,
-        ).to_client()
-
-    return client_fn
+def client_fn(context: Context) -> FlowerClient:
+    """Create a Flower client representing a single organization."""
+    partition_id = context.node_config["partition-id"]
+    num_partitions = context.node_config["num-partitions"]
+    num_rounds = context.run_config["num-server-rounds"]
+    cfg = DictConfig(replace_keys(unflatten_dict(context.run_config)))
+
+    # Let's get the client partition
+    client_trainset = load_data(partition_id, num_partitions, cfg.static.dataset.name)
+    (
+        tokenizer,
+        data_collator,
+        formatting_prompts_func,
+    ) = get_tokenizer_and_data_collator_and_propt_formatting(cfg.model.name)
+
+    return FlowerClient(
+        cfg.model,
+        cfg.train,
+        client_trainset,
+        tokenizer,
+        formatting_prompts_func,
+        data_collator,
+        num_rounds,
+    ).to_client()
+
+
+# Flower ClientApp
+app = ClientApp(client_fn)

flwr/cli/new/templates/app/code/flwr_tune/dataset.py.tpl

@@ -1,8 +1,12 @@
 """$project_name: A Flower / FlowerTune app."""
 
+from flwr_datasets import FederatedDataset
+from flwr_datasets.partitioner import IidPartitioner
 from transformers import AutoTokenizer
 from trl import DataCollatorForCompletionOnlyLM
 
+FDS = None  # Cache FederatedDataset
+
 
 def formatting_prompts_func(example):
     """Construct prompts."""
@@ -24,7 +28,6 @@ def formatting_prompts_func(example):
 
 def get_tokenizer_and_data_collator_and_propt_formatting(model_name: str):
     """Get tokenizer, data_collator and prompt formatting."""
-    # From: https://huggingface.co/docs/trl/en/sft_trainer
     tokenizer = AutoTokenizer.from_pretrained(
         model_name, use_fast=True, padding_side="right"
     )
@@ -49,9 +52,36 @@ def formatting(dataset):
 def reformat(dataset, llm_task):
     """Reformat datasets."""
     dataset = dataset.rename_column("output", "response")
-    if llm_task == "finance" or llm_task == "code":
+    if llm_task in ["finance", "code"]:
         dataset = dataset.map(formatting, remove_columns=["input"])
     if llm_task == "medical":
         dataset = dataset.remove_columns(["instruction"])
         dataset = dataset.rename_column("input", "instruction")
     return dataset
+
+
+def load_data(partition_id: int, num_partitions: int, dataset_name: str):
+    """Load partition data."""
+    # Only initialize `FederatedDataset` once
+    global FDS
+    if FDS is None:
+        partitioner = IidPartitioner(num_partitions=num_partitions)
+        FDS = FederatedDataset(
+            dataset=dataset_name,
+            partitioners={"train": partitioner},
+        )
+    client_trainset = FDS.load_partition(partition_id, "train")
+    client_trainset = reformat(client_trainset, llm_task="generalnlp")
+    return client_trainset
+
+
+def replace_keys(input_dict, match="-", target="_"):
+    """Recursively replace match string with target string in dictionary keys."""
+    new_dict = {}
+    for key, value in input_dict.items():
+        new_key = key.replace(match, target)
+        if isinstance(value, dict):
+            new_dict[new_key] = replace_keys(value, match, target)
+        else:
+            new_dict[new_key] = value
+    return new_dict

flwr/cli/new/templates/app/code/flwr_tune/models.py.tpl

@@ -22,9 +22,6 @@ def cosine_annealing(
 
 def get_model(model_cfg: DictConfig):
     """Load model with appropriate quantization config and other optimizations.
-
-    Please refer to this example for `peft + BitsAndBytes`:
-    https://github.com/huggingface/peft/blob/main/examples/fp4_finetuning/finetune_fp4_opt_bnb_peft.py
     """
     if model_cfg.quantization == 4:
         quantization_config = BitsAndBytesConfig(load_in_4bit=True)

flwr/cli/new/templates/app/code/flwr_tune/server_app.py.tpl

@@ -0,0 +1,95 @@
+"""$project_name: A Flower / FlowerTune app."""
+
+import os
+from datetime import datetime
+
+from flwr.common import Context, ndarrays_to_parameters
+from flwr.common.config import unflatten_dict
+from flwr.server import ServerApp, ServerAppComponents, ServerConfig
+from omegaconf import DictConfig
+
+from $import_name.client_app import get_parameters, set_parameters
+from $import_name.models import get_model
+from $import_name.dataset import replace_keys
+from $import_name.strategy import FlowerTuneLlm
+
+
+# Get function that will be executed by the strategy's evaluate() method
+# Here we use it to save global model checkpoints
+def get_evaluate_fn(model_cfg, save_every_round, total_round, save_path):
+    """Return an evaluation function for saving global model."""
+
+    def evaluate(server_round: int, parameters, config):
+        # Save model
+        if server_round != 0 and (
+            server_round == total_round or server_round % save_every_round == 0
+        ):
+            # Init model
+            model = get_model(model_cfg)
+            set_parameters(model, parameters)
+
+            model.save_pretrained(f"{save_path}/peft_{server_round}")
+
+        return 0.0, {}
+
+    return evaluate
+
+
+def get_on_fit_config(save_path):
+    """Return a function that will be used to construct the config that the
+    client's fit() method will receive."""
+
+    def fit_config_fn(server_round: int):
+        fit_config = {}
+        fit_config["current_round"] = server_round
+        fit_config["save_path"] = save_path
+        return fit_config
+
+    return fit_config_fn
+
+
+def fit_weighted_average(metrics):
+    """Aggregate (federated) evaluation metrics."""
+    # Multiply accuracy of each client by number of examples used
+    losses = [num_examples * m["train_loss"] for num_examples, m in metrics]
+    examples = [num_examples for num_examples, _ in metrics]
+
+    # Aggregate and return custom metric (weighted average)
+    return {"train_loss": sum(losses) / sum(examples)}
+
+
+def server_fn(context: Context):
+    """Construct components that set the ServerApp behaviour."""
+    # Create output directory given current timestamp
+    current_time = datetime.now()
+    folder_name = current_time.strftime("%Y-%m-%d_%H-%M-%S")
+    save_path = os.path.join(os.getcwd(), f"results/{folder_name}")
+    os.makedirs(save_path, exist_ok=True)
+
+    # Read from config
+    num_rounds = context.run_config["num-server-rounds"]
+    cfg = DictConfig(replace_keys(unflatten_dict(context.run_config)))
+
+    # Get initial model weights
+    init_model = get_model(cfg.model)
+    init_model_parameters = get_parameters(init_model)
+    init_model_parameters = ndarrays_to_parameters(init_model_parameters)
+
+    # Define strategy
+    strategy = FlowerTuneLlm(
+        fraction_fit=cfg.strategy.fraction_fit,
+        fraction_evaluate=cfg.strategy.fraction_evaluate,
+        on_fit_config_fn=get_on_fit_config(save_path),
+        fit_metrics_aggregation_fn=fit_weighted_average,
+        initial_parameters=init_model_parameters,
+        evaluate_fn=get_evaluate_fn(
+            cfg.model, cfg.train.save_every_round, num_rounds, save_path
+        ),
+    )
+    config = ServerConfig(num_rounds=num_rounds)
+
+    return ServerAppComponents(strategy=strategy, config=config)
+
+
+# Flower ServerApp
+app = ServerApp(server_fn=server_fn)

flwr/cli/new/templates/app/code/flwr_tune/strategy.py.tpl

@@ -0,0 +1,83 @@
+"""$project_name: A Flower / FlowerTune app."""
+
+from io import BytesIO
+from logging import INFO, WARN
+from typing import List, Tuple, Union
+
+from flwr.common import FitIns, FitRes, Parameters, log, parameters_to_ndarrays
+from flwr.server.client_manager import ClientManager
+from flwr.server.client_proxy import ClientProxy
+from flwr.server.strategy import FedAvg
+
+
+class FlowerTuneLlm(FedAvg):
+    """Customised FedAvg strategy implementation.
+    
+    This class behaves just like FedAvg but also tracks the communication
+    costs associated with `fit` over FL rounds.
+    """
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.comm_tracker = CommunicationTracker()
+
+    def configure_fit(
+        self, server_round: int, parameters: Parameters, client_manager: ClientManager
+    ):
+        """Configure the next round of training."""
+        return_clients = super().configure_fit(server_round, parameters, client_manager)
+
+        # Test communication costs
+        fit_ins_list = [fit_ins for _, fit_ins in return_clients]
+        self.comm_tracker.track(fit_ins_list)
+
+        return return_clients
+
+    def aggregate_fit(
+        self,
+        server_round: int,
+        results: List[Tuple[ClientProxy, FitRes]],
+        failures: List[Union[Tuple[ClientProxy, FitRes], BaseException]],
+    ):
+        """Aggregate fit results using weighted average."""
+        # Test communication costs
+        fit_res_list = [fit_res for _, fit_res in results]
+        self.comm_tracker.track(fit_res_list)
+
+        parameters_aggregated, metrics_aggregated = super().aggregate_fit(
+            server_round, results, failures
+        )
+
+        return parameters_aggregated, metrics_aggregated
+
+
+class CommunicationTracker:
+    """Communication costs tracker over FL rounds."""
+    def __init__(self):
+        self.curr_comm_cost = 0.0
+
+    @staticmethod
+    def _compute_bytes(parameters):
+        return sum([BytesIO(t).getbuffer().nbytes for t in parameters.tensors])
+
+    def track(self, fit_list: List[Union[FitIns, FitRes]]):
+        size_bytes_list = [
+            self._compute_bytes(fit_ele.parameters)
+            for fit_ele in fit_list
+        ]
+        comm_cost = sum(size_bytes_list) / 1024**2
+
+        self.curr_comm_cost += comm_cost
+        log(
+            INFO,
+            "Communication budget: used %.2f MB (+%.2f MB this round) / 200,000 MB",
+            self.curr_comm_cost,
+            comm_cost,
+        )
+
+        if self.curr_comm_cost > 2e5:
+            log(
+                WARN,
+                "The accumulated communication cost has exceeded 200,000 MB. "
+                "Please consider reducing it if you plan to participate "
+                "FlowerTune LLM Leaderboard.",
+            )

flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl

@@ -8,15 +8,15 @@ version = "1.0.0"
 description = ""
 license = "Apache-2.0"
 dependencies = [
-    "flwr[simulation]>=1.9.0,<2.0",
-    "flwr-datasets>=0.1.0,<1.0.0",
-    "hydra-core==1.3.2",
+    "flwr[simulation]>=1.10.0",
+    "flwr-datasets>=0.3.0",
     "trl==0.8.1",
     "bitsandbytes==0.43.0",
     "scipy==1.13.0",
     "peft==0.6.2",
     "transformers==4.39.3",
     "sentencepiece==0.2.0",
+    "omegaconf==2.3.0",
 ]
 
 [tool.hatch.build.targets.wheel]
@@ -26,14 +26,41 @@ packages = ["."]
 publisher = "$username"
 
 [tool.flwr.app.components]
-serverapp = "$import_name.app:server"
-clientapp = "$import_name.app:client"
+serverapp = "$import_name.server_app:app"
+clientapp = "$import_name.client_app:app"
 
 [tool.flwr.app.config]
-num-server-rounds = 3
+model.name = "mistralai/Mistral-7B-v0.3"
+model.quantization = 4
+model.gradient-checkpointing = true
+model.lora.peft-lora-r = 32
+model.lora.peft-lora-alpha = 64
+train.save-every-round = 5
+train.learning-rate-max = 5e-5
+train.learning-rate-min = 1e-6
+train.seq-length = 512
+train.training-arguments.output-dir = ""
+train.training-arguments.learning-rate = ""
+train.training-arguments.per-device-train-batch-size = 16
+train.training-arguments.gradient-accumulation-steps = 1
+train.training-arguments.logging-steps = 10
+train.training-arguments.num-train-epochs = 3
+train.training-arguments.max-steps = 10
+train.training-arguments.save-steps = 1000
+train.training-arguments.save-total-limit = 10
+train.training-arguments.gradient-checkpointing = true
+train.training-arguments.lr-scheduler-type = "constant"
+strategy.fraction-fit = $fraction_fit
+strategy.fraction-evaluate = 0.0
+num-server-rounds = 200
+
+[tool.flwr.app.config.static]
+dataset.name = "$dataset_name"
 
 [tool.flwr.federations]
 default = "local-simulation"
 
 [tool.flwr.federations.local-simulation]
-options.num-supernodes = 10
+options.num-supernodes = $num_clients
+options.backend.client-resources.num-cpus = 6
+options.backend.client-resources.num-gpus = 1.0

flwr/cli/run/run.py

@@ -124,14 +124,14 @@ def run(
 
 
 def _run_with_superexec(
-    app: Optional[Path],
+    app: Path,
     federation_config: Dict[str, Any],
     config_overrides: Optional[List[str]],
 ) -> None:
 
     insecure_str = federation_config.get("insecure")
     if root_certificates := federation_config.get("root-certificates"):
-        root_certificates_bytes = Path(root_certificates).read_bytes()
+        root_certificates_bytes = (app / root_certificates).read_bytes()
         if insecure := bool(insecure_str):
             typer.secho(
                 "❌ `root_certificates` were provided but the `insecure` parameter"

{flwr_nightly-1.11.0.dev20240824.dist-info → flwr_nightly-1.11.0.dev20240827.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: flwr-nightly
-Version: 1.11.0.dev20240824
+Version: 1.11.0.dev20240827
 Summary: Flower: A Friendly Federated Learning Framework
 Home-page: https://flower.ai
 License: Apache-2.0
@@ -195,7 +195,7 @@ Other [examples](https://github.com/adap/flower/tree/main/examples):
 - [PyTorch: From Centralized to Federated](https://github.com/adap/flower/tree/main/examples/pytorch-from-centralized-to-federated)
 - [Vertical FL](https://github.com/adap/flower/tree/main/examples/vertical-fl)
 - [Federated Finetuning of OpenAI's Whisper](https://github.com/adap/flower/tree/main/examples/whisper-federated-finetuning)
-- [Federated Finetuning of Large Language Model](https://github.com/adap/flower/tree/main/examples/llm-flowertune)
+- [Federated Finetuning of Large Language Model](https://github.com/adap/flower/tree/main/examples/flowertune-llm)
 - [Federated Finetuning of a Vision Transformer](https://github.com/adap/flower/tree/main/examples/flowertune-vit)
 - [Advanced Flower with TensorFlow/Keras](https://github.com/adap/flower/tree/main/examples/advanced-tensorflow)
 - [Advanced Flower with PyTorch](https://github.com/adap/flower/tree/main/examples/advanced-pytorch)

{flwr_nightly-1.11.0.dev20240824.dist-info → flwr_nightly-1.11.0.dev20240827.dist-info}/RECORD

@@ -1,15 +1,15 @@
 flwr/__init__.py,sha256=VmBWedrCxqmt4QvUHBLqyVEH6p7zaFMD_oCHerXHSVw,937
 flwr/cli/__init__.py,sha256=cZJVgozlkC6Ni2Hd_FAIrqefrkCGOV18fikToq-6iLw,720
-flwr/cli/app.py,sha256=FBcSrE35ll88VE11ib67qgsJe2GYDN25UswV9-cYcX8,1267
+flwr/cli/app.py,sha256=v66W3QJ226JUy9y51hRwftX53R3oEKl3oOl1Xx-IwgU,1215
 flwr/cli/build.py,sha256=YrzjnwP1A1WQnyPuxtGy4kylcPoj0lexaF99ST872VQ,5174
 flwr/cli/config_utils.py,sha256=mDGXbcIxG14UpkUplILBYUkSk5M1LeTzZYDGNx-pFpU,7540
 flwr/cli/example.py,sha256=1bGDYll3BXQY2kRqSN-oICqS5n1b9m0g0RvXTopXHl4,2215
 flwr/cli/install.py,sha256=tUncrbZYRbC9QEcWSeTER16plPEoU-ERP0-nMgWiSPo,7094
 flwr/cli/new/__init__.py,sha256=cQzK1WH4JP2awef1t2UQ2xjl1agVEz9rwutV18SWV1k,789
-flwr/cli/new/new.py,sha256=VNb31-NLedm-_OK_D0aed0QxHO-tVlXjnf9UWVhC_Jk,9612
+flwr/cli/new/new.py,sha256=Q4iNccbodwfZY2th-Ws83txm-VolI236dscOziDVX2c,9396
 flwr/cli/new/templates/__init__.py,sha256=4luU8RL-CK8JJCstQ_ON809W9bNTkY1l9zSaPKBkgwY,725
 flwr/cli/new/templates/app/.gitignore.tpl,sha256=XixnHdyeMB2vwkGtGnwHqoWpH-9WChdyG0GXe57duhc,3078
-flwr/cli/new/templates/app/README.flowertune.md.tpl,sha256=PqzkGm0g6Zy-vZK9_0EO3f_U6g1r69lGc4UL8kds5Q8,2696
+flwr/cli/new/templates/app/README.flowertune.md.tpl,sha256=lxr_RCGfiDy8QGcMVdjXsUXWM_gLf6cY7UQanGL_FFQ,3304
 flwr/cli/new/templates/app/README.md.tpl,sha256=t7w4YFZEcJOxAnuJmNPw5-fDdIJu7PfLd8gFJDiBwwo,436
 flwr/cli/new/templates/app/__init__.py,sha256=DU7QMY7IhMQyuwm_tja66xU0KXTWQFqzfTqwg-_NJdE,729
 flwr/cli/new/templates/app/code/__init__.py,sha256=EM6vfvgAILKPaPn7H1wMV1Wi01WyZCP_Eg6NxD6oWg8,736
@@ -22,13 +22,11 @@ flwr/cli/new/templates/app/code/client.pytorch.py.tpl,sha256=WczaR5avJUhfw2Grn2K
 flwr/cli/new/templates/app/code/client.sklearn.py.tpl,sha256=xW9cuKhybk5S8IeDZhbeb0DNegDIJGEYrzMKsxgc2GE,2978
 flwr/cli/new/templates/app/code/client.tensorflow.py.tpl,sha256=u3KKf7hC9xGqOIUJXYCHJ_jiIu3aVbsC8pxVxm4yN6I,1759
 flwr/cli/new/templates/app/code/flwr_tune/__init__.py,sha256=JgNgBtKdm1jKM9625WxappCAVUGtYAmcjKSsXJ1u3ZQ,748
-flwr/cli/new/templates/app/code/flwr_tune/app.py.tpl,sha256=LOtmEYYayMCpF4rJRdo059n5w_cZRmzyuEkCN2qpTyQ,2826
-flwr/cli/new/templates/app/code/flwr_tune/client.py.tpl,sha256=3s41XYHRvmcLhyA60tZyGCNK2wxo2shl6mwmPfZANlA,4155
-flwr/cli/new/templates/app/code/flwr_tune/config.yaml.tpl,sha256=aPjrwFU020tPkJmOjzSUjb9dSi2bhs4ZTMEyd0uRlCA,867
-flwr/cli/new/templates/app/code/flwr_tune/dataset.py.tpl,sha256=kPG4AIXQfNNHZGYC3amet3ttI23N72N6jjoDkp_wYIA,2028
-flwr/cli/new/templates/app/code/flwr_tune/models.py.tpl,sha256=cEq9ZWM3zImJVceNtxHC_bYBLE8OChK0BdjpWs5Wz-0,1881
-flwr/cli/new/templates/app/code/flwr_tune/server.py.tpl,sha256=tz4hgAGV6pn5cOFW10ELRNRsUzLBSssHxrH_gSs_jtk,1552
-flwr/cli/new/templates/app/code/flwr_tune/static_config.yaml.tpl,sha256=cBPpBVN_N7p4T2a3rqChlngmE0dB_jveOLHesNcEHvs,268
+flwr/cli/new/templates/app/code/flwr_tune/client_app.py.tpl,sha256=DbotzaXzLDwplVBkJLOe5Lt5b6Yutwv9rJ69oVwyrvU,4397
+flwr/cli/new/templates/app/code/flwr_tune/dataset.py.tpl,sha256=iAujo8WubDGrz0gg_6zl-TUvkIbNRJM-VJmwKJ9tGY8,3051
+flwr/cli/new/templates/app/code/flwr_tune/models.py.tpl,sha256=UCLEKUpXarkz9tMFtDrxmLv6QuKe5zCimTuoopQedUM,1717
+flwr/cli/new/templates/app/code/flwr_tune/server_app.py.tpl,sha256=yMVcbfGkTPV9AV16bVdi5fTX1a6jmtszTUrvLXSosio,3305
+flwr/cli/new/templates/app/code/flwr_tune/strategy.py.tpl,sha256=BhiqRg9w1MGuU5h2_vrLhRc0oHItYzE69qX_JI411k8,2754
 flwr/cli/new/templates/app/code/server.huggingface.py.tpl,sha256=etpjLvGu6pVXzYQBKZp4tTbD3zm461qFo24NliKo74U,591
 flwr/cli/new/templates/app/code/server.jax.py.tpl,sha256=pIdUH-LgWRAGWQYLlivMNf8XnDSNDe2cCuRjlxbRzys,529
 flwr/cli/new/templates/app/code/server.mlx.py.tpl,sha256=RqiZ0k468SOlm9dcPr-fvA8xcWv4zwDCbJfBwL7P9Us,529
@@ -41,7 +39,7 @@ flwr/cli/new/templates/app/code/task.jax.py.tpl,sha256=F05eg149c9icRyVNdfcLyZvAX
 flwr/cli/new/templates/app/code/task.mlx.py.tpl,sha256=jWtCULLRr_9bCIJvoTLMx037-SDl_LF8udtA1UGoXDk,2946
 flwr/cli/new/templates/app/code/task.pytorch.py.tpl,sha256=NgbPix74X1t3ybaGjqdls30vF1i5oY3L7EQExhWhN74,3812
 flwr/cli/new/templates/app/code/task.tensorflow.py.tpl,sha256=SKXAZdgBnPpbAbJ90Rb7oQ5ilnopBx_j_JNFoUDeEAI,1732
-flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl,sha256=chTKewidM-st9Uk5o95tGOqJ5FSa1L2R0UdSW2wTr0c,789
+flwr/cli/new/templates/app/pyproject.flowertune.toml.tpl,sha256=pogRZLrwSfN_XH4NxDdMkhMh1O_7DP90VOoP-cP0HvI,1827
 flwr/cli/new/templates/app/pyproject.huggingface.toml.tpl,sha256=nD0rRUyr_Cj0TaSH8PsiaMhCwu_BuOVX4oqWfFSvOcE,765
 flwr/cli/new/templates/app/pyproject.jax.toml.tpl,sha256=Tq6jeGcoOKzMwWWYxMVnzMcipLURHLiW69iYlD1ywMg,659
 flwr/cli/new/templates/app/pyproject.mlx.toml.tpl,sha256=SHwYAA2qgIlOAU3Sb9BKSZcZ7O9biACg27MHexXUtDw,741
@@ -50,7 +48,7 @@ flwr/cli/new/templates/app/pyproject.pytorch.toml.tpl,sha256=vIO1ArukTC76ogYLNmJ
 flwr/cli/new/templates/app/pyproject.sklearn.toml.tpl,sha256=jk_5teoyOVM9QdBea8J-nk10S6TKw81QZiiKB54ATF0,654
 flwr/cli/new/templates/app/pyproject.tensorflow.toml.tpl,sha256=bRIvPCPvTTI4Eo5b61Rmw8WdDw3sjcohciTXgULN5l8,702
 flwr/cli/run/__init__.py,sha256=oCd6HmQDx-sqver1gecgx-uMA38BLTSiiKpl7RGNceg,789
-flwr/cli/run/run.py,sha256=8JziIz-NITpCVcGj1OgOdyjxa80SyifV0a_jElXaCU0,7987
+flwr/cli/run/run.py,sha256=RI6MgLBNYxmacjQg8XMAQ7VKxbV0DkRyJTfe4GsDFuw,7979
 flwr/cli/utils.py,sha256=l65Ul0YsSBPuypk0uorAtEDmLEYiUrzpCXi6zCg9mJ4,4506
 flwr/client/__init__.py,sha256=wzJZsYJIHf_8-PMzvfbinyzzjgh1UP1vLrAw2_yEbKI,1345
 flwr/client/app.py,sha256=o_2bhmlBeZATtWnAPZhL-Q1Ly0QZxc9ou4i7t0HKumE,31956
@@ -286,8 +284,8 @@ flwr/superexec/exec_grpc.py,sha256=PhqGoZEpTMxSQmUSV8Wgtzb1Za_pHJ-adZqo5RYnDyE,1
 flwr/superexec/exec_servicer.py,sha256=jl0aKVjm0PLQABcTL5c3jdSIzb0Z6hpVOtrAn4Ob7ts,2323
 flwr/superexec/executor.py,sha256=k_adivto6R2U82DADOHNvdtobehBYreRek1gOEBIQnQ,2318
 flwr/superexec/simulation.py,sha256=J6pw-RqCSiUed8I_3MasZH4tl57ZmDebPAHNnbb0-vE,7420
-flwr_nightly-1.11.0.dev20240824.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
-flwr_nightly-1.11.0.dev20240824.dist-info/METADATA,sha256=c4bYJYoDKPDrknqbeXipSxwNQYCoRjWHlQo_cvQXqvI,15703
-flwr_nightly-1.11.0.dev20240824.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
-flwr_nightly-1.11.0.dev20240824.dist-info/entry_points.txt,sha256=3cDQVJEBRCSLzJrVYAgjXpoCjuQ74I3A9NZ61DOHdVo,388
-flwr_nightly-1.11.0.dev20240824.dist-info/RECORD,,
+flwr_nightly-1.11.0.dev20240827.dist-info/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
+flwr_nightly-1.11.0.dev20240827.dist-info/METADATA,sha256=EzNACXTNbj22gGXrQbuOE8JeQfD7NxYOHb-FGCLOBTQ,15703
+flwr_nightly-1.11.0.dev20240827.dist-info/WHEEL,sha256=FMvqSimYX_P7y0a7UY-_Mc83r5zkBZsCYPm7Lr0Bsq4,88
+flwr_nightly-1.11.0.dev20240827.dist-info/entry_points.txt,sha256=3cDQVJEBRCSLzJrVYAgjXpoCjuQ74I3A9NZ61DOHdVo,388
+flwr_nightly-1.11.0.dev20240827.dist-info/RECORD,,

flwr/cli/new/templates/app/code/flwr_tune/app.py.tpl

@@ -1,89 +0,0 @@
-"""$project_name: A Flower / FlowerTune app."""
-
-import os
-import warnings
-from datetime import datetime
-
-from flwr_datasets import FederatedDataset
-from hydra import compose, initialize
-from hydra.utils import instantiate
-
-from flwr.client import ClientApp
-from flwr.common import Context, ndarrays_to_parameters
-from flwr.server import ServerApp, ServerAppComponents, ServerConfig
-
-from $import_name.client_app import gen_client_fn, get_parameters
-from $import_name.dataset import get_tokenizer_and_data_collator_and_propt_formatting
-from $import_name.models import get_model
-from $import_name.server_app import fit_weighted_average, get_evaluate_fn, get_on_fit_config
-
-# Avoid warnings
-warnings.filterwarnings("ignore", category=UserWarning)
-os.environ["TOKENIZERS_PARALLELISM"] = "true"
-os.environ["RAY_DISABLE_DOCKER_CPU_WARNING"] = "1"
-
-# Initialise regular config
-with initialize(config_path="conf", version_base="1.1"):
-    cfg = compose(config_name="config")
-
-# Initialise static config
-with initialize(config_path="conf", version_base="1.1"):
-    cfg_static = compose(config_name="static_config")
-
-cfg.train.num_rounds = cfg_static.num_rounds
-
-# Create output directory given current timestamp
-current_time = datetime.now()
-folder_name = current_time.strftime("%Y-%m-%d_%H-%M-%S")
-save_path = os.path.join(os.getcwd(), f"results/{folder_name}")
-os.makedirs(save_path, exist_ok=True)
-
-# Partition dataset and get dataloaders
-partitioner = instantiate(cfg_static.partitioner)
-fds = FederatedDataset(
-    dataset=cfg_static.dataset.name, partitioners={"train": partitioner}
-)
-(
-    tokenizer,
-    data_collator,
-    formatting_prompts_func,
-) = get_tokenizer_and_data_collator_and_propt_formatting(cfg.model.name)
-
-# ClientApp for Flower Next
-client = ClientApp(
-    client_fn=gen_client_fn(
-        fds,
-        tokenizer,
-        formatting_prompts_func,
-        data_collator,
-        cfg.model,
-        cfg.train,
-        save_path,
-    ),
-)
-
-# Get initial model weights
-init_model = get_model(cfg.model)
-init_model_parameters = get_parameters(init_model)
-init_model_parameters = ndarrays_to_parameters(init_model_parameters)
-
-def server_fn(context: Context):
-    # Instantiate strategy according to config. Here we pass other arguments
-    # that are only defined at runtime.
-    strategy = instantiate(
-        cfg.strategy,
-        on_fit_config_fn=get_on_fit_config(),
-        fit_metrics_aggregation_fn=fit_weighted_average,
-        initial_parameters=init_model_parameters,
-        evaluate_fn=get_evaluate_fn(
-            cfg.model, cfg.train.save_every_round, cfg_static.num_rounds, save_path
-        ),
-    )
-
-    config = ServerConfig(num_rounds=cfg_static.num_rounds)
-
-    return ServerAppComponents(strategy=strategy, config=config)
-
-
-# ServerApp for Flower Next
-server = ServerApp(server_fn=server_fn)

flwr/cli/new/templates/app/code/flwr_tune/config.yaml.tpl

@@ -1,34 +0,0 @@
-# Federated Instruction Tuning
----
-model:
-  name: "mistralai/Mistral-7B-v0.3"
-  quantization: 4 # 8 or 4 if you want to do quantization with BitsAndBytes
-  gradient_checkpointing: True
-  lora:
-    peft_lora_r: 32
-    peft_lora_alpha: 64
-
-train:
-  num_rounds: null
-  save_every_round: 5
-  learning_rate_max: 5e-5
-  learning_rate_min: 1e-6
-  seq_length: 512
-  training_arguments:
-    output_dir: null # to be set by hydra
-    learning_rate: null # to be set by the client
-    per_device_train_batch_size: 16
-    gradient_accumulation_steps: 1
-    logging_steps: 10
-    num_train_epochs: 3
-    max_steps: 10
-    report_to: null
-    save_steps: 1000
-    save_total_limit: 10
-    gradient_checkpointing: True
-    lr_scheduler_type: "constant"
-
-strategy:
-  _target_: flwr.server.strategy.FedAvg
-  fraction_fit: $fraction_fit
-  fraction_evaluate: 0.0 # no client evaluation

flwr/cli/new/templates/app/code/flwr_tune/server.py.tpl

@@ -1,48 +0,0 @@
-"""$project_name: A Flower / FlowerTune app."""
-
-from $import_name.client_app import set_parameters
-from $import_name.models import get_model
-
-
-# Get function that will be executed by the strategy's evaluate() method
-# Here we use it to save global model checkpoints
-def get_evaluate_fn(model_cfg, save_every_round, total_round, save_path):
-    """Return an evaluation function for saving global model."""
-
-    def evaluate(server_round: int, parameters, config):
-        # Save model
-        if server_round != 0 and (
-            server_round == total_round or server_round % save_every_round == 0
-        ):
-            # Init model
-            model = get_model(model_cfg)
-            set_parameters(model, parameters)
-
-            model.save_pretrained(f"{save_path}/peft_{server_round}")
-
-        return 0.0, {}
-
-    return evaluate
-
-
-def get_on_fit_config():
-    """
-    Return a function that will be used to construct the config
-    that the client's fit() method will receive.
-    """
-
-    def fit_config_fn(server_round: int):
-        fit_config = {"current_round": server_round}
-        return fit_config
-
-    return fit_config_fn
-
-
-def fit_weighted_average(metrics):
-    """Aggregate (federated) evaluation metrics."""
-    # Multiply accuracy of each client by number of examples used
-    losses = [num_examples * m["train_loss"] for num_examples, m in metrics]
-    examples = [num_examples for num_examples, _ in metrics]
-
-    # Aggregate and return custom metric (weighted average)
-    return {"train_loss": sum(losses) / sum(examples)}

flwr/cli/new/templates/app/code/flwr_tune/static_config.yaml.tpl

@@ -1,11 +0,0 @@
-# Federated Instruction Tuning (static)
----
-dataset:
-  name: $dataset_name
-
-# FL experimental settings
-num_clients: $num_clients # total number of clients
-num_rounds: 200
-partitioner:
-  _target_: flwr_datasets.partitioner.IidPartitioner
-  num_partitions: $num_clients