gcn-python 1.0.0__py3-none-any.whl

gcn_python/taxonomy/loader.py

@@ -0,0 +1,74 @@
+"""
+TaxonomyIndex: charge les fichiers YAML de taxonomies GCN et expose
+l'appartenance par gcn_class_key = "taxonomy_name.class_name".
+
+Pas de règles ici — uniquement le chargement des données.
+"""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from pathlib import Path
+import yaml
+
+
+@dataclass
+class TaxonomyIndex:
+    """
+    Mappe chaque gcn_class_key vers un frozenset de lemmes.
+    Ex: "verbes.etat" -> {"être", "avoir", "savoir", ...}
+
+    Pour une langue sans fichier → ensemble vide → feature = 0.
+    Le modèle apprend le poids de chaque dimension.
+    """
+    data: dict[str, frozenset[str]] = field(default_factory=dict)
+
+    @classmethod
+    def load(cls, taxonomies_dir: Path, lang_code: str = "fr") -> "TaxonomyIndex":
+        """
+        Charge les taxonomies depuis:
+          1. taxonomies_dir/{lang_code}/  (spécifiques à la langue)
+          2. taxonomies_dir/              (partagées, fallback)
+        """
+        index: dict[str, frozenset[str]] = {}
+
+        dirs_to_scan = []
+        lang_dir = taxonomies_dir / lang_code
+        if lang_dir.is_dir():
+            dirs_to_scan.append(lang_dir)
+        if taxonomies_dir.is_dir():
+            dirs_to_scan.append(taxonomies_dir)
+
+        for scan_dir in dirs_to_scan:
+            for yaml_path in sorted(scan_dir.glob("*.yaml")):
+                try:
+                    doc = yaml.safe_load(yaml_path.read_text(encoding="utf-8"))
+                except Exception:
+                    continue
+                if not isinstance(doc, dict) or "taxonomy" not in doc:
+                    continue
+                tax_name = doc["taxonomy"]
+                classes = doc.get("classes") or {}
+                for class_name, class_data in classes.items():
+                    key = f"{tax_name}.{class_name}"
+                    if key in index:
+                        continue  # lang-specific already loaded
+                    lemmas: set[str] = set()
+                    examples = (class_data or {}).get("examples_fr") or []
+                    for entry in examples:
+                        if isinstance(entry, dict) and "lemma" in entry:
+                            lemmas.add(entry["lemma"].lower().strip())
+                        elif isinstance(entry, str):
+                            lemmas.add(entry.lower().strip())
+                    index[key] = frozenset(lemmas)
+
+        return cls(data=index)
+
+    def membership(self, lemma: str) -> dict[str, bool]:
+        """Retourne {gcn_class_key: True/False} pour un lemme."""
+        lower = lemma.lower().strip()
+        return {key: lower in lemmas for key, lemmas in self.data.items()}
+
+    def keys(self) -> list[str]:
+        return sorted(self.data.keys())
+
+    def __len__(self) -> int:
+        return len(self.data)

gcn_python/training/bootstrap.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import click
+
+
+@click.command("gcn-bootstrap")
+@click.option("--input", "input_file", required=True, type=click.Path(path_type=Path),
+              help="Fichier texte (.txt) — une phrase par ligne")
+@click.option("--lang", default="fr", show_default=True)
+@click.option("--out-dir", required=True, type=click.Path(path_type=Path),
+              help="Répertoire de sortie pour les fichiers JSON générés")
+@click.option("--taxonomy-dir", default=None, type=click.Path(path_type=Path),
+              envvar="GCN_TAXONOMY_DIR",
+              help="Répertoire des taxonomies (ou env GCN_TAXONOMY_DIR)")
+@click.option("--gcn-bin", default="gcn", show_default=True,
+              help="Chemin vers le binaire gcn-cli Rust")
+def bootstrap_cmd(
+    input_file: Path, lang: str, out_dir: Path,
+    taxonomy_dir: Path | None, gcn_bin: str,
+) -> None:
+    """Génère des données d'entraînement JSON depuis des phrases brutes via gcn-cli Rust.
+
+    Appelle `gcn analyze <texte>` pour chaque ligne, convertit le CausalIR JSON
+    produit au format gcn-nl (document.sentences), et écrit les fichiers dans out-dir.
+    """
+    out_dir.mkdir(parents=True, exist_ok=True)
+    texts = [l.strip() for l in input_file.read_text("utf-8").splitlines() if l.strip()]
+
+    if not texts:
+        raise click.ClickException(f"Aucune phrase dans {input_file}")
+
+    click.echo(f"Génération de {len(texts)} exemples vers {out_dir} ...")
+    success = 0
+    errors = 0
+
+    for i, text in enumerate(texts):
+        try:
+            cmd_args = [gcn_bin, "analyze", text]
+            if taxonomy_dir:
+                cmd_args += ["--data-dir", str(taxonomy_dir)]
+            result = subprocess.run(
+                cmd_args,
+                capture_output=True, text=True, timeout=30,
+            )
+            if result.returncode != 0:
+                click.echo(f"  [{i+1}] Erreur gcn-cli : {result.stderr.strip()}", err=True)
+                errors += 1
+                continue
+
+            cir = json.loads(result.stdout)
+            doc = _cir_to_doc(text, lang, cir)
+            out_path = out_dir / f"generated_{i+1:04d}.json"
+            out_path.write_text(
+                json.dumps(doc, ensure_ascii=False, indent=2),
+                encoding="utf-8",
+            )
+            success += 1
+        except subprocess.TimeoutExpired:
+            click.echo(f"  [{i+1}] Timeout", err=True)
+            errors += 1
+        except (json.JSONDecodeError, KeyError) as exc:
+            click.echo(f"  [{i+1}] Parse error: {exc}", err=True)
+            errors += 1
+
+    click.echo(f"Terminé : {success} succès, {errors} erreurs.")
+
+
+def _cir_to_doc(text: str, lang: str, cir: dict) -> dict:
+    """Convertit un CausalIR dict (format Rust/JSON) en document JSON gcn-nl."""
+    nodes = cir.get("nodes", [])
+    edges = cir.get("edges", [])
+
+    doc_nodes = [
+        {
+            "id": n.get("id", f"n{i+1:03d}"),
+            "type": n.get("node_type", "action"),
+            "label": n.get("label", ""),
+            "token_span": list(n.get("token_span", [0, 0])),
+            "scope": n.get("scope", "specific"),
+            "temporal_index": n.get("temporal_index", 0),
+            "origin": n.get("origin", "explicit"),
+        }
+        for i, n in enumerate(nodes)
+    ]
+
+    doc_edges = [
+        {
+            "source": e.get("source", ""),
+            "target": e.get("target", ""),
+            "relation": e.get("relation_type", e.get("relation", "cause")),
+            "confidence": float(e.get("confidence", 1.0)),
+            "explicit": bool(e.get("explicit", True)),
+            "negated": bool(e.get("negated", False)),
+        }
+        for e in edges
+    ]
+
+    return {
+        "document": {
+            "lang": lang,
+            "sentences": [
+                {
+                    "id": "s001",
+                    "text": text,
+                    "tokens": [],
+                    "cir": {
+                        "nodes": doc_nodes,
+                        "edges": doc_edges,
+                    },
+                }
+            ],
+        }
+    }

gcn_python/training/checkpoint.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+from pathlib import Path
+import numpy as np
+
+from ..pipeline.cgnp import CGNPipeline
+from ..layer1.features import FeatureVocabulary
+
+
+def save_checkpoint(pipeline: CGNPipeline, path: Path) -> None:
+    """Sérialise tous les poids du pipeline dans un fichier .npz."""
+    arrays: dict[str, np.ndarray] = {}
+
+    encoder_params = pipeline.encoder.parameters()
+    for i, p in enumerate(encoder_params):
+        arrays[f"encoder_{i}"] = p
+
+    graph_params = pipeline.graph.parameters()
+    for i, p in enumerate(graph_params):
+        arrays[f"graph_{i}"] = p
+
+    vocab_json = pipeline.vocabulary.to_json()
+    arrays["_vocab_json"] = np.array([vocab_json], dtype=object)
+
+    if pipeline.decoder is not None and hasattr(pipeline.decoder, 'parameters'):
+        decoder_params = pipeline.decoder.parameters()
+        for i, p in enumerate(decoder_params):
+            arrays[f"decoder_{i}"] = p
+        if hasattr(pipeline.decoder, 'to_json'):
+            arrays["_decoder_meta_json"] = np.array([pipeline.decoder.to_json()], dtype=object)
+
+    np.savez(path, **arrays)
+
+
+def load_checkpoint(pipeline: CGNPipeline, path: Path) -> None:
+    """Restaure les poids depuis un fichier .npz produit par save_checkpoint.
+
+    Atomique : toutes les shapes sont validées avant toute mutation du pipeline.
+    Un ValueError laisse le pipeline intact (vocabulary, encoder et graph inchangés).
+    """
+    data = np.load(path, allow_pickle=True)
+
+    new_vocab = None
+    if "_vocab_json" in data:
+        new_vocab = FeatureVocabulary.from_json(str(data["_vocab_json"][0]))
+
+    # Validation de toutes les formes avant toute mutation
+    encoder_params = pipeline.encoder.parameters()
+    for i, p in enumerate(encoder_params):
+        key = f"encoder_{i}"
+        if key in data and data[key].shape != p.shape:
+            raise ValueError(
+                f"Incompatibilité de dimension pour encoder_{i} : "
+                f"checkpoint={data[key].shape} ≠ pipeline={p.shape}. "
+                f"Reconstruisez le pipeline avec la même taxonomie que le checkpoint."
+            )
+
+    graph_params = pipeline.graph.parameters()
+    for i, p in enumerate(graph_params):
+        key = f"graph_{i}"
+        if key in data and data[key].shape != p.shape:
+            raise ValueError(
+                f"Incompatibilité de dimension pour graph_{i} : "
+                f"checkpoint={data[key].shape} ≠ pipeline={p.shape}."
+            )
+
+    # Toutes les formes validées — mutation sûre
+    if new_vocab is not None:
+        pipeline.vocabulary = new_vocab
+
+    for i, p in enumerate(encoder_params):
+        key = f"encoder_{i}"
+        if key in data:
+            p[:] = data[key]
+
+    for i, p in enumerate(graph_params):
+        key = f"graph_{i}"
+        if key in data:
+            p[:] = data[key]
+
+    if "_decoder_meta_json" in data:
+        from ..verbalizer.trainable import TrainableDecoder
+        decoder = TrainableDecoder.from_json(str(data["_decoder_meta_json"][0]))
+        decoder_params = decoder.parameters()
+        for i, p in enumerate(decoder_params):
+            key = f"decoder_{i}"
+            if key in data:
+                if data[key].shape != p.shape:
+                    raise ValueError(
+                        f"Incompatibilité de dimension pour decoder_{i} : "
+                        f"checkpoint={data[key].shape} ≠ decoder={p.shape}."
+                    )
+                p[:] = data[key]
+        pipeline.decoder = decoder

gcn_python/training/train.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+import csv
+import json
+import warnings
+from pathlib import Path
+
+import click
+import numpy as np
+
+from ..layer1.features import FeatureVocabulary
+from ..layer2.reference import MLPEncoder
+from ..layer3.reference import RGCNLayer
+from ..pipeline.cgnp import CGNPipeline
+from ..data.loader import GCNDataLoader, reps_from_sentence
+from ..constants import NODE_TYPES, RELATION_TYPES
+from ..evaluation.metrics import node_accuracy, node_macro_f1, edge_accuracy, edge_macro_f1
+from ..evaluation.recorder import TrainingRecorder
+from .checkpoint import save_checkpoint
+
+
+@click.command("gcn-train")
+@click.option("--data-dir", required=True, type=click.Path(path_type=Path),
+              help="Répertoire contenant les fichiers JSON d'entraînement")
+@click.option("--lang", default="fr", show_default=True)
+@click.option("--epochs", default=50, show_default=True, type=int)
+@click.option("--lr", default=0.001, show_default=True, type=float)
+@click.option("--output", default="model.npz", show_default=True,
+              type=click.Path(path_type=Path), help="Chemin du checkpoint de sortie")
+@click.option("--log-csv", default=None, type=click.Path(path_type=Path),
+              help="CSV des métriques par epoch (optionnel)")
+@click.option("--verbalize-dir", default=None, type=click.Path(path_type=Path),
+              help="Répertoire contenant les paires verbalize JSON (optionnel, active l'entraînement conjoint)")
+def train_cmd(
+    data_dir: Path,
+    lang: str,
+    epochs: int,
+    lr: float,
+    output: Path,
+    log_csv: Path | None,
+    verbalize_dir: Path | None,
+) -> None:
+    """Entraîne le pipeline CGNP (NumPy référence) par descente de gradient."""
+    from ..data.verbalize_loader import VerbalizerDataLoader
+    from ..verbalizer.trainable import TrainableDecoder
+
+    vocab = FeatureVocabulary()
+    encoder = MLPEncoder(d_clause=vocab.d_clause, d_edge=vocab.d_edge)
+    graph = RGCNLayer(d_in=vocab.d_clause, d_out=vocab.d_clause)
+
+    verb_loader: VerbalizerDataLoader | None = None
+    verb_source_map: dict[str, list] = {}
+    decoder: TrainableDecoder | None = None
+    if verbalize_dir is not None:
+        verb_loader = VerbalizerDataLoader(verbalize_dir)
+        if len(verb_loader) == 0:
+            raise click.ClickException(f"Aucune paire verbalize dans {verbalize_dir}")
+        decoder = TrainableDecoder(verb_loader.vocab)
+        verb_source_map = verb_loader.source_text_map()
+        click.echo(f"Verbalize : {len(verb_loader)} paires | vocab={len(verb_loader.vocab)} tokens")
+
+    pipeline = CGNPipeline(encoder=encoder, graph=graph, lang=lang, vocabulary=vocab,
+                           decoder=decoder)
+
+    loader = GCNDataLoader(data_dir, lang=lang)
+    if len(loader) == 0:
+        raise click.ClickException(f"Aucune sentence dans {data_dir}")
+
+    click.echo(f"Données : {len(loader)} sentences | epochs={epochs} lr={lr}")
+
+    recorder = TrainingRecorder()
+    history: list[dict] = []
+    csv_writer = None
+    csv_file = None
+    if log_csv:
+        csv_file = open(log_csv, "w", newline="", encoding="utf-8")
+        csv_writer = csv.DictWriter(
+            csv_file,
+            fieldnames=["epoch", "loss", "node_accuracy", "node_macro_f1",
+                        "edge_accuracy", "edge_macro_f1"],
+        )
+        csv_writer.writeheader()
+
+    try:
+        for epoch in range(1, epochs + 1):
+            epoch_loss = 0.0
+            n_samples = 0
+            epoch_node_preds: list[str] = []
+            epoch_node_gold: list[str] = []
+            epoch_edge_preds: list[str] = []
+            epoch_edge_gold: list[str] = []
+
+            for sample in loader:
+                if not sample.sentence.clauses:
+                    continue
+
+                try:
+                    reps, valid_clause_idxs, connector_reps = reps_from_sentence(sample.sentence)
+                    if not reps:
+                        continue
+                    pipeline.forward(
+                        reps, sample.sentence.text,
+                        clause_positions=valid_clause_idxs,
+                        n_total_clauses=len(sample.sentence.clauses),
+                        connector_reps=connector_reps,
+                    )
+                except ValueError:
+                    raise  # misconfiguration (d_out, clause_positions…) — non ignorable
+                except Exception as exc:
+                    warnings.warn(
+                        f"[{sample.sentence.id}] forward ignoré : "
+                        f"{type(exc).__name__}: {exc}",
+                        UserWarning, stacklevel=2,
+                    )
+                    continue
+
+                node_logits = pipeline._cached_node_logits
+                edge_logits = pipeline._cached_edge_logits
+                if node_logits is None or len(node_logits) == 0:
+                    continue
+
+                # Aligner les gold labels sur les seules clauses converties en reps
+                if valid_clause_idxs:
+                    gold_node = sample.gold_node_labels[
+                        np.array(valid_clause_idxs, dtype=np.int64)
+                    ]
+                else:
+                    gold_node = sample.gold_node_labels
+                # Aligner les gold edges sur les paires consécutives prédites via edge_map
+                if (valid_clause_idxs and len(valid_clause_idxs) >= 2
+                        and sample.edge_map
+                        and edge_logits is not None and len(edge_logits) > 0):
+                    pairs = [
+                        (valid_clause_idxs[k], valid_clause_idxs[k + 1])
+                        for k in range(len(valid_clause_idxs) - 1)
+                    ]
+                    gold_edge_full = np.array(
+                        [sample.edge_map.get(p, -1) for p in pairs], dtype=np.int64
+                    )
+                    valid_edge_mask = gold_edge_full >= 0
+                    if valid_edge_mask.any():
+                        valid_edge_idxs = np.where(valid_edge_mask)[0]
+                        gold_edge = gold_edge_full[valid_edge_idxs]
+                        edge_logits_arg = edge_logits[valid_edge_idxs]
+                        pipeline.filter_edge_cache(valid_edge_idxs)
+                    else:
+                        gold_edge = None
+                        edge_logits_arg = None
+                else:
+                    gold_edge = None
+                    edge_logits_arg = None
+
+                # Joint training : chercher une surface gold pour ce sample
+                _gold_surface = None
+                if verb_source_map:
+                    _surfaces = verb_source_map.get(sample.sentence.text, [])
+                    if _surfaces:
+                        _gold_surface = _surfaces[0]
+
+                loss_val, d_node, d_edge = pipeline.loss(
+                    node_logits, edge_logits_arg, gold_node, gold_edge,
+                    gold_surface=_gold_surface,
+                )
+
+                # Backward
+                pipeline.backward(d_node, d_edge, lr=lr)
+
+                # Accumuler les prédictions pour les métriques de l'époque
+                node_pred_idxs = np.argmax(node_logits, axis=1)
+                if valid_clause_idxs:
+                    gold_node_aligned = sample.gold_node_labels[
+                        np.array(valid_clause_idxs, dtype=np.int64)
+                    ]
+                else:
+                    gold_node_aligned = sample.gold_node_labels
+                epoch_node_preds.extend(NODE_TYPES[i] for i in node_pred_idxs)
+                epoch_node_gold.extend(NODE_TYPES[i] for i in gold_node_aligned)
+
+                if gold_edge is not None and edge_logits_arg is not None and len(edge_logits_arg) > 0:
+                    edge_pred_idxs = np.argmax(edge_logits_arg, axis=1)
+                    epoch_edge_preds.extend(RELATION_TYPES[i] for i in edge_pred_idxs)
+                    epoch_edge_gold.extend(RELATION_TYPES[i] for i in gold_edge)
+
+                epoch_loss += loss_val
+                n_samples += 1
+
+            # Entraînement standalone du décodeur sur les paires verbalize
+            if verb_loader is not None and pipeline.decoder is not None:
+                for vsample in verb_loader:
+                    if len(vsample.gold_tokens) == 0:
+                        continue
+                    dec_logits = pipeline.decoder.forward_decode(vsample.node_type_embeddings)
+                    dec_loss, d_dec = pipeline.decoder.loss_decode(dec_logits, vsample.gold_tokens)
+                    if not np.isfinite(dec_loss):
+                        continue
+                    _, dec_grads = pipeline.decoder.backward_decode(d_dec)
+                    pipeline.decoder.update(dec_grads, lr)
+                    epoch_loss += dec_loss
+                    n_samples += 1
+
+            avg_loss = epoch_loss / max(n_samples, 1)
+            metrics = {
+                "node_accuracy": node_accuracy(epoch_node_preds, epoch_node_gold),
+                "node_macro_f1": node_macro_f1(epoch_node_preds, epoch_node_gold),
+                "edge_accuracy": edge_accuracy(epoch_edge_preds, epoch_edge_gold),
+                "edge_macro_f1": edge_macro_f1(epoch_edge_preds, epoch_edge_gold),
+            }
+            recorder.record(epoch, avg_loss, metrics)
+            history.append({"epoch": epoch, "loss": avg_loss, **metrics})
+
+            if csv_writer:
+                csv_writer.writerow({"epoch": epoch, "loss": avg_loss, **metrics})
+
+            if epoch % max(1, epochs // 10) == 0 or epoch == 1:
+                click.echo(
+                    f"Epoch {epoch:4d}/{epochs}  loss={avg_loss:.4f}"
+                    f"  node_acc={metrics['node_accuracy']:.3f}"
+                    f"  edge_acc={metrics['edge_accuracy']:.3f}"
+                )
+    finally:
+        if csv_file:
+            csv_file.close()
+
+    save_checkpoint(pipeline, output)
+    click.echo(f"Checkpoint sauvegardé : {output}")
+
+    if log_csv:
+        json_path = Path(log_csv).with_suffix(".json")
+        recorder.to_json(json_path)
+        click.echo(f"Courbe d'entraînement : {json_path}")
+
+    # Vérification : la loss doit décroître sur les 10 dernières epochs
+    if len(history) >= 10:
+        first = sum(r["loss"] for r in history[:5]) / 5
+        last = sum(r["loss"] for r in history[-5:]) / 5
+        if last < first:
+            click.echo(f"Loss décroissante : {first:.4f} → {last:.4f}  ✓")
+        else:
+            click.echo(f"Avertissement : loss non décroissante ({first:.4f} → {last:.4f})")

gcn_python/verbalizer/__init__.py

@@ -0,0 +1,4 @@
+from .decoder import ReferenceDecoder
+from .interface import VerbalizerDecoder
+
+__all__ = ["ReferenceDecoder", "VerbalizerDecoder"]

gcn_python/verbalizer/cli.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+import sys
+import click
+from .decoder import ReferenceDecoder
+
+
+@click.command("gcn-verbalize")
+@click.argument("ir", type=click.File("r"), default="-")
+def verbalize_cmd(ir: click.File) -> None:
+    """
+    Decode a CausalIR JSON to a surface form.
+
+    IR: path to a CausalIR JSON file, or '-' to read from stdin.
+
+    What the decoder produces depends on its training data.
+    No explicit parameter needed.
+    """
+    ir_json = ir.read()
+    decoder = ReferenceDecoder()
+    try:
+        result = decoder.decode(ir_json)
+    except Exception as exc:
+        click.echo(f"error: {exc}", err=True)
+        sys.exit(1)
+    click.echo(result)

gcn_python/verbalizer/decoder.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+import json
+
+
+class ReferenceDecoder:
+    """
+    Reference verbalizer decoder — format-agnostic linearization placeholder.
+
+    Implements the VerbalizerDecoder protocol.
+    Replace with a trained model: the trained model learns what to produce
+    from its training data — no output format is presupposed here.
+
+    This reference produces a structural linearization of the CausalIR graph:
+    one fragment per edge, joined by newlines. It makes no assumption about
+    whether the output should be natural language, code, or any other form.
+    """
+
+    def decode(self, ir_json: str) -> str:
+        ir = json.loads(ir_json)
+        nodes: list[dict] = ir.get("nodes", [])
+        edges: list[list] = ir.get("edges", [])
+
+        node_map: dict[int, str] = {n["id"]: n["label"] for n in nodes}
+
+        fragments: list[str] = []
+        for edge_tuple in edges:
+            src_id, dst_id, edge = edge_tuple
+            src = node_map.get(src_id, str(src_id))
+            dst = node_map.get(dst_id, str(dst_id))
+            relation: str = edge.get("relation", "?")
+            negated: bool = edge.get("negated", False)
+            neg = "¬" if negated else ""
+            fragments.append(f"{src} -{neg}[{relation}]-> {dst}")
+
+        if not fragments:
+            return " | ".join(n["label"] for n in nodes)
+
+        return "\n".join(fragments)

gcn_python/verbalizer/interface.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class VerbalizerDecoder(Protocol):
+    """
+    Protocol for verbalizer decoders: CausalIR JSON → surface string.
+
+    The reference implementation is ReferenceDecoder (linearization placeholder).
+    Replace with a trained model: what the model produces depends on its training data.
+    """
+
+    def decode(self, ir_json: str) -> str:
+        """
+        Decode a CausalIR JSON string into a surface form.
+
+        What the decoder produces depends entirely on its training data.
+        No output format is presupposed by the architecture.
+        """
+        ...