gcn-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- gcn_python/__init__.py +2 -0
- gcn_python/constants.py +28 -0
- gcn_python/data/__init__.py +0 -0
- gcn_python/data/json_reader.py +124 -0
- gcn_python/data/loader.py +212 -0
- gcn_python/data/schema.py +50 -0
- gcn_python/data/verbalize_loader.py +96 -0
- gcn_python/evaluation/__init__.py +0 -0
- gcn_python/evaluation/eval_runner.py +123 -0
- gcn_python/evaluation/metrics.py +257 -0
- gcn_python/evaluation/recorder.py +158 -0
- gcn_python/layer1/__init__.py +0 -0
- gcn_python/layer1/features.py +126 -0
- gcn_python/layer1/representation.py +37 -0
- gcn_python/layer2/__init__.py +0 -0
- gcn_python/layer2/interface.py +52 -0
- gcn_python/layer2/reference.py +159 -0
- gcn_python/layer3/__init__.py +0 -0
- gcn_python/layer3/interface.py +32 -0
- gcn_python/layer3/pytorch_rgcn.py +190 -0
- gcn_python/layer3/reference.py +102 -0
- gcn_python/pipeline/__init__.py +0 -0
- gcn_python/pipeline/cgnp.py +419 -0
- gcn_python/pipeline/cli.py +77 -0
- gcn_python/pipeline/ir_emitter.py +71 -0
- gcn_python/pipeline/label_builder.py +82 -0
- gcn_python/taxonomy/__init__.py +0 -0
- gcn_python/taxonomy/loader.py +74 -0
- gcn_python/training/__init__.py +0 -0
- gcn_python/training/bootstrap.py +117 -0
- gcn_python/training/checkpoint.py +93 -0
- gcn_python/training/train.py +238 -0
- gcn_python/verbalizer/__init__.py +4 -0
- gcn_python/verbalizer/cli.py +25 -0
- gcn_python/verbalizer/decoder.py +38 -0
- gcn_python/verbalizer/interface.py +21 -0
- gcn_python/verbalizer/trainable.py +204 -0
- gcn_python-1.0.0.dist-info/METADATA +12 -0
- gcn_python-1.0.0.dist-info/RECORD +41 -0
- gcn_python-1.0.0.dist-info/WHEEL +4 -0
- gcn_python-1.0.0.dist-info/entry_points.txt +6 -0
gcn_python/__init__.py
ADDED
gcn_python/constants.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Sync avec gcn-core/crates/gcn-ir/src/{ir,node,edge}.rs
|
|
2
|
+
NODE_TYPES = ["etat", "action", "transition", "processus", "condition", "entite", "etat_systemique"]
|
|
3
|
+
RELATION_TYPES = ["cause", "enable", "prevent", "condition", "concession", "sequence",
|
|
4
|
+
"motivation", "filter", "opposition", "data_dependency", "control_dependency"]
|
|
5
|
+
SCOPE_VALUES = ["universal", "existential", "partial", "null", "specific", "unknown"]
|
|
6
|
+
NODE_ORIGIN_VALUES = ["explicit", "inferred", "hypothetical"]
|
|
7
|
+
AGENT_TYPE_VALUES = ["human", "collective", "institutional", "natural"]
|
|
8
|
+
|
|
9
|
+
# Universal Dependencies constants
|
|
10
|
+
UPOS_TAGS = ["ADJ", "ADP", "ADV", "AUX", "CCONJ", "DET", "INTJ", "NOUN", "NUM",
|
|
11
|
+
"PART", "PRON", "PROPN", "PUNCT", "SCONJ", "SYM", "VERB", "X", "_unk"] # 18 + _unk = 19
|
|
12
|
+
|
|
13
|
+
UD_DEP_RELS = ["acl", "advcl", "advmod", "amod", "appos", "aux", "case", "cc", "ccomp", "clf",
|
|
14
|
+
"compound", "conj", "cop", "csubj", "dep", "det", "discourse", "dislocated",
|
|
15
|
+
"expl", "fixed", "flat", "goeswith", "iobj", "list", "mark", "nmod", "nsubj",
|
|
16
|
+
"nummod", "obj", "obl", "orphan", "parataxis", "punct", "reparandum", "root",
|
|
17
|
+
"vocative", "xcomp", "_unk"] # 37 + _unk = 38
|
|
18
|
+
|
|
19
|
+
UD_TENSE_VALUES = ["Pres", "Past", "Fut", "Imp", "_absent"] # 5
|
|
20
|
+
UD_ASPECT_VALUES = ["Perf", "Imp", "Prog", "_absent"] # 4
|
|
21
|
+
UD_MOOD_VALUES = ["Ind", "Sub", "Cond", "Imp", "_absent"] # 5
|
|
22
|
+
UD_POLARITY_VALUES = ["Neg"] # 1 binaire
|
|
23
|
+
|
|
24
|
+
# D_clause = 19 + 38 + 5(subject_pos) + 5 + 4 + 5 + 1 + 3(flags) + N_taxonomy
|
|
25
|
+
# D_conn = 19(marker_upos) + N_taxonomy + 2(position)
|
|
26
|
+
|
|
27
|
+
# Subject POS categories
|
|
28
|
+
SUBJECT_POS_CATS = ["PRON", "NOUN", "PROPN", "_other", "_absent"] # 5
|
|
File without changes
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
import json
|
|
4
|
+
import warnings
|
|
5
|
+
from .schema import SentenceRecord, TokenRecord, ClauseRecord, EdgeRecord
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def load_sentences(path: Path, lang: str = "fr") -> list[SentenceRecord]:
|
|
9
|
+
"""Charge un fichier JSON GCN-NL → List[SentenceRecord]."""
|
|
10
|
+
doc = json.loads(path.read_text(encoding="utf-8"))
|
|
11
|
+
if not isinstance(doc, dict):
|
|
12
|
+
return []
|
|
13
|
+
|
|
14
|
+
# Format paper_examples.json
|
|
15
|
+
if "examples" in doc:
|
|
16
|
+
return [_parse_paper_example(ex, lang) for ex in doc["examples"] if "expected_cir" in ex]
|
|
17
|
+
|
|
18
|
+
# Format dataset (document.sentences)
|
|
19
|
+
if "document" in doc:
|
|
20
|
+
doc_lang = doc["document"].get("lang", lang)
|
|
21
|
+
sentences = doc["document"].get("sentences") or []
|
|
22
|
+
return [_parse_dataset_sentence(s, doc_lang) for s in sentences if "cir" in s]
|
|
23
|
+
|
|
24
|
+
return []
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def load_all_sentences(data_dir: Path, lang: str = "fr") -> list[SentenceRecord]:
|
|
28
|
+
"""Charge tous les fichiers JSON d'un répertoire."""
|
|
29
|
+
records = []
|
|
30
|
+
for p in sorted(data_dir.glob("*.json")):
|
|
31
|
+
records.extend(load_sentences(p, lang))
|
|
32
|
+
return records
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _parse_paper_example(ex: dict, lang: str) -> SentenceRecord:
|
|
36
|
+
cir = ex.get("expected_cir", {})
|
|
37
|
+
clauses = [_parse_clause_node(n) for n in cir.get("nodes", [])]
|
|
38
|
+
edges = [_parse_edge(e) for e in cir.get("edges", [])]
|
|
39
|
+
return SentenceRecord(
|
|
40
|
+
id=ex.get("id", ""),
|
|
41
|
+
text=ex.get("text", ""),
|
|
42
|
+
lang=lang,
|
|
43
|
+
tokens=[],
|
|
44
|
+
clauses=clauses,
|
|
45
|
+
edges=edges,
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _parse_dataset_sentence(s: dict, lang: str) -> SentenceRecord:
|
|
50
|
+
tokens = [_parse_token(t) for t in s.get("tokens", [])]
|
|
51
|
+
cir = s.get("cir", {})
|
|
52
|
+
clauses = [_parse_clause_node(n) for n in cir.get("nodes", [])]
|
|
53
|
+
edges = [_parse_edge(e) for e in cir.get("edges", [])]
|
|
54
|
+
return SentenceRecord(
|
|
55
|
+
id=s.get("id", ""),
|
|
56
|
+
text=s.get("text", ""),
|
|
57
|
+
lang=lang,
|
|
58
|
+
tokens=tokens,
|
|
59
|
+
clauses=clauses,
|
|
60
|
+
edges=edges,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _parse_token(t: dict) -> TokenRecord:
|
|
65
|
+
gcn = t.get("gcn", {}) or {}
|
|
66
|
+
morph_raw = t.get("morph") or {}
|
|
67
|
+
return TokenRecord(
|
|
68
|
+
id=int(t.get("id", 0)),
|
|
69
|
+
form=t.get("form", ""),
|
|
70
|
+
lemma=t.get("lemma", ""),
|
|
71
|
+
pos=t.get("pos", ""),
|
|
72
|
+
dep_rel=t.get("dep_rel", ""),
|
|
73
|
+
dep_head=int(t.get("dep_head", 0)),
|
|
74
|
+
morph=morph_raw if isinstance(morph_raw, dict) else {},
|
|
75
|
+
gcn_causal_type=gcn.get("causal_type"),
|
|
76
|
+
gcn_causal_class=gcn.get("causal_class"),
|
|
77
|
+
)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _parse_clause_node(n: dict) -> ClauseRecord:
|
|
81
|
+
span = n.get("token_span", [0, 0])
|
|
82
|
+
attrs = dict(n.get("attributes", {}) or {})
|
|
83
|
+
for f in ("entity", "quality", "agent", "patient", "agent_type", "temporal_index", "scope"):
|
|
84
|
+
if f in n and f not in attrs:
|
|
85
|
+
attrs[f] = n[f]
|
|
86
|
+
node_type = n.get("type") or ""
|
|
87
|
+
if not node_type:
|
|
88
|
+
warnings.warn(
|
|
89
|
+
f"Nœud {n.get('id', '?')} sans champ 'type' — défaut 'action' appliqué.",
|
|
90
|
+
UserWarning, stacklevel=3,
|
|
91
|
+
)
|
|
92
|
+
node_type = "action"
|
|
93
|
+
return ClauseRecord(
|
|
94
|
+
node_id=n.get("id", ""),
|
|
95
|
+
node_type=node_type,
|
|
96
|
+
label=n.get("label", ""),
|
|
97
|
+
token_span=(span[0], span[1]) if len(span) >= 2 else (0, 0),
|
|
98
|
+
scope=n.get("scope", attrs.get("scope", "specific")),
|
|
99
|
+
temporal_index=int(n.get("temporal_index", attrs.get("temporal_index", 0))),
|
|
100
|
+
origin=n.get("origin", "explicit"),
|
|
101
|
+
attributes=attrs,
|
|
102
|
+
modifiers=n.get("modifiers", []) or [],
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _parse_edge(e: dict) -> EdgeRecord:
|
|
107
|
+
attrs = e.get("attributes") or {}
|
|
108
|
+
relation = e.get("relation") or ""
|
|
109
|
+
if not relation:
|
|
110
|
+
warnings.warn(
|
|
111
|
+
f"Arête {e.get('source', '?')}→{e.get('target', '?')} sans champ 'relation' "
|
|
112
|
+
f"— défaut 'cause' appliqué.",
|
|
113
|
+
UserWarning, stacklevel=3,
|
|
114
|
+
)
|
|
115
|
+
relation = "cause"
|
|
116
|
+
return EdgeRecord(
|
|
117
|
+
source=e.get("source", ""),
|
|
118
|
+
target=e.get("target", ""),
|
|
119
|
+
relation=relation,
|
|
120
|
+
confidence=float(attrs.get("confidence", e.get("confidence", 1.0))),
|
|
121
|
+
explicit=bool(attrs.get("explicit", e.get("explicit", True))),
|
|
122
|
+
negated=bool(attrs.get("negated", e.get("negated", False))),
|
|
123
|
+
marker_token=attrs.get("marker_token", e.get("marker_token")),
|
|
124
|
+
)
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import warnings
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from .json_reader import load_all_sentences
|
|
8
|
+
from .schema import SentenceRecord, TokenRecord, ClauseRecord
|
|
9
|
+
from ..constants import NODE_TYPES, RELATION_TYPES
|
|
10
|
+
from ..layer1.representation import UDRepresentation
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _node_type_idx(node_type: str, sentence_id: str) -> int:
|
|
14
|
+
if node_type not in NODE_TYPES:
|
|
15
|
+
raise ValueError(
|
|
16
|
+
f"[sentence {sentence_id}] node_type inconnu : {node_type!r}. "
|
|
17
|
+
f"Valeurs autorisées : {NODE_TYPES}"
|
|
18
|
+
)
|
|
19
|
+
return NODE_TYPES.index(node_type)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _relation_idx(relation: str, sentence_id: str) -> int:
|
|
23
|
+
if relation not in RELATION_TYPES:
|
|
24
|
+
raise ValueError(
|
|
25
|
+
f"[sentence {sentence_id}] relation inconnue : {relation!r}. "
|
|
26
|
+
f"Valeurs autorisées : {RELATION_TYPES}"
|
|
27
|
+
)
|
|
28
|
+
return RELATION_TYPES.index(relation)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class TrainingSample:
|
|
33
|
+
sentence: SentenceRecord
|
|
34
|
+
gold_node_labels: np.ndarray # (N,) int — indices dans NODE_TYPES
|
|
35
|
+
edge_map: dict # {(src_clause_idx, tgt_clause_idx): rel_idx} — seule source de vérité pour les arêtes
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class GCNDataLoader:
|
|
39
|
+
"""Itère sur les sentences YAML d'un répertoire et produit des TrainingSample."""
|
|
40
|
+
|
|
41
|
+
def __init__(self, data_dir: Path, lang: str = "fr", repeat: bool = False):
|
|
42
|
+
self.data_dir = data_dir
|
|
43
|
+
self.lang = lang
|
|
44
|
+
self.repeat = repeat
|
|
45
|
+
self._records = load_all_sentences(data_dir, lang)
|
|
46
|
+
|
|
47
|
+
def __len__(self) -> int:
|
|
48
|
+
return len(self._records)
|
|
49
|
+
|
|
50
|
+
def __iter__(self):
|
|
51
|
+
while True:
|
|
52
|
+
for rec in self._records:
|
|
53
|
+
try:
|
|
54
|
+
yield self._to_sample(rec)
|
|
55
|
+
except ValueError as exc:
|
|
56
|
+
warnings.warn(f"[{rec.id}] sample ignoré : {exc}", UserWarning, stacklevel=2)
|
|
57
|
+
if not self.repeat:
|
|
58
|
+
break
|
|
59
|
+
|
|
60
|
+
def _to_sample(self, rec: SentenceRecord) -> TrainingSample:
|
|
61
|
+
# _relation_idx est appelé après les guards gap>1 et backward (src>tgt)
|
|
62
|
+
# pour éviter de rejeter toute la phrase sur une arête non-supervisable
|
|
63
|
+
# dont la relation serait inconnue.
|
|
64
|
+
node_id_to_idx = {c.node_id: i for i, c in enumerate(rec.clauses)}
|
|
65
|
+
node_labels = np.array(
|
|
66
|
+
[_node_type_idx(c.node_type, rec.id) for c in rec.clauses],
|
|
67
|
+
dtype=np.int64,
|
|
68
|
+
)
|
|
69
|
+
edge_map: dict[tuple[int, int], int] = {}
|
|
70
|
+
n_backward = 0
|
|
71
|
+
for e in rec.edges:
|
|
72
|
+
src_idx = node_id_to_idx.get(e.source)
|
|
73
|
+
tgt_idx = node_id_to_idx.get(e.target)
|
|
74
|
+
if src_idx is None or tgt_idx is None:
|
|
75
|
+
warnings.warn(
|
|
76
|
+
f"[{rec.id}] arête {e.source}→{e.target} : node_id inconnu — arête ignorée.",
|
|
77
|
+
UserWarning, stacklevel=2,
|
|
78
|
+
)
|
|
79
|
+
continue
|
|
80
|
+
gap = abs(tgt_idx - src_idx)
|
|
81
|
+
if gap > 1:
|
|
82
|
+
warnings.warn(
|
|
83
|
+
f"[{rec.id}] arête longue distance {e.source}→{e.target} "
|
|
84
|
+
f"(gap={gap}) : aucune supervision d'arête possible (forward prédit "
|
|
85
|
+
f"uniquement les paires consécutives).",
|
|
86
|
+
UserWarning, stacklevel=2,
|
|
87
|
+
)
|
|
88
|
+
continue # arête non-supervisable, ne pas insérer dans edge_map
|
|
89
|
+
if src_idx > tgt_idx:
|
|
90
|
+
n_backward += 1
|
|
91
|
+
continue # arête backward non-supervisable, ne pas insérer dans edge_map
|
|
92
|
+
rel_idx = _relation_idx(e.relation, rec.id)
|
|
93
|
+
edge_map[(src_idx, tgt_idx)] = rel_idx
|
|
94
|
+
if n_backward:
|
|
95
|
+
warnings.warn(
|
|
96
|
+
f"[{rec.id}] {n_backward} arête(s) gold en direction inverse "
|
|
97
|
+
f"(src > tgt) — non supervisées (le forward prédit uniquement la "
|
|
98
|
+
f"direction consécutive croissante).",
|
|
99
|
+
UserWarning, stacklevel=2,
|
|
100
|
+
)
|
|
101
|
+
return TrainingSample(rec, node_labels, edge_map)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def reps_from_sentence(
|
|
105
|
+
rec: SentenceRecord,
|
|
106
|
+
) -> tuple[list[UDRepresentation], list[int], list[UDRepresentation | None]]:
|
|
107
|
+
"""Une UDRepresentation par ClauseRecord non-vide, construite depuis les tokens YAML.
|
|
108
|
+
|
|
109
|
+
Bypass spaCy : garantit l'alignement exact features ↔ gold labels.
|
|
110
|
+
Retourne ([], [], []) si le SentenceRecord n'a pas de tokens annotés.
|
|
111
|
+
|
|
112
|
+
Retourne un triplet :
|
|
113
|
+
- reps : UDRepresentation par clause valide
|
|
114
|
+
- valid_indices : indices des clauses converties dans rec.clauses
|
|
115
|
+
- connector_reps : UDRepresentation du connecteur entre reps[k] et reps[k+1],
|
|
116
|
+
ou None si aucun connecteur trouvé (longueur = len(reps) - 1)
|
|
117
|
+
"""
|
|
118
|
+
if not rec.tokens or not rec.clauses:
|
|
119
|
+
return [], [], []
|
|
120
|
+
result: list[UDRepresentation] = []
|
|
121
|
+
valid_indices: list[int] = []
|
|
122
|
+
for i, clause in enumerate(rec.clauses):
|
|
123
|
+
rep = _rep_from_clause(clause, rec.tokens, rec.lang)
|
|
124
|
+
if rep is not None:
|
|
125
|
+
result.append(rep)
|
|
126
|
+
valid_indices.append(i)
|
|
127
|
+
connector_reps: list[UDRepresentation | None] = [
|
|
128
|
+
_connector_between(
|
|
129
|
+
rec.clauses[valid_indices[k]],
|
|
130
|
+
rec.clauses[valid_indices[k + 1]],
|
|
131
|
+
rec.tokens,
|
|
132
|
+
rec.lang,
|
|
133
|
+
)
|
|
134
|
+
for k in range(len(result) - 1)
|
|
135
|
+
]
|
|
136
|
+
return result, valid_indices, connector_reps
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _connector_between(
|
|
140
|
+
clause_a: ClauseRecord,
|
|
141
|
+
clause_b: ClauseRecord,
|
|
142
|
+
all_tokens: list[TokenRecord],
|
|
143
|
+
lang: str,
|
|
144
|
+
) -> UDRepresentation | None:
|
|
145
|
+
"""Retourne une UDRepresentation pour le token connecteur entre deux spans consécutives."""
|
|
146
|
+
end_a = clause_a.token_span[1]
|
|
147
|
+
start_b = clause_b.token_span[0]
|
|
148
|
+
gap_toks = [t for t in all_tokens if end_a < t.id < start_b]
|
|
149
|
+
if not gap_toks:
|
|
150
|
+
return None
|
|
151
|
+
tok = (
|
|
152
|
+
next((t for t in gap_toks if t.gcn_causal_type == "conjonction"), None)
|
|
153
|
+
or next((t for t in gap_toks if t.pos in {"SCONJ", "CCONJ", "ADP"}), None)
|
|
154
|
+
)
|
|
155
|
+
if tok is None:
|
|
156
|
+
return None
|
|
157
|
+
return UDRepresentation(
|
|
158
|
+
tokens=[{"lemma": tok.lemma, "pos": tok.pos, "dep_rel": tok.dep_rel, "morph": tok.morph}],
|
|
159
|
+
root_lemma=tok.lemma,
|
|
160
|
+
root_pos=tok.pos,
|
|
161
|
+
root_dep_rel=tok.dep_rel,
|
|
162
|
+
root_morph=tok.morph,
|
|
163
|
+
subject_pos=None,
|
|
164
|
+
has_object=False,
|
|
165
|
+
has_advcl=False,
|
|
166
|
+
has_temporal_obl=False,
|
|
167
|
+
token_span=(tok.id, tok.id),
|
|
168
|
+
lang=lang,
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _rep_from_clause(
|
|
173
|
+
clause: ClauseRecord,
|
|
174
|
+
all_tokens: list[TokenRecord],
|
|
175
|
+
lang: str,
|
|
176
|
+
) -> UDRepresentation | None:
|
|
177
|
+
span_start, span_end = clause.token_span
|
|
178
|
+
if span_start > span_end:
|
|
179
|
+
warnings.warn(
|
|
180
|
+
f"Span inversée dans {clause.node_id} : ({span_start}, {span_end}) — clause ignorée.",
|
|
181
|
+
UserWarning, stacklevel=3,
|
|
182
|
+
)
|
|
183
|
+
return None
|
|
184
|
+
span_toks = [t for t in all_tokens if span_start <= t.id <= span_end]
|
|
185
|
+
if not span_toks:
|
|
186
|
+
return None
|
|
187
|
+
|
|
188
|
+
# Priorité : VERB annoté gcn_causal_type="verbe", sinon premier VERB/AUX, sinon premier token
|
|
189
|
+
root_tok = (
|
|
190
|
+
next((t for t in span_toks
|
|
191
|
+
if t.gcn_causal_type == "verbe" and t.pos in {"VERB", "AUX"}), None)
|
|
192
|
+
or next((t for t in span_toks if t.pos in {"VERB", "AUX"}), span_toks[0])
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
subject = next((t for t in span_toks if t.dep_rel in {"nsubj", "nsubj:pass"}), None)
|
|
196
|
+
|
|
197
|
+
return UDRepresentation(
|
|
198
|
+
tokens=[
|
|
199
|
+
{"lemma": t.lemma, "pos": t.pos, "dep_rel": t.dep_rel, "morph": t.morph}
|
|
200
|
+
for t in span_toks
|
|
201
|
+
],
|
|
202
|
+
root_lemma=root_tok.lemma,
|
|
203
|
+
root_pos=root_tok.pos,
|
|
204
|
+
root_dep_rel=root_tok.dep_rel,
|
|
205
|
+
root_morph=root_tok.morph,
|
|
206
|
+
subject_pos=subject.pos if subject else None,
|
|
207
|
+
has_object=any(t.dep_rel in {"obj", "iobj", "nobj"} for t in span_toks),
|
|
208
|
+
has_advcl=any(t.dep_rel == "advcl" for t in span_toks),
|
|
209
|
+
has_temporal_obl=any(t.dep_rel in {"obl", "obl:tmod"} for t in span_toks),
|
|
210
|
+
token_span=clause.token_span,
|
|
211
|
+
lang=lang,
|
|
212
|
+
)
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class TokenRecord:
|
|
8
|
+
id: int # 1-based
|
|
9
|
+
form: str
|
|
10
|
+
lemma: str
|
|
11
|
+
pos: str # UPOS
|
|
12
|
+
dep_rel: str # UD dep relation
|
|
13
|
+
dep_head: int # 0 = root
|
|
14
|
+
morph: dict[str, str] = field(default_factory=dict)
|
|
15
|
+
gcn_causal_type: Optional[str] = None
|
|
16
|
+
gcn_causal_class: Optional[str] = None
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class ClauseRecord:
|
|
21
|
+
node_id: str # "n001"
|
|
22
|
+
node_type: str # NodeType snake_case
|
|
23
|
+
label: str
|
|
24
|
+
token_span: tuple[int, int]
|
|
25
|
+
scope: str
|
|
26
|
+
temporal_index: int
|
|
27
|
+
origin: str
|
|
28
|
+
attributes: dict[str, object] = field(default_factory=dict)
|
|
29
|
+
modifiers: list[dict] = field(default_factory=list)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class EdgeRecord:
|
|
34
|
+
source: str # "n001"
|
|
35
|
+
target: str # "n002"
|
|
36
|
+
relation: str # RelationType snake_case
|
|
37
|
+
confidence: float
|
|
38
|
+
explicit: bool
|
|
39
|
+
negated: bool
|
|
40
|
+
marker_token: Optional[int]
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
@dataclass
|
|
44
|
+
class SentenceRecord:
|
|
45
|
+
id: str
|
|
46
|
+
text: str
|
|
47
|
+
lang: str
|
|
48
|
+
tokens: list[TokenRecord]
|
|
49
|
+
clauses: list[ClauseRecord]
|
|
50
|
+
edges: list[EdgeRecord]
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import json
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import numpy as np
|
|
7
|
+
|
|
8
|
+
from ..verbalizer.trainable import SurfaceVocabulary
|
|
9
|
+
from ..constants import NODE_TYPES
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def _node_type_embeddings(nodes: list[dict]) -> np.ndarray:
|
|
13
|
+
"""Build one-hot node_type embeddings from a list of CausalIR node dicts."""
|
|
14
|
+
if not nodes:
|
|
15
|
+
return np.zeros((1, len(NODE_TYPES)), dtype=np.float32)
|
|
16
|
+
embs = []
|
|
17
|
+
for node in nodes:
|
|
18
|
+
nt = node.get("node_type", "")
|
|
19
|
+
idx = NODE_TYPES.index(nt) if nt in NODE_TYPES else 0
|
|
20
|
+
onehot = np.zeros(len(NODE_TYPES), dtype=np.float32)
|
|
21
|
+
onehot[idx] = 1.0
|
|
22
|
+
embs.append(onehot)
|
|
23
|
+
return np.stack(embs) # (N, 7)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class VerbalizeSample:
|
|
28
|
+
ir_json: str # CausalIR JSON (for inference)
|
|
29
|
+
node_type_embeddings: np.ndarray # (N, 7) one-hot — fallback when R-GCN not available
|
|
30
|
+
gold_tokens: np.ndarray # (T,) int indices in SurfaceVocabulary
|
|
31
|
+
source_text: str # used to match encoding dataset samples
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class VerbalizerDataLoader:
|
|
35
|
+
"""Loads verbalize JSON files and produces VerbalizeSample per (CausalIR, surface) pair.
|
|
36
|
+
|
|
37
|
+
Reads files matching verbalize_*.json in data_dir.
|
|
38
|
+
Each cross-modal example with N surfaces produces N VerbalizeSamples.
|
|
39
|
+
Only gold and silver quality surfaces are used for training.
|
|
40
|
+
gcn-verbalize.schema.yaml is documentation for annotators — never read here.
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
def __init__(
|
|
44
|
+
self,
|
|
45
|
+
data_dir: Path,
|
|
46
|
+
vocab: SurfaceVocabulary | None = None,
|
|
47
|
+
) -> None:
|
|
48
|
+
raw = self._load_raw(data_dir)
|
|
49
|
+
|
|
50
|
+
if vocab is None:
|
|
51
|
+
vocab = SurfaceVocabulary()
|
|
52
|
+
surfaces = [
|
|
53
|
+
surf["text"]
|
|
54
|
+
for ex in raw
|
|
55
|
+
for surf in ex.get("surfaces", [])
|
|
56
|
+
if surf.get("quality") in ("gold", "silver")
|
|
57
|
+
]
|
|
58
|
+
vocab.build(surfaces)
|
|
59
|
+
self.vocab = vocab
|
|
60
|
+
|
|
61
|
+
self._samples: list[VerbalizeSample] = []
|
|
62
|
+
for ex in raw:
|
|
63
|
+
ir = ex["causal_ir"]
|
|
64
|
+
ir_json = json.dumps(ir)
|
|
65
|
+
source_text: str = ir.get("source_text", "")
|
|
66
|
+
nodes: list[dict] = ir.get("nodes", [])
|
|
67
|
+
node_embs = _node_type_embeddings(nodes)
|
|
68
|
+
for surf in ex.get("surfaces", []):
|
|
69
|
+
if surf.get("quality") not in ("gold", "silver"):
|
|
70
|
+
continue
|
|
71
|
+
gold_tokens = np.array(vocab.encode(surf["text"]), dtype=np.int64)
|
|
72
|
+
self._samples.append(
|
|
73
|
+
VerbalizeSample(ir_json, node_embs, gold_tokens, source_text)
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
@staticmethod
|
|
77
|
+
def _load_raw(data_dir: Path) -> list[dict]:
|
|
78
|
+
examples: list[dict] = []
|
|
79
|
+
for p in sorted(data_dir.glob("verbalize_*.json")):
|
|
80
|
+
with p.open(encoding="utf-8") as f:
|
|
81
|
+
data = json.load(f)
|
|
82
|
+
examples.extend(data.get("examples", []))
|
|
83
|
+
return examples
|
|
84
|
+
|
|
85
|
+
def source_text_map(self) -> dict[str, list[np.ndarray]]:
|
|
86
|
+
"""Returns {source_text: [gold_tokens, ...]} for joint training lookup."""
|
|
87
|
+
result: dict[str, list[np.ndarray]] = {}
|
|
88
|
+
for s in self._samples:
|
|
89
|
+
result.setdefault(s.source_text, []).append(s.gold_tokens)
|
|
90
|
+
return result
|
|
91
|
+
|
|
92
|
+
def __len__(self) -> int:
|
|
93
|
+
return len(self._samples)
|
|
94
|
+
|
|
95
|
+
def __iter__(self):
|
|
96
|
+
yield from self._samples
|
|
File without changes
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
import json
|
|
3
|
+
import warnings
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
import click
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from ..data.loader import GCNDataLoader, reps_from_sentence
|
|
10
|
+
from ..pipeline.cgnp import CGNPipeline
|
|
11
|
+
from ..layer1.features import FeatureVocabulary
|
|
12
|
+
from ..layer2.reference import MLPEncoder
|
|
13
|
+
from ..layer3.reference import RGCNLayer
|
|
14
|
+
from ..training.checkpoint import load_checkpoint
|
|
15
|
+
from ..evaluation.metrics import (
|
|
16
|
+
node_accuracy, node_macro_f1,
|
|
17
|
+
edge_accuracy, edge_macro_f1,
|
|
18
|
+
)
|
|
19
|
+
from ..constants import NODE_TYPES, RELATION_TYPES
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def run_eval(data_dir: Path, model_path: Path, lang: str = "fr") -> dict:
|
|
23
|
+
"""Évalue le pipeline CGNP sur toutes les sentences d'un répertoire.
|
|
24
|
+
|
|
25
|
+
Retourne un dict avec : n_samples, n_skipped, node_accuracy,
|
|
26
|
+
node_macro_f1, edge_accuracy, edge_macro_f1.
|
|
27
|
+
"""
|
|
28
|
+
vocab = FeatureVocabulary()
|
|
29
|
+
encoder = MLPEncoder(d_clause=vocab.d_clause, d_edge=vocab.d_edge)
|
|
30
|
+
graph = RGCNLayer(d_in=vocab.d_clause, d_out=vocab.d_clause)
|
|
31
|
+
pipeline = CGNPipeline(encoder=encoder, graph=graph, lang=lang, vocabulary=vocab)
|
|
32
|
+
load_checkpoint(pipeline, model_path)
|
|
33
|
+
|
|
34
|
+
loader = GCNDataLoader(data_dir, lang=lang)
|
|
35
|
+
|
|
36
|
+
all_node_preds: list[str] = []
|
|
37
|
+
all_node_gold: list[str] = []
|
|
38
|
+
all_edge_preds: list[str] = []
|
|
39
|
+
all_edge_gold: list[str] = []
|
|
40
|
+
n_samples = 0
|
|
41
|
+
n_skipped = 0
|
|
42
|
+
|
|
43
|
+
for sample in loader:
|
|
44
|
+
if not sample.sentence.clauses:
|
|
45
|
+
n_skipped += 1
|
|
46
|
+
continue
|
|
47
|
+
try:
|
|
48
|
+
reps, valid_clause_idxs, connector_reps = reps_from_sentence(sample.sentence)
|
|
49
|
+
if not reps:
|
|
50
|
+
n_skipped += 1
|
|
51
|
+
continue
|
|
52
|
+
pipeline.forward(
|
|
53
|
+
reps, sample.sentence.text,
|
|
54
|
+
clause_positions=valid_clause_idxs,
|
|
55
|
+
n_total_clauses=len(sample.sentence.clauses),
|
|
56
|
+
connector_reps=connector_reps,
|
|
57
|
+
)
|
|
58
|
+
except Exception as exc:
|
|
59
|
+
warnings.warn(f"[{sample.sentence.id}] forward ignoré : {exc}", stacklevel=2)
|
|
60
|
+
n_skipped += 1
|
|
61
|
+
continue
|
|
62
|
+
|
|
63
|
+
node_logits = pipeline._cached_node_logits
|
|
64
|
+
edge_logits = pipeline._cached_edge_logits
|
|
65
|
+
if node_logits is None or len(node_logits) == 0:
|
|
66
|
+
n_skipped += 1
|
|
67
|
+
continue
|
|
68
|
+
|
|
69
|
+
if valid_clause_idxs:
|
|
70
|
+
gold_node = sample.gold_node_labels[np.array(valid_clause_idxs, dtype=np.int64)]
|
|
71
|
+
else:
|
|
72
|
+
gold_node = sample.gold_node_labels
|
|
73
|
+
|
|
74
|
+
node_pred_idxs = np.argmax(node_logits, axis=1)
|
|
75
|
+
all_node_preds.extend(NODE_TYPES[i] for i in node_pred_idxs)
|
|
76
|
+
all_node_gold.extend(NODE_TYPES[i] for i in gold_node)
|
|
77
|
+
|
|
78
|
+
if (valid_clause_idxs and len(valid_clause_idxs) >= 2
|
|
79
|
+
and sample.edge_map and edge_logits is not None and len(edge_logits) > 0):
|
|
80
|
+
pairs = [
|
|
81
|
+
(valid_clause_idxs[k], valid_clause_idxs[k + 1])
|
|
82
|
+
for k in range(len(valid_clause_idxs) - 1)
|
|
83
|
+
]
|
|
84
|
+
gold_edge_full = np.array(
|
|
85
|
+
[sample.edge_map.get(p, -1) for p in pairs], dtype=np.int64
|
|
86
|
+
)
|
|
87
|
+
valid_mask = gold_edge_full >= 0
|
|
88
|
+
if valid_mask.any():
|
|
89
|
+
valid_idxs = np.where(valid_mask)[0]
|
|
90
|
+
gold_edge = gold_edge_full[valid_idxs]
|
|
91
|
+
edge_pred_idxs = np.argmax(edge_logits[valid_idxs], axis=1)
|
|
92
|
+
all_edge_preds.extend(RELATION_TYPES[i] for i in edge_pred_idxs)
|
|
93
|
+
all_edge_gold.extend(RELATION_TYPES[i] for i in gold_edge)
|
|
94
|
+
|
|
95
|
+
n_samples += 1
|
|
96
|
+
|
|
97
|
+
return {
|
|
98
|
+
"n_samples": n_samples,
|
|
99
|
+
"n_skipped": n_skipped,
|
|
100
|
+
"node_accuracy": node_accuracy(all_node_preds, all_node_gold),
|
|
101
|
+
"node_macro_f1": node_macro_f1(all_node_preds, all_node_gold),
|
|
102
|
+
"edge_accuracy": edge_accuracy(all_edge_preds, all_edge_gold),
|
|
103
|
+
"edge_macro_f1": edge_macro_f1(all_edge_preds, all_edge_gold),
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@click.command("gcn-eval")
|
|
108
|
+
@click.option("--data-dir", required=True, type=click.Path(path_type=Path))
|
|
109
|
+
@click.option("--model-path", required=True, type=click.Path(path_type=Path))
|
|
110
|
+
@click.option("--lang", default="fr", show_default=True)
|
|
111
|
+
@click.option("--output", default=None, type=click.Path(path_type=Path),
|
|
112
|
+
help="Chemin JSON du rapport (optionnel, sinon stdout)")
|
|
113
|
+
def eval_cmd(
|
|
114
|
+
data_dir: Path, model_path: Path, lang: str, output: Path | None
|
|
115
|
+
) -> None:
|
|
116
|
+
"""Évalue le pipeline CGNP sur un répertoire de données annotées."""
|
|
117
|
+
report = run_eval(data_dir, model_path, lang)
|
|
118
|
+
text = json.dumps(report, indent=2, ensure_ascii=False)
|
|
119
|
+
if output:
|
|
120
|
+
Path(output).write_text(text, encoding="utf-8")
|
|
121
|
+
click.echo(f"Rapport écrit : {output}")
|
|
122
|
+
else:
|
|
123
|
+
click.echo(text)
|