ppi-analyser 0.1.0__tar.gz

ppi_analyser-0.1.0/MANIFEST.in

@@ -0,0 +1,4 @@
+exclude .env
+exclude .env.*
+recursive-exclude * .env
+recursive-exclude * .env.*

ppi_analyser-0.1.0/PKG-INFO

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: ppi-analyser
+Version: 0.1.0
+Summary: Analyse automatique de phrases préfabriquées d'interaction (PPI) — pipeline NLP + serveur FastAPI
+Author: Youssef Zeroual
+License: MIT
+Project-URL: Homepage, https://github.com/YoussefZeroual/ppi-analyser
+Project-URL: Repository, https://github.com/YoussefZeroual/ppi-analyser
+Project-URL: Issues, https://github.com/YoussefZeroual/ppi-analyser/issues
+Keywords: nlp,phraseology,french,corpus-linguistics,fastapi,stanza
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing :: Linguistic
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: fastapi>=0.111.0
+Requires-Dist: uvicorn[standard]>=0.29.0
+Requires-Dist: python-multipart>=0.0.9
+Requires-Dist: flask>=3.0.0
+Requires-Dist: flask-cors>=4.0.0
+Requires-Dist: torch>=2.3.1
+Requires-Dist: stanza>=1.8.0
+Requires-Dist: nltk>=3.8.1
+Requires-Dist: mistralai>=1.0.0
+Requires-Dist: openai>=1.30.0
+Requires-Dist: ollama>=0.2.0
+Requires-Dist: google-generativeai>=0.7.0
+Requires-Dist: groq>=0.9.0
+Requires-Dist: pandas>=2.2.0
+Requires-Dist: openpyxl>=3.1.2
+Requires-Dist: xlsxwriter>=3.2.0
+Requires-Dist: reportlab>=4.1.0
+Requires-Dist: markdown
+Requires-Dist: weasyprint
+Requires-Dist: python-dotenv>=1.0.0
+Requires-Dist: pyyaml>=6.0.1
+Requires-Dist: requests>=2.31.0
+Requires-Dist: httpx>=0.27.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: mypy; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: build; extra == "dev"
+Requires-Dist: twine; extra == "dev"
+
+# PPI Analyser
+
+Outil d'analyse automatique de phrases préfabriquées d'interaction (PPI) françaises.
+
+## Installation
+```bash
+python -m venv myenv
+source myenv/bin/activate
+pip install -e .
+```
+
+## Configuration
+
+Copy `.env.example` to `.env` and fill in your API keys:
+```bash
+cp .env.example .env
+```
+
+## Usage
+```python
+from ppi_analyser.core import PPIAnalyser
+from ppi_analyser.config import PipelineConfig, AnalysisMode
+
+analyser = PPIAnalyser()
+config = PipelineConfig(
+    models=["mistral_mistral-medium-latest"],
+    expression="je t'en prie",
+    sentence_file="path/to/corpus.xlsx",
+    mode=AnalysisMode.ORAL,
+    output_dir="path/to/output",
+)
+df, state = analyser.process_sentences(config)
+```
+
+## Modes
+
+- `ORAL` — corpus oral
+- `ECRIT` — corpus écrit (segmentation automatique)
+- `ECRIT_IA` — corpus écrit avec détection des tours de parole par LLM
+- `ECRIT_TEST` — mode test rapide
+
+## Models supported
+
+- Mistral (`mistral_mistral-medium-latest`)
+- Ollama (`ollama_mistral:7b`)
+- Groq (`groq_moonshotai/kimi-k2-instruct`)
+- DeepSeek (`deepseek_deepseek-chat`)
+- Gemini (`gemini_gemini-3-flash-preview`)
+# ppi_analyser

ppi_analyser-0.1.0/README.md

@@ -0,0 +1,49 @@
+# PPI Analyser
+
+Outil d'analyse automatique de phrases préfabriquées d'interaction (PPI) françaises.
+
+## Installation
+```bash
+python -m venv myenv
+source myenv/bin/activate
+pip install -e .
+```
+
+## Configuration
+
+Copy `.env.example` to `.env` and fill in your API keys:
+```bash
+cp .env.example .env
+```
+
+## Usage
+```python
+from ppi_analyser.core import PPIAnalyser
+from ppi_analyser.config import PipelineConfig, AnalysisMode
+
+analyser = PPIAnalyser()
+config = PipelineConfig(
+    models=["mistral_mistral-medium-latest"],
+    expression="je t'en prie",
+    sentence_file="path/to/corpus.xlsx",
+    mode=AnalysisMode.ORAL,
+    output_dir="path/to/output",
+)
+df, state = analyser.process_sentences(config)
+```
+
+## Modes
+
+- `ORAL` — corpus oral
+- `ECRIT` — corpus écrit (segmentation automatique)
+- `ECRIT_IA` — corpus écrit avec détection des tours de parole par LLM
+- `ECRIT_TEST` — mode test rapide
+
+## Models supported
+
+- Mistral (`mistral_mistral-medium-latest`)
+- Ollama (`ollama_mistral:7b`)
+- Groq (`groq_moonshotai/kimi-k2-instruct`)
+- DeepSeek (`deepseek_deepseek-chat`)
+- Gemini (`gemini_gemini-3-flash-preview`)
+# ppi_analyser

ppi_analyser-0.1.0/ppi_analyser/analysis/analysis_cache.py

@@ -0,0 +1,74 @@
+# analysis/analysis_cache.py
+
+import json
+import hashlib
+import logging
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_cache: dict | None = None
+_cache_path: Path | None = None
+
+
+def init(cache_path: str) -> None:
+    """Call once at the start of a run to set the cache path and load existing entries."""
+    global _cache, _cache_path
+    _cache_path = Path(cache_path)
+    if _cache_path.exists():
+        try:
+            _cache = json.loads(_cache_path.read_text(encoding="utf-8"))
+            logger.info("Analysis cache loaded: %d entries from %s", len(_cache), _cache_path)
+        except Exception as e:
+            logger.warning("Could not load analysis cache: %s — starting fresh", e)
+            _cache = {}
+    else:
+        _cache = {}
+
+
+def _require_init() -> None:
+    if _cache is None:
+        raise RuntimeError(
+            "Analysis cache not initialised — call analysis_cache.init(path) first, "
+            "or set use_analysis_cache=True and analysis_cache_path in PipelineConfig."
+        )
+
+
+def _key(conversation: str, expression: str, model: str, submodel: str, prompt_type: str) -> str:
+    payload = f"{conversation}|{expression}|{model}|{submodel}|{prompt_type}"
+    return hashlib.md5(payload.encode("utf-8")).hexdigest()
+
+
+def get(conversation: str, expression: str, model: str, submodel: str, prompt_type: str) -> str | None:
+    _require_init()
+    result = _cache.get(_key(conversation, expression, model, submodel, prompt_type))
+    if result is not None:
+        logger.debug("Analysis cache HIT — %s / %s / %s", prompt_type, model, expression[:40])
+    return result
+
+
+def set(conversation: str, expression: str, model: str, submodel: str, prompt_type: str, result: str) -> None:
+    _require_init()
+    _cache[_key(conversation, expression, model, submodel, prompt_type)] = result
+    _save()
+
+
+def _save() -> None:
+    try:
+        _cache_path.parent.mkdir(parents=True, exist_ok=True)
+        _cache_path.write_text(json.dumps(_cache, ensure_ascii=False, indent=2), encoding="utf-8")
+    except Exception as e:
+        logger.warning("Could not save analysis cache: %s", e)
+
+
+def cache_size() -> int:
+    _require_init()
+    return len(_cache)
+
+
+def clear() -> None:
+    global _cache
+    _require_init()
+    _cache = {}
+    _save()
+    logger.info("Analysis cache cleared")

ppi_analyser-0.1.0/ppi_analyser/analysis/expansion.py

@@ -0,0 +1,140 @@
+import sys
+import re
+import pandas as pd
+from ppi_analyser.stanza.stanza_client import StanzaClient
+from ppi_analyser.exporters.excel import format_ppi_bold
+
+client = StanzaClient()
+
+
+def extract_ppi_sentence(tagged_line):
+    match = re.search(r'<PPI>(.*?)</PPI>', tagged_line, re.IGNORECASE)
+    if not match:
+        return None, None
+    ppi_text = match.group(1).strip()
+    # Trouve la limite gauche : dernier séparateur avant la balise
+    #pre = tagged_line[:match.start()]
+    post = tagged_line[match.end():]
+    # Coupe à gauche sur / ou début de ligne
+    #left = re.split(r'/', pre)[-1]
+    # Coupe à droite sur / 
+    right = re.split(r'/', post)[0]
+    clean_seg = re.sub(r'</?PPI>', '', right, flags=re.IGNORECASE).strip() # <-- removed left be cause exp is always in the right
+    return ppi_text, clean_seg
+
+def get_ppi_ids(sentence, ppi_text):
+    words = sentence.words
+    ppi_clean = re.sub(r'\s*-\s*', '-', ppi_text.lower()).strip()
+    for i in range(len(words)):
+        for j in range(i+1, len(words)+1):
+            window = words[i:j]
+            surface = re.sub(r'\s*-\s*', '-', " ".join(w.text for w in window).lower())
+            if surface == ppi_clean:
+                return set(w.id for w in window)
+    return set()
+
+
+def get_ppi_head(sentence, ppi_ids):
+    for w in sentence.words:
+        if w.id in ppi_ids and w.head not in ppi_ids:
+            return w
+    return None
+
+
+def get_subtree(head_word, words, exclude_ids=set()):
+    subtree_ids = {head_word.id}
+    changed = True
+    while changed:
+        changed = False
+        for w in words:
+            if w.head in subtree_ids and w.id not in subtree_ids and w.id not in exclude_ids:
+                subtree_ids.add(w.id)
+                changed = True
+    return sorted([w for w in words if w.id in subtree_ids], key=lambda w: w.id)
+
+
+def get_expansion_from_sentence(sentence, ppi_text):
+    import logging
+    logger = logging.getLogger(__name__)
+    
+    #logger.warning("%s",[f"{w.text}_{w.upos}:{w.deprel}" for w in sentence.words])
+    ppi_ids = get_ppi_ids(sentence, ppi_text)
+    if not ppi_ids:
+        return [{"type": None, "tokens": []}]
+    ppi_head = get_ppi_head(sentence, ppi_ids)
+    if not ppi_head:
+        return [{"type": None, "tokens": []}]
+    words = sentence.words
+    dependants = [w for w in words if w.head == ppi_head.id and w.id not in ppi_ids]
+    expansions = []
+    #logger.warning("%s",[f"{dep.text}_{dep.deprel}_{dep.upos}" for dep in dependants])
+    for dep in dependants:
+        deprel = dep.deprel
+        upos = dep.upos
+        if deprel == "xcomp" and upos == "VERB":
+            subtree = get_subtree(dep, words, exclude_ids=ppi_ids)
+            expansions.append({"type": "infinitive", "tokens": subtree})
+        elif deprel in ("ccomp", "csubj"):
+            subtree = get_subtree(dep, words, exclude_ids=ppi_ids)
+            expansions.append({"type": "completive_que", "tokens": subtree})
+        elif deprel in ("nmod", "obl", "obl:arg", "obj","advcl") and upos in ("NOUN", "PRON", "VERB"):
+            subtree = get_subtree(dep, words, exclude_ids=ppi_ids)
+            expansions.append({"type": "nominal_prep", "tokens": subtree})
+    return expansions[:1] if expansions else [{"type": None, "tokens": []}]
+
+
+def detect_expansion(doc, ppi_text, occurrence=0):
+    count = 0
+    for sentence in doc.sentences:
+        result = get_expansion_from_sentence(sentence, ppi_text)
+        if result[0]["type"] is not None or result[0]["tokens"]:
+            if count == occurrence:
+                return result
+            count += 1
+    return [{"type": None, "tokens": []}]
+
+
+def process_file(input_path):
+    if input_path.endswith(".csv"):
+        df_in = pd.read_csv(input_path)
+    else:
+        df_in = pd.read_excel(input_path)
+    print(df_in.columns.tolist())
+    print(df_in.head(2))
+    rows = []
+    for _, row in df_in.iterrows():
+        conversation = str(row.get("Conversation", ""))
+        lines = conversation.split("\n")
+        for line in lines:
+            if not re.search(r'<PPI>', line, re.IGNORECASE):
+                continue
+            ppi_text, clean_seg = extract_ppi_sentence(line)
+            if not ppi_text:
+                continue
+            expansions = detect_expansion(clean_seg, ppi_text)
+            exp = expansions[0]
+            expansion_text = " ".join(w["text"] for w in exp["tokens"]) if exp["tokens"] else ""
+            print(f"Tour : {line.strip()}")
+            print(f"  Type     : {exp['type']}")
+            print(f"  Expansion: {expansion_text}\n")
+            rows.append({
+                "Tour": line.strip(),
+                "PPI": ppi_text,
+                "Type_expansion_1": exp["type"] if len(expansions) > 0 else "",
+                "Expansion_1": expansion_text,
+                "Type_expansion_2": expansions[1]["type"] if len(expansions) > 1 else "",
+                "Expansion_2": " ".join(w["text"] for w in expansions[1]["tokens"]) if len(expansions) > 1 and expansions[1]["tokens"] else "",
+            })
+
+    df_out = pd.DataFrame(rows)
+    base = re.sub(r'\.(xlsx|csv)$', '', input_path)
+    output_path = f"{base}_expansion.xlsx"
+    format_ppi_bold(df_out, output_path)
+    print(f"Résultat enregistré : {output_path}")
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python expansion.py <fichier.csv|xlsx>")
+        sys.exit(1)
+    process_file(sys.argv[1])

ppi_analyser-0.1.0/ppi_analyser/analysis/mistral_batch_pipeline.py

@@ -0,0 +1,288 @@
+# analysis/mistral_batch_pipeline.py
+
+import json
+import logging
+from pathlib import Path
+
+from ppi_analyser.analysis.pipeline import PreprocessedSentence, _chunk
+from ppi_analyser.analysis.prompts import get_prompts_batch, get_prompt_type
+from ppi_analyser.analysis.sentence import _handle_no_model_batch, _parse_batch_response
+from ppi_analyser.models.factory import get_mistral_batch_provider
+from ppi_analyser.config import PipelineConfig
+from ppi_analyser.state import SessionState
+
+logger = logging.getLogger(__name__)
+
+#NON_IA = dict(state.no_ia)#{0, 1, 5, 7,8}  # Forme, Lemme, Position, expansion — handled locally, not submitted to Mistral
+
+
+def _custom_id(chunk_idx: int, prop_idx: int) -> str:
+    return f"c{chunk_idx}_p{prop_idx}"
+
+
+# ---------------------------------------------------------------------------
+# Main entry point
+# ---------------------------------------------------------------------------
+
+def analyse_batch_mistral_async(
+    preprocessed: list[PreprocessedSentence],
+    lemmes: list[str] | None,
+    config: PipelineConfig,
+    state: SessionState,
+) -> tuple[list[PreprocessedSentence], list[list[str]]]:
+    NON_IA = set(state.no_ia)
+  
+    submodel = _resolve_submodel(config.models)
+    provider = get_mistral_batch_provider(submodel, config.output_dir)
+
+    # Check for saved job state from a previous interrupted run
+    saved = provider.load_job_state()
+    if saved:
+        logger.info("Resuming Mistral batch jobs from saved state")
+        all_results = _poll_and_assemble(saved["job_map"], provider, preprocessed, lemmes, config, state)
+        if all_results is None:
+            raise InterruptedError("Mistral batch jobs still running. Re-run to resume.")
+        provider.clear_job_state()
+        return preprocessed, all_results
+
+    # Save preprocessed so we can reassemble on resume
+    preprocessed_path = Path(config.output_dir) / "mistral_batch_preprocessed.json"
+    _save_preprocessed(preprocessed, lemmes, preprocessed_path)
+
+    chunks       = _chunk(preprocessed, config.batch_size)
+    lemme_chunks = _chunk(lemmes, config.batch_size) if lemmes else [None] * len(chunks)
+
+    # Submit one job per (chunk × property), skip NON_IA
+    job_map        = {}  # "c{i}_p{j}" -> job_id
+    cached_results = {}  # "c{i}_p{j}" -> raw_response (from analysis cache)
+    prompt_map     = {}  # "c{i}_p{j}" -> {user, prompt_type} for cache saving
+
+    for chunk_idx, (chunk, lemme_chunk) in enumerate(zip(chunks, lemme_chunks)):
+        expression = lemme_chunk[0] if lemme_chunk else config.expression
+        if lemme_chunk:
+            state.expression_list.extend(lemme_chunk)
+
+        system_prompts, batched_prompts = get_prompts_batch(
+            expression=expression,
+            forme_relevee_list=[s.forme_relevee for s in chunk],
+            conv_list=[s.cleaned for s in chunk],
+            locuteur_list=[s.locuteur for s in chunk],
+            interlocuteur_list=[s.interlocuteurs for s in chunk],
+            mode=config.mode,
+        )
+
+        for prop_idx, (system_prompt, batched_prompt) in enumerate(
+            zip(system_prompts, batched_prompts)
+        ):
+            if prop_idx in NON_IA:
+                continue
+            prompt_type = get_prompt_type(system_prompt)
+            if config.properties and prompt_type not in config.properties:
+                continue
+            if state.custom_properties_list is not None and prompt_type not in state.custom_properties_list: # fixed added custom props
+                continue
+
+            cid = _custom_id(chunk_idx, prop_idx)
+
+            # Check analysis cache before submitting
+            if state.use_analysis_cache:
+                from ppi_analyser.analysis.analysis_cache import get as acache_get
+                cached = acache_get(batched_prompt, "", "mistral_batch", submodel, prompt_type)
+                if cached is not None:
+                    logger.debug("Analysis cache HIT for %s chunk %d prop %d", prompt_type, chunk_idx, prop_idx)
+                    cached_results[cid] = cached
+                    continue
+
+            job_id = provider.submit([{
+                "custom_id": cid,
+                "system":    system_prompt,
+                "user":      batched_prompt,
+            }])
+            job_map[cid]    = job_id
+            prompt_map[cid] = {"user": batched_prompt, "prompt_type": prompt_type}
+            logger.info("Submitted job %s for chunk %d prop %d", job_id, chunk_idx, prop_idx)
+
+    # Poll all jobs
+    all_results = _poll_and_assemble(job_map, provider, preprocessed, lemmes, config, state,
+                                     preprocessed_path=str(preprocessed_path),
+                                     cached_results=cached_results,
+                                     submodel=submodel,
+                                     prompt_map=prompt_map)
+    if all_results is None:
+        raise InterruptedError("Mistral batch jobs still running. Re-run to resume.")
+
+    provider.clear_job_state()
+    return preprocessed, all_results
+
+
+# ---------------------------------------------------------------------------
+# Poll + assemble
+# ---------------------------------------------------------------------------
+
+def _poll_and_assemble(
+    job_map: dict,
+    provider,
+    preprocessed: list[PreprocessedSentence],
+    lemmes: list[str] | None,
+    config: PipelineConfig,
+    state: SessionState,
+    preprocessed_path: str | None = None,
+    cached_results: dict | None = None,
+    submodel: str = "",
+    prompt_map: dict | None = None,
+) -> list[list[str]] | None:
+    """
+    Poll all jobs in job_map. Returns assembled results or None if any timed out.
+    job_map: {"c{i}_p{j}": job_id}
+    """
+    raw_results = dict(cached_results) if cached_results else {}
+    timed_out   = []
+
+    for cid, job_id in job_map.items():
+        result = provider.poll_or_save(job_id, preprocessed_json=preprocessed_path)
+        if result is None:
+            timed_out.append(cid)
+        else:
+            raw_response = result.get(cid, "")
+            raw_results[cid] = raw_response
+            # Save to analysis cache using prompt_map for key info
+            if state.use_analysis_cache and raw_response and prompt_map and cid in prompt_map:
+                from ppi_analyser.analysis.analysis_cache import set as acache_set
+                entry = prompt_map[cid]
+                acache_set(entry["user"], "", "mistral_batch", submodel,
+                           entry["prompt_type"], raw_response)
+                logger.debug("Analysis cache SET for %s", entry["prompt_type"])
+
+    if timed_out:
+        combined = {"job_map": job_map, "preprocessed_json": preprocessed_path}
+        state_path = Path(config.output_dir) / "mistral_batch_job.json"
+        state_path.write_text(json.dumps(combined, indent=2), encoding="utf-8")
+        logger.warning("%d jobs timed out. State saved to %s", len(timed_out), state_path)
+        return None
+
+    return _assemble(raw_results, preprocessed, lemmes, config, state)
+
+
+
+# ---------------------------------------------------------------------------
+# Assemble
+# ---------------------------------------------------------------------------
+
+def _assemble(
+    raw_results: dict,
+    preprocessed: list[PreprocessedSentence],
+    lemmes: list[str] | None,
+    config: PipelineConfig,
+    state: SessionState,
+) -> list[list[str]]:
+    """
+    Reassemble per-sentence results from raw_results.
+    raw_results: {"c{chunk}_p{prop}": raw_response_string}
+    Returns results_per_sentence[sent][prop].
+    """
+    chunks       = _chunk(preprocessed, config.batch_size)
+    lemme_chunks = _chunk(lemmes, config.batch_size) if lemmes else [None] * len(chunks)
+    all_results  = []
+    from ppi_analyser.config import AnalysisMode
+    if state.analysis_mode ==AnalysisMode.ORAL:
+    	NON_IA = [0,1,5]
+    else:
+    	NON_IA = set(state.no_ia)
+    for chunk_idx, (chunk, lemme_chunk) in enumerate(zip(chunks, lemme_chunks)):
+        n_sents    = len(chunk)
+        expression = lemme_chunk[0] if lemme_chunk else config.expression
+
+        system_prompts, _ = get_prompts_batch(
+            expression=expression,
+            forme_relevee_list=[s.forme_relevee for s in chunk],
+            conv_list=[s.cleaned for s in chunk],
+            locuteur_list=[s.locuteur for s in chunk],
+            interlocuteur_list=[s.interlocuteurs for s in chunk],
+            mode=config.mode,
+        )
+
+        n_props              = len(system_prompts)
+        results_per_property = []
+
+        for prop_idx, system_prompt in enumerate(system_prompts):
+            if prop_idx in NON_IA:
+                prop_results = _handle_no_model_batch(
+                    system_prompt=system_prompt,
+                    conversations=[s.cleaned for s in chunk],
+                    expression=expression,
+                    forme_relevee_list=[s.forme_relevee for s in chunk],
+                    state=state,
+                    mode=config.mode,
+                    n_sentences=n_sents,
+                    start_offset=chunk_idx * config.batch_size
+                )
+            else:
+                cid = _custom_id(chunk_idx, prop_idx)
+                prompt_type = get_prompt_type(system_prompt)
+                if config.properties and prompt_type not in config.properties:
+                    prop_results = [None] * n_sents
+                elif state.custom_properties_list is not None and prompt_type not in state.custom_properties_list:
+                        prop_results = [None] * n_sents        
+                else:
+                    raw_response = raw_results.get(cid, "")
+                    if not raw_response:
+                        similar = [k for k in raw_results if k.startswith(f"c{chunk_idx}_")]
+                        logger.warning("No result for %s — keys for this chunk: %s", cid, similar)
+                    prop_results = _parse_batch_response(raw_response, n_sents)
+
+            results_per_property.append(prop_results)
+
+        # Transpose: prop × sent -> sent × prop
+        for sent_idx in range(n_sents):
+            row = []
+            for prop_results in results_per_property:
+                if prop_results is None:
+                    row.append(None)
+                else:
+                    row.append(
+                        prop_results[sent_idx] if sent_idx < len(prop_results) else None
+                    )
+            all_results.append(row)
+
+    return all_results
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _resolve_submodel(models: list[str]) -> str:
+    for m in models:
+        if m.startswith("mistral_batch_"):
+            return m[len("mistral_batch_"):]
+    raise ValueError("No mistral_batch_<submodel> entry found in models list")
+
+
+def _save_preprocessed(preprocessed, lemmes, path: Path) -> None:
+    data = {
+        "preprocessed": [
+            {
+                "raw":            s.raw,
+                "cleaned":        s.cleaned,
+                "locuteur":       s.locuteur,
+                "interlocuteurs": s.interlocuteurs,
+                "forme_relevee":  s.forme_relevee,
+            }
+            for s in preprocessed
+        ],
+        "lemmes": lemmes,
+    }
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+    logger.info("Preprocessed sentences saved to %s", path)
+
+
+def _load_preprocessed(path: str) -> tuple[list[PreprocessedSentence], list[str] | None]:
+    data = json.loads(Path(path).read_text(encoding="utf-8"))
+    preprocessed = [
+        PreprocessedSentence(
+            raw=s["raw"], cleaned=s["cleaned"], locuteur=s["locuteur"],
+            interlocuteurs=s["interlocuteurs"], forme_relevee=s["forme_relevee"],
+        )
+        for s in data["preprocessed"]
+    ]
+    return preprocessed, data.get("lemmes")

ppi_analyser-0.1.0/ppi_analyser/analysis/modifier_rules.yaml

@@ -0,0 +1,27 @@
+# Les parties du discours des modifieurs
+upos:
+  - ADV
+  - ADJ
+  - NOUN
+# dépendances syntaxiques par rapport à la tête de la phrase
+deprel:
+  - obl:mod
+  - nmod
+  - acl:relcl
+  - dislocated
+  - amod
+ 
+# lemmes surajoutés à la PPI standard
+lemma:
+  - dieu
+  - diable
+
+# à exclure
+
+excluded_upos:
+  - PUNKT
+
+excluded_deprel:
+  - 
+excluded_lemma:
+  -