requipso 0.2.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- requipso/__init__.py +27 -0
- requipso/cli.py +11 -0
- requipso/completer.py +103 -0
- requipso/data.py +72 -0
- requipso/descriptive_pipeline.py +271 -0
- requipso/docs/macros.py +226 -0
- requipso/docs/mkdocs.yml +140 -0
- requipso/docs/pages/assets/consort_flowchart.png +0 -0
- requipso/docs/pages/assets/demo.gif +0 -0
- requipso/docs/pages/blocs/age.md +61 -0
- requipso/docs/pages/blocs/ald.md +65 -0
- requipso/docs/pages/blocs/biometry.md +98 -0
- requipso/docs/pages/blocs/cim10.md +80 -0
- requipso/docs/pages/blocs/cisp2.md +59 -0
- requipso/docs/pages/blocs/code-postal.md +47 -0
- requipso/docs/pages/blocs/consultations.md +65 -0
- requipso/docs/pages/blocs/departement.md +50 -0
- requipso/docs/pages/blocs/est-vivant.md +44 -0
- requipso/docs/pages/blocs/genre.md +46 -0
- requipso/docs/pages/blocs/index.md +40 -0
- requipso/docs/pages/blocs/insurance.md +55 -0
- requipso/docs/pages/blocs/medecin-traitant.md +56 -0
- requipso/docs/pages/blocs/paquet-annees.md +70 -0
- requipso/docs/pages/blocs/prescription-atc.md +69 -0
- requipso/docs/pages/blocs/regex-consultation.md +64 -0
- requipso/docs/pages/blocs/regex-nlp.md +107 -0
- requipso/docs/pages/blocs/regex-patient-observation.md +81 -0
- requipso/docs/pages/blocs/regex-prescription.md +64 -0
- requipso/docs/pages/guide/application.md +81 -0
- requipso/docs/pages/guide/combiner-blocs.md +61 -0
- requipso/docs/pages/guide/demarrage.md +74 -0
- requipso/docs/pages/guide/erreurs.md +83 -0
- requipso/docs/pages/guide/exemple-complet.md +67 -0
- requipso/docs/pages/guide/google-sheet.md +50 -0
- requipso/docs/pages/guide/onglet-inclusion.md +91 -0
- requipso/docs/pages/guide/onglet-projet.md +48 -0
- requipso/docs/pages/guide/onglet-tableau-descriptif.md +124 -0
- requipso/docs/pages/guide/parametres.md +162 -0
- requipso/docs/pages/index.md +73 -0
- requipso/docs/pages/stylesheets/extra.css +3 -0
- requipso/docs/pages/technique/ajouter-un-bloc.md +147 -0
- requipso/docs/pages/technique/architecture.md +80 -0
- requipso/docs/pages/technique/blocs-internes.md +176 -0
- requipso/docs/pages/technique/chargement-sheet.md +107 -0
- requipso/docs/pages/technique/developpement.md +124 -0
- requipso/docs/pages/technique/donnees.md +106 -0
- requipso/docs/pages/technique/export-signatures.md +131 -0
- requipso/docs/pages/technique/nlp.md +82 -0
- requipso/docs/pages/technique/parametres-internes.md +140 -0
- requipso/docs/pages/technique/pipeline.md +66 -0
- requipso/docs/pages/technique/tableau-descriptif.md +74 -0
- requipso/google.py +168 -0
- requipso/io.py +95 -0
- requipso/notebooks/marimo/run.py +352 -0
- requipso/pipeline.py +364 -0
- requipso/pipeline_loader.py +572 -0
- requipso/pipes/__init__.py +43 -0
- requipso/pipes/atcd.py +365 -0
- requipso/pipes/base.py +431 -0
- requipso/pipes/biometry.py +170 -0
- requipso/pipes/consultations.py +117 -0
- requipso/pipes/insurance.py +90 -0
- requipso/pipes/params.py +283 -0
- requipso/pipes/patient.py +377 -0
- requipso/pipes/prescription.py +134 -0
- requipso/pipes/processors.py +250 -0
- requipso/polars.py +24 -0
- requipso/resources/pathology/__init__.py +7 -0
- requipso/signature_exporter.py +473 -0
- requipso/stats.py +130 -0
- requipso-0.2.2.dist-info/METADATA +102 -0
- requipso-0.2.2.dist-info/RECORD +75 -0
- requipso-0.2.2.dist-info/WHEEL +5 -0
- requipso-0.2.2.dist-info/entry_points.txt +2 -0
- requipso-0.2.2.dist-info/top_level.txt +1 -0
requipso/google.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
from io import StringIO
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
from typing import Self, cast
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
from google.auth.transport.requests import Request
|
|
7
|
+
from google.oauth2.credentials import Credentials
|
|
8
|
+
from google_auth_oauthlib.flow import InstalledAppFlow # type: ignore
|
|
9
|
+
from googleapiclient.discovery import build # type: ignore
|
|
10
|
+
from googleapiclient.http import MediaFileUpload # type: ignore
|
|
11
|
+
from gspread_pandas import Client, Spread # type: ignore
|
|
12
|
+
|
|
13
|
+
from requipso import BASE_DIR
|
|
14
|
+
|
|
15
|
+
SCOPES = [
|
|
16
|
+
"https://www.googleapis.com/auth/drive",
|
|
17
|
+
"https://www.googleapis.com/auth/spreadsheets",
|
|
18
|
+
]
|
|
19
|
+
CLIENT_SECRET_PATH = BASE_DIR / "google_oauth_client.json"
|
|
20
|
+
TOKEN_PATH = Path.home() / ".config" / "requipso" / "google_token.json"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def get_user_credentials(
|
|
24
|
+
client_secret_path: Path = CLIENT_SECRET_PATH,
|
|
25
|
+
token_path: Path = TOKEN_PATH,
|
|
26
|
+
) -> Credentials:
|
|
27
|
+
"""Return the user's Google credentials, opening a browser login if needed.
|
|
28
|
+
|
|
29
|
+
The token is cached in ``token_path``; delete it to log in with another account.
|
|
30
|
+
"""
|
|
31
|
+
creds: Credentials | None = None
|
|
32
|
+
if token_path.exists():
|
|
33
|
+
creds = Credentials.from_authorized_user_file(token_path, SCOPES)
|
|
34
|
+
if creds and creds.valid:
|
|
35
|
+
return creds
|
|
36
|
+
if creds and creds.expired and creds.refresh_token:
|
|
37
|
+
creds.refresh(Request())
|
|
38
|
+
else:
|
|
39
|
+
flow = InstalledAppFlow.from_client_secrets_file(client_secret_path, SCOPES)
|
|
40
|
+
creds = flow.run_local_server(port=0)
|
|
41
|
+
token_path.parent.mkdir(parents=True, exist_ok=True)
|
|
42
|
+
token_path.write_text(creds.to_json())
|
|
43
|
+
return creds
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class GoogleConnector:
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
folder_id: str = "1vTA1zVLbyU6mVMkiTL1J1bLolOlsut7A",
|
|
50
|
+
credentials: Credentials | None = None,
|
|
51
|
+
):
|
|
52
|
+
self.folder_id = folder_id
|
|
53
|
+
self.credentials = credentials or get_user_credentials()
|
|
54
|
+
|
|
55
|
+
# Build Drive service
|
|
56
|
+
self.service = build("drive", "v3", credentials=self.credentials)
|
|
57
|
+
|
|
58
|
+
self.sheets = self.list_sheets()
|
|
59
|
+
|
|
60
|
+
self.spreadsheet_name: str | None = None
|
|
61
|
+
self.spreadsheet_key: str | None = None
|
|
62
|
+
self.spread: Spread | None = None
|
|
63
|
+
|
|
64
|
+
def set_spreadsheet(self, spreadsheet_name: str) -> Self:
|
|
65
|
+
self.spreadsheet_name = spreadsheet_name
|
|
66
|
+
self.spreadsheet_key = self.sheets.get(self.spreadsheet_name)
|
|
67
|
+
if self.spreadsheet_key is None:
|
|
68
|
+
raise ValueError(
|
|
69
|
+
f"Spreadsheet {self.spreadsheet_name} not found. Available spreadsheets: {self.sheets.keys()}"
|
|
70
|
+
)
|
|
71
|
+
self.spread = self.open_spread()
|
|
72
|
+
return self
|
|
73
|
+
|
|
74
|
+
def open_sheet(self, sheet_name: str, header_rows: int = 1) -> pd.DataFrame:
|
|
75
|
+
"""Read a worksheet through CSV so pandas handles parsing consistently."""
|
|
76
|
+
if self.spread is None:
|
|
77
|
+
raise ValueError("Spreadsheet not open")
|
|
78
|
+
df = self.spread.sheet_to_df(
|
|
79
|
+
sheet=sheet_name,
|
|
80
|
+
index=False,
|
|
81
|
+
header_rows=header_rows,
|
|
82
|
+
)
|
|
83
|
+
csv_buffer = StringIO()
|
|
84
|
+
df.to_csv(csv_buffer, index=False)
|
|
85
|
+
csv_buffer.seek(0)
|
|
86
|
+
df = pd.read_csv(csv_buffer)
|
|
87
|
+
df = df.where(pd.notna(df), None)
|
|
88
|
+
|
|
89
|
+
return cast(pd.DataFrame, df)
|
|
90
|
+
|
|
91
|
+
def list_sheets(self) -> dict[str, str]:
|
|
92
|
+
results = (
|
|
93
|
+
self.service.files()
|
|
94
|
+
.list(
|
|
95
|
+
q=f"'{self.folder_id}' in parents and mimeType='application/vnd.google-apps.spreadsheet'",
|
|
96
|
+
fields="files(id, name)",
|
|
97
|
+
supportsAllDrives=True,
|
|
98
|
+
includeItemsFromAllDrives=True,
|
|
99
|
+
)
|
|
100
|
+
.execute()
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
files = results.get("files", [])
|
|
104
|
+
|
|
105
|
+
return {f["name"]: f["id"] for f in files}
|
|
106
|
+
|
|
107
|
+
def open_spread(self) -> Spread:
|
|
108
|
+
"""Return a :class:`Spread` for the current spreadsheet."""
|
|
109
|
+
client = Client(creds=self.credentials)
|
|
110
|
+
return Spread(spread=self.spreadsheet_key, client=client)
|
|
111
|
+
|
|
112
|
+
def upload_file(self, file_path: Path, upload_name: str | None = None) -> str:
|
|
113
|
+
# 1. Search for existing file
|
|
114
|
+
upload_name = upload_name or file_path.name
|
|
115
|
+
query = (
|
|
116
|
+
f"'{self.folder_id}' in parents and name='{upload_name}' and trashed=false"
|
|
117
|
+
)
|
|
118
|
+
|
|
119
|
+
results = (
|
|
120
|
+
self.service.files()
|
|
121
|
+
.list(
|
|
122
|
+
q=query,
|
|
123
|
+
fields="files(id, name)",
|
|
124
|
+
supportsAllDrives=True,
|
|
125
|
+
includeItemsFromAllDrives=True,
|
|
126
|
+
)
|
|
127
|
+
.execute()
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
files = results.get("files", [])
|
|
131
|
+
|
|
132
|
+
media = MediaFileUpload(file_path)
|
|
133
|
+
|
|
134
|
+
if files:
|
|
135
|
+
# File exists → UPDATE it
|
|
136
|
+
file_id = files[0]["id"]
|
|
137
|
+
|
|
138
|
+
_ = (
|
|
139
|
+
self.service.files()
|
|
140
|
+
.update(
|
|
141
|
+
fileId=file_id,
|
|
142
|
+
media_body=media,
|
|
143
|
+
supportsAllDrives=True,
|
|
144
|
+
)
|
|
145
|
+
.execute()
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
print("File replaced:", file_id)
|
|
149
|
+
|
|
150
|
+
else:
|
|
151
|
+
# File does not exist → CREATE it
|
|
152
|
+
file_metadata = {"name": upload_name, "parents": [self.folder_id]}
|
|
153
|
+
|
|
154
|
+
created = (
|
|
155
|
+
self.service.files()
|
|
156
|
+
.create(
|
|
157
|
+
body=file_metadata,
|
|
158
|
+
media_body=media,
|
|
159
|
+
fields="id",
|
|
160
|
+
supportsAllDrives=True,
|
|
161
|
+
)
|
|
162
|
+
.execute()
|
|
163
|
+
)
|
|
164
|
+
file_id = created["id"]
|
|
165
|
+
print("File uploaded:", file_id)
|
|
166
|
+
|
|
167
|
+
url = f"https://drive.google.com/file/d/{file_id}/view?usp=drive_link"
|
|
168
|
+
return url
|
requipso/io.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
from typing import Generic, TypeVar
|
|
5
|
+
from urllib.parse import urlparse
|
|
6
|
+
|
|
7
|
+
import boto3
|
|
8
|
+
import polars as pl
|
|
9
|
+
|
|
10
|
+
T = TypeVar("T", Path, str)
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class BaseIO(ABC, Generic[T]):
|
|
14
|
+
data_path: T
|
|
15
|
+
|
|
16
|
+
def __init__(self, data_path: T):
|
|
17
|
+
self.data_path = data_path
|
|
18
|
+
|
|
19
|
+
@abstractmethod
|
|
20
|
+
def path_from_table_name(self, table_name: str) -> T:
|
|
21
|
+
"""Get the full path for a given table name."""
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
def list_tables(self) -> list[str]:
|
|
25
|
+
"""Return a list of available table identifiers."""
|
|
26
|
+
|
|
27
|
+
@abstractmethod
|
|
28
|
+
def read_table(self, table_name: str, n_rows: int | None = None) -> pl.DataFrame:
|
|
29
|
+
"""Load a parquet table into a Polars DataFrame."""
|
|
30
|
+
|
|
31
|
+
@abstractmethod
|
|
32
|
+
def write_table(self, table_name: str, table: pl.DataFrame) -> None:
|
|
33
|
+
"""Write a Polars DataFrame to a parquet table."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class LocalIO(BaseIO[Path]):
|
|
37
|
+
def __init__(self, data_path: Path | str):
|
|
38
|
+
super().__init__(Path(data_path))
|
|
39
|
+
|
|
40
|
+
def path_from_table_name(self, table_name: str) -> Path:
|
|
41
|
+
return self.data_path / f"{table_name}.parquet"
|
|
42
|
+
|
|
43
|
+
def list_tables(self) -> list[str]:
|
|
44
|
+
return [p.stem for p in self.data_path.glob("*.parquet")]
|
|
45
|
+
|
|
46
|
+
def read_table(self, table_name: str, n_rows: int | None = None) -> pl.DataFrame:
|
|
47
|
+
table_path = self.path_from_table_name(table_name)
|
|
48
|
+
if not table_path.exists():
|
|
49
|
+
raise FileNotFoundError(f"Table {table_name} not found at {table_path}")
|
|
50
|
+
return pl.read_parquet(table_path, n_rows=n_rows)
|
|
51
|
+
|
|
52
|
+
def write_table(self, table_name: str, table: pl.DataFrame) -> None:
|
|
53
|
+
table_path = self.path_from_table_name(table_name)
|
|
54
|
+
table.write_parquet(table_path)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class S3IO(BaseIO[str]):
|
|
58
|
+
def __init__(self, data_path: str):
|
|
59
|
+
super().__init__(data_path.strip("/"))
|
|
60
|
+
|
|
61
|
+
self.s3 = boto3.client("s3")
|
|
62
|
+
|
|
63
|
+
if not self.data_path.startswith("s3://"):
|
|
64
|
+
raise ValueError("data_path must be an s3 URI like s3://bucket/prefix")
|
|
65
|
+
|
|
66
|
+
if not os.getenv("AWS_ACCESS_KEY_ID") or not os.getenv("AWS_SECRET_ACCESS_KEY"):
|
|
67
|
+
raise EnvironmentError("AWS credentials not found in environment variables")
|
|
68
|
+
|
|
69
|
+
parsed = urlparse(self.data_path)
|
|
70
|
+
if parsed.scheme != "s3" or not parsed.netloc:
|
|
71
|
+
raise ValueError("data_path must be an s3 URI like s3://bucket/prefix")
|
|
72
|
+
|
|
73
|
+
self.bucket_name = parsed.netloc
|
|
74
|
+
self.prefix = parsed.path.lstrip("/")
|
|
75
|
+
|
|
76
|
+
def path_from_table_name(self, table_name: str) -> str:
|
|
77
|
+
if self.prefix:
|
|
78
|
+
return f"s3://{self.bucket_name}/{self.prefix}/{table_name}.parquet"
|
|
79
|
+
else:
|
|
80
|
+
return f"s3://{self.bucket_name}/{table_name}.parquet"
|
|
81
|
+
|
|
82
|
+
def list_tables(self) -> list[str]:
|
|
83
|
+
response = self.s3.list_objects_v2(Bucket=self.bucket_name, Prefix=self.prefix)
|
|
84
|
+
return [
|
|
85
|
+
obj["Key"].split(".parquet")[0]
|
|
86
|
+
for obj in response.get("Contents", [])
|
|
87
|
+
if obj["Key"].endswith(".parquet")
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
def read_table(self, table_name: str, n_rows: int | None = None) -> pl.DataFrame:
|
|
91
|
+
return pl.read_parquet(self.path_from_table_name(table_name), n_rows=n_rows)
|
|
92
|
+
|
|
93
|
+
def write_table(self, table_name: str, table: pl.DataFrame) -> None:
|
|
94
|
+
table_path = self.path_from_table_name(table_name)
|
|
95
|
+
table.write_parquet(table_path)
|
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
import marimo
|
|
2
|
+
|
|
3
|
+
__generated_with = "0.23.14"
|
|
4
|
+
app = marimo.App(width="medium")
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@app.cell
|
|
8
|
+
def _():
|
|
9
|
+
import json
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
from importlib.metadata import version
|
|
13
|
+
|
|
14
|
+
import marimo as mo
|
|
15
|
+
|
|
16
|
+
config_path = Path(__file__).parent / ".requipso_last.json"
|
|
17
|
+
|
|
18
|
+
def load_config() -> dict:
|
|
19
|
+
try:
|
|
20
|
+
return json.loads(config_path.read_text()) # type: ignore
|
|
21
|
+
except (FileNotFoundError, json.JSONDecodeError):
|
|
22
|
+
return {}
|
|
23
|
+
|
|
24
|
+
def save_config(data_dir: str, sheet_name: str) -> None:
|
|
25
|
+
config_path.write_text(
|
|
26
|
+
json.dumps(
|
|
27
|
+
{
|
|
28
|
+
"data_dir": data_dir,
|
|
29
|
+
"sheet_name": sheet_name,
|
|
30
|
+
}
|
|
31
|
+
)
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
version=version("requipso")
|
|
35
|
+
last_config = load_config()
|
|
36
|
+
return Path, last_config, mo, save_config, version
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@app.cell
|
|
40
|
+
def _(mo, version):
|
|
41
|
+
mo.md(
|
|
42
|
+
"# Requipso\n\n"
|
|
43
|
+
+ f"* Version installée: {version}\n"
|
|
44
|
+
+ "* [Documentation en ligne](https://ipsosante.github.io/requipso/)"
|
|
45
|
+
)
|
|
46
|
+
return
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
@app.cell
|
|
50
|
+
def _(Path, last_config, mo):
|
|
51
|
+
_initial = (
|
|
52
|
+
Path(last_config["data_dir"]).parent
|
|
53
|
+
if "data_dir" in last_config
|
|
54
|
+
else Path.home()
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
file_browser = mo.ui.file_browser(
|
|
58
|
+
initial_path=_initial,
|
|
59
|
+
selection_mode="directory",
|
|
60
|
+
multiple=False,
|
|
61
|
+
label="choisir le dossier local contenant les données",
|
|
62
|
+
)
|
|
63
|
+
file_browser
|
|
64
|
+
return (file_browser,)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@app.cell
|
|
68
|
+
def _():
|
|
69
|
+
from requipso.google import GoogleConnector
|
|
70
|
+
|
|
71
|
+
google_connector = GoogleConnector()
|
|
72
|
+
return (google_connector,)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
@app.cell
|
|
76
|
+
def _(google_connector, last_config, mo):
|
|
77
|
+
_options = sorted(google_connector.sheets)
|
|
78
|
+
_last_sheet = last_config.get("sheet_name")
|
|
79
|
+
_initial_value = _last_sheet if _last_sheet in _options else None
|
|
80
|
+
|
|
81
|
+
sheet_dropdown = mo.ui.dropdown(
|
|
82
|
+
options=_options,
|
|
83
|
+
value=_initial_value,
|
|
84
|
+
label="choisir le Google Sheet",
|
|
85
|
+
)
|
|
86
|
+
sheet_dropdown
|
|
87
|
+
return (sheet_dropdown,)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@app.cell
|
|
91
|
+
def _(file_browser, google_connector, mo, sheet_dropdown):
|
|
92
|
+
from requipso import Data
|
|
93
|
+
|
|
94
|
+
mo.stop(
|
|
95
|
+
not (len(file_browser.value) > 0 and sheet_dropdown.value),
|
|
96
|
+
mo.md(
|
|
97
|
+
"renseigner le dossier contenant les données et choisir le Google Sheet pour continuer"
|
|
98
|
+
),
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
data_dir = file_browser.path(0)
|
|
102
|
+
sheet_name = sheet_dropdown.value
|
|
103
|
+
sheet_id = google_connector.sheets[sheet_name]
|
|
104
|
+
|
|
105
|
+
data = Data(data_dir)
|
|
106
|
+
|
|
107
|
+
run_button = mo.ui.run_button(
|
|
108
|
+
label="(re)lancer la pipeline",
|
|
109
|
+
kind="success",
|
|
110
|
+
tooltip="cliquer pour lancer la pipeline",
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
mo.vstack(
|
|
114
|
+
[
|
|
115
|
+
mo.md(f"**Dossier:** `{data_dir}`"),
|
|
116
|
+
mo.md(
|
|
117
|
+
f"**Google Sheet:** [{sheet_name}](https://docs.google.com/spreadsheets/d/{sheet_id}/edit)"
|
|
118
|
+
),
|
|
119
|
+
run_button,
|
|
120
|
+
]
|
|
121
|
+
)
|
|
122
|
+
return data, data_dir, run_button, sheet_name
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@app.cell
|
|
126
|
+
def _(mo):
|
|
127
|
+
get_pipeline_hash, set_pipeline_hash = mo.state(0)
|
|
128
|
+
get_flowchart_url, set_flowchart_url = mo.state("")
|
|
129
|
+
return (
|
|
130
|
+
get_flowchart_url,
|
|
131
|
+
get_pipeline_hash,
|
|
132
|
+
set_flowchart_url,
|
|
133
|
+
set_pipeline_hash,
|
|
134
|
+
)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
@app.cell
|
|
138
|
+
def _(
|
|
139
|
+
Path,
|
|
140
|
+
data,
|
|
141
|
+
data_dir,
|
|
142
|
+
get_pipeline_hash,
|
|
143
|
+
google_connector,
|
|
144
|
+
mo,
|
|
145
|
+
run_button,
|
|
146
|
+
save_config,
|
|
147
|
+
set_flowchart_url,
|
|
148
|
+
set_pipeline_hash,
|
|
149
|
+
sheet_name,
|
|
150
|
+
):
|
|
151
|
+
mo.stop(not run_button.value)
|
|
152
|
+
|
|
153
|
+
get_success_pipeline, set_success_pipeline = mo.state(False)
|
|
154
|
+
get_success_descriptive, set_success_descriptive = mo.state(False)
|
|
155
|
+
|
|
156
|
+
import time
|
|
157
|
+
|
|
158
|
+
from requipso.pipeline_loader import PipelineBuilder
|
|
159
|
+
|
|
160
|
+
save_config(str(data_dir), sheet_name)
|
|
161
|
+
|
|
162
|
+
google_connector.set_spreadsheet(sheet_name)
|
|
163
|
+
|
|
164
|
+
builder = PipelineBuilder(google_connector)
|
|
165
|
+
inclusion = builder.build()
|
|
166
|
+
|
|
167
|
+
if get_pipeline_hash() == builder.pipeline_hash:
|
|
168
|
+
mo.output.append(mo.md("pipeline déjà exécutée, pas de recalcul nécessaire"))
|
|
169
|
+
else:
|
|
170
|
+
data.reset()
|
|
171
|
+
|
|
172
|
+
n_steps = len(inclusion.steps_names)
|
|
173
|
+
with mo.status.progress_bar(
|
|
174
|
+
total=n_steps, title="inclusion ...", subtitle=""
|
|
175
|
+
) as _bar:
|
|
176
|
+
for step_name, n in inclusion.iter_run(data):
|
|
177
|
+
time.sleep(0.5)
|
|
178
|
+
_bar.update(
|
|
179
|
+
increment=1,
|
|
180
|
+
subtitle=f"bloc {step_name} : {n} patients",
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
inclusion.graph("graph")
|
|
184
|
+
url = google_connector.upload_file(
|
|
185
|
+
Path("graph.png"), upload_name=f"{sheet_name}.png"
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
# Run was successful, we store the hash and url:
|
|
189
|
+
set_pipeline_hash(builder.pipeline_hash)
|
|
190
|
+
set_flowchart_url(url)
|
|
191
|
+
|
|
192
|
+
set_success_pipeline(True)
|
|
193
|
+
print("NEW HASH", get_pipeline_hash())
|
|
194
|
+
return (
|
|
195
|
+
builder,
|
|
196
|
+
get_success_descriptive,
|
|
197
|
+
get_success_pipeline,
|
|
198
|
+
inclusion,
|
|
199
|
+
set_success_descriptive,
|
|
200
|
+
time,
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@app.cell
|
|
205
|
+
def _(
|
|
206
|
+
builder,
|
|
207
|
+
data,
|
|
208
|
+
get_flowchart_url,
|
|
209
|
+
get_success_pipeline,
|
|
210
|
+
google_connector,
|
|
211
|
+
inclusion,
|
|
212
|
+
mo,
|
|
213
|
+
set_success_descriptive,
|
|
214
|
+
time,
|
|
215
|
+
):
|
|
216
|
+
|
|
217
|
+
mo.stop(
|
|
218
|
+
not get_success_pipeline(),
|
|
219
|
+
mo.md("pipeline non exécutée, pas de tableau descriptif disponible"),
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
descriptive = builder.build_descriptive(
|
|
223
|
+
data=data,
|
|
224
|
+
inclusion_pipeline=inclusion,
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
n_rows = len(descriptive.rows)
|
|
228
|
+
|
|
229
|
+
with mo.status.progress_bar(
|
|
230
|
+
total=n_rows, title="tableau descriptif ...", subtitle=""
|
|
231
|
+
) as _bar:
|
|
232
|
+
for row, _ in zip(descriptive.rows, descriptive.run_iter(data)):
|
|
233
|
+
time.sleep(0.5)
|
|
234
|
+
_bar.update(
|
|
235
|
+
increment=1,
|
|
236
|
+
subtitle=f"variable: {row['display_name']}",
|
|
237
|
+
)
|
|
238
|
+
|
|
239
|
+
stats = descriptive.format_stats(descriptive.stats)
|
|
240
|
+
descriptive.upload_stats(stats)
|
|
241
|
+
|
|
242
|
+
set_success_descriptive(True)
|
|
243
|
+
|
|
244
|
+
mo.vstack(
|
|
245
|
+
[
|
|
246
|
+
mo.md(f"""
|
|
247
|
+
**pipeline terminée**
|
|
248
|
+
* Tableau descriptif [dispo plus bas](#tableau-descriptif) et sur le [Google Sheet](https://docs.google.com/spreadsheets/d/{google_connector.spreadsheet_key}), onglet "🔒 Results"
|
|
249
|
+
* Flowchart d'inclusion sur le [Google Drive]({get_flowchart_url()})
|
|
250
|
+
"""),
|
|
251
|
+
mo.md("### Flowchart d'inclusion"),
|
|
252
|
+
mo.image(src="graph.png"),
|
|
253
|
+
mo.md("### Tableau descriptif"),
|
|
254
|
+
mo.ui.table(stats, label="Statistiques descriptives", page_size=100),
|
|
255
|
+
]
|
|
256
|
+
)
|
|
257
|
+
return
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
@app.cell
|
|
261
|
+
def _(get_success_descriptive, inclusion, mo):
|
|
262
|
+
mo.stop(not get_success_descriptive())
|
|
263
|
+
|
|
264
|
+
import polars as pl
|
|
265
|
+
import polars_permutation as pp
|
|
266
|
+
|
|
267
|
+
AVAILABLE_STEPS = list(inclusion.steps_names.keys())
|
|
268
|
+
|
|
269
|
+
def get_patient_list(
|
|
270
|
+
data, step_name: str, shuffling_key: str | None
|
|
271
|
+
) -> pl.DataFrame:
|
|
272
|
+
if step_name not in AVAILABLE_STEPS:
|
|
273
|
+
raise ValueError(f"step_name {step_name} not found in data.all_results")
|
|
274
|
+
df = (
|
|
275
|
+
data.all_results[step_name]["post"]
|
|
276
|
+
.filter(pl.col(step_name))
|
|
277
|
+
.select("patient_id")
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
patients = data.get_table("patient").select("patient_id", "patient_identifier")
|
|
281
|
+
if not shuffling_key:
|
|
282
|
+
return df.join(patients, on="patient_id", how="inner").select(
|
|
283
|
+
shuffled_patient_id="patient_id",
|
|
284
|
+
patient_identifier="patient_identifier",
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
df_ids = pl.select(
|
|
288
|
+
pl.int_range(0, 100_000_000, eager=False).alias("final_patient_id")
|
|
289
|
+
).with_columns(
|
|
290
|
+
patient_id=pp.feistel_permute(
|
|
291
|
+
pl.col("final_patient_id"), n=100_000_000, seed=int(shuffling_key)
|
|
292
|
+
)
|
|
293
|
+
)
|
|
294
|
+
df_ids = df_ids.join(patients, on="patient_id", how="inner")
|
|
295
|
+
df = df.join(df_ids, on="patient_id", how="inner")
|
|
296
|
+
return df.select(
|
|
297
|
+
patient_id="final_patient_id",
|
|
298
|
+
shuffled_patient_id="patient_id",
|
|
299
|
+
patient_identifier="patient_identifier",
|
|
300
|
+
)
|
|
301
|
+
|
|
302
|
+
step_dropdown = mo.ui.dropdown(
|
|
303
|
+
options=AVAILABLE_STEPS,
|
|
304
|
+
value=AVAILABLE_STEPS[0] if AVAILABLE_STEPS else None,
|
|
305
|
+
label="choisir l'étape",
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
shuffle_key_input = mo.ui.text(
|
|
309
|
+
label="clé de permutation des ID patients (1Password)",
|
|
310
|
+
placeholder="optionnel",
|
|
311
|
+
)
|
|
312
|
+
return get_patient_list, shuffle_key_input, step_dropdown
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
@app.cell
|
|
316
|
+
def _(data, get_patient_list, mo, shuffle_key_input, step_dropdown):
|
|
317
|
+
mo.stop(not step_dropdown.value)
|
|
318
|
+
|
|
319
|
+
def _make_csv():
|
|
320
|
+
return (
|
|
321
|
+
get_patient_list(
|
|
322
|
+
data, step_dropdown.value, shuffling_key=shuffle_key_input.value
|
|
323
|
+
)
|
|
324
|
+
.write_csv()
|
|
325
|
+
.encode("utf-8")
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
_download = mo.download(
|
|
329
|
+
data=_make_csv,
|
|
330
|
+
filename=f"patients.csv",
|
|
331
|
+
mimetype="text/csv",
|
|
332
|
+
label="télécharger la liste des patients (CSV)",
|
|
333
|
+
)
|
|
334
|
+
|
|
335
|
+
mo.vstack(
|
|
336
|
+
[
|
|
337
|
+
mo.md("### Liste de patients"),
|
|
338
|
+
step_dropdown,
|
|
339
|
+
shuffle_key_input,
|
|
340
|
+
_download,
|
|
341
|
+
]
|
|
342
|
+
)
|
|
343
|
+
return
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
@app.cell
|
|
347
|
+
def _():
|
|
348
|
+
return
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
if __name__ == "__main__":
|
|
352
|
+
app.run()
|