kcai-data-sampling 0.0.6__tar.gz → 0.7__tar.gz

kcai_data_sampling-0.7/.gitignore

@@ -0,0 +1,265 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be added to the global gitignore or merged into this project gitignore.  For a PyCharm
+#  project, it is recommended to include the following files:
+#  .idea/
+#  *.iml
+#  *.ipr
+#  *.iws
+
+# VS Code
+.vscode/
+
+# macOS
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# Windows
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# Linux
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+# uv specific
+.uv/
+
+# Project specific
+# Uncomment if you want to ignore these directories
+# input/
+# output/
+# examples/
+
+# Temporary files
+*.tmp
+*.temp
+*.swp
+*.swo
+*~
+
+# Logs
+*.log
+logs/
+
+# IDE specific
+.idea/
+*.iml
+*.ipr
+*.iws
+
+# Data files (uncomment if you don't want to track data)
+*.parquet
+*.csv
+# *.json
+*.pkl
+*.pickle

kcai_data_sampling-0.7/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: kcai-data-sampling
+Version: 0.7
+Summary: Samples generation utilities managed with uv
+Author-email: Safenai <support@safenai.io>
+License-Expression: Apache-2.0
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: pyarrow>=22.0.0
+Requires-Dist: numpy>=1.20.0
+Requires-Dist: pillow>=8.0.0
+Requires-Dist: scikit-learn>=1.0.0
+Requires-Dist: torch>=2.8.0
+Requires-Dist: ipykernel>=6.29.5
+Requires-Dist: torchvision>=0.20.1
+Requires-Dist: captum==0.7.0
+Requires-Dist: torch_pruning>=1.3.7
+Requires-Dist: tqdm==4.66.5
+Requires-Dist: click==8.1.7
+Requires-Dist: pyyaml
+Requires-Dist: numpy<2
+Requires-Dist: neural-compressor==3.1.1
+Requires-Dist: thop
+Requires-Dist: tabulate
+Requires-Dist: albumentations>=2.0.8
+Requires-Dist: pyautoattack>=0.1.1
+Requires-Dist: boto3>=1.40.51

{kcai_data_sampling-0.0.6 → kcai_data_sampling-0.7}/pyproject.toml

@@ -4,24 +4,16 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "kcai-data-sampling"
-version = "0.0.6"
+
 description = "Samples generation utilities managed with uv"
-dynamic = ["dependencies"] #, "version"]
+dynamic = ["dependencies", "version"]
 authors = [
   {name = "Safenai", email="support@safenai.io"},   
-  {name = "IRT SystemX", email="support@irt-systemx.fr"},
 ]
 readme = "README.md"
 requires-python = ">=3.11"
 license ="Apache-2.0"
 
-[tool.pytest.ini_options]
-addopts = "-q"
-
-[tool.ruff]
-line-length = 100
-
-
 [tool.setuptools.dynamic]
 dependencies = {file = ["requirements.txt"]}
 

{kcai_data_sampling-0.0.6 → kcai_data_sampling-0.7}/requirements.txt

@@ -1,16 +1,13 @@
 # Main dependencies for kc-dataset
-pyarrow>=10.0.0
+pyarrow>=22.0.0
 numpy>=1.20.0
 pillow>=8.0.0
 scikit-learn>=1.0.0
-torch>=1.10.0
+torch>=2.8.0
 ipykernel>=6.29.5
 torchvision>=0.20.1
-
-torch==2.5.1
-torchvision==0.20.1
 captum==0.7.0
-torch_pruning==1.3.7
+torch_pruning>=1.3.7
 tqdm==4.66.5
 click==8.1.7
 pyyaml
@@ -18,8 +15,7 @@ numpy<2
 neural-compressor==3.1.1
 thop
 tabulate
-pyarrow==17.0.0
 albumentations>=2.0.8
-autoattack>=0.1
-cleverhans>=4.0.0
+pyautoattack>=0.1.1
+
 boto3>=1.40.51

kcai_data_sampling-0.7/src/kcai_data_sampling/Description_artefacts.md

@@ -0,0 +1,62 @@
+# Description artefacts
+
+## Samples
+
+Raw_samples
+    - samples_raw.parquet : table de référence unique avec colonne split (train/val/test). Clés minimales : sample_id, split, class_name, transform_name, transform_id, image_path|image_bytes.
+    - samples_train.parquet, samples_val.parquet, samples_test.parquet : mêmes colonnes que samples_raw mais filtrées par split, pour des traitements cibles (calcule de métriques separement)
+
+Perturbed_samples
+    - Corrupted sample (augmentations/corruptions classiques)
+        - Colonnes: sample_id , class_name, transform_name, transform_level, transform_param, split, model_name,  orig_pred, perturbed_pred, image_bytes.
+    - Adversarial sample (attaques adverses)
+        - Colonnes : sample_id, class_name, transform_name (attack_name), transform_level, transform_param (epsilon|steps), split, model_name , class_name, image_bytes.
+
+## Features (based on image_byte) : garder les colonnes et enlever les différentes images.
+
+- By sample
+    - raw_sample : une ligne par sample_id, features dérivées de image_bytes . 
+        - Corrupted sample + colonnes de features (luminosité, contrast, blur, entropy), 
+            - retirer image_bytes.
+    - perturbed_sample : Corrupted sample + features (luminosité, contrast, blur, entropy) ; 
+        - retirer image_bytes.
+
+## Métriques
+### dqm-ml 
+#### Representativness [chi-2 goodness, grte, ks, shannon]
+    - By feature 
+    - By split: train/test/val
+        - By class name
+        - By models
+        - By transform_name
+
+### autres métriques
+
+#### Classification (à envoyer directement sur klarity ? )
+    - Accuracy globale et par classe
+    - Precision / Recall / F1 ...
+    - ROC-AUC et PR-AUC one-vs-rest (macro / weighted)
+    - Matrices de confusion (par split et par transform_name)
+
+#### Robustesse (corruptions/augmentations)
+    - Robust accuracy par transform_name et transform_level
+    - Moyenne sur niveaux de sévérité 
+    - Worst-case accuracy sur l'ensemble des corruptions: trouver le couple (transform_name, transform_level) qui minimise l’accuracy.
+    - Flip rate (taux de changement de prédiction vs original)
+    - Consistency orig_pred vs perturbed_pred (accord modèle)
+    - Degradation relative vs baseline propre: (acc_clean - acc_corrupted) / acc_clean
+
+#### Robustesse (adversarial)
+    - Robust accuracy par attaque (transform_name) et intensité (epsilon/steps)
+    - Courbes accuracy vs epsilon et Aire Sous la Courbe (AUC)
+    - Worst-case accuracy sur l'ensemble des attaques
+    - Degradation relative vs baseline propre
+
+
+
+### Agrégations / découpages
+    - By split: train / val / test
+    - By model_name
+    - By class_name
+    - By transform_name et transform_level (sévérité)
+    - Moyennes pondérées par fréquence de classe si pertinent

kcai_data_sampling-0.7/src/kcai_data_sampling/_version_.py

@@ -0,0 +1,34 @@
+# file generated by setuptools-scm
+# don't change, don't track in version control
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    from typing import Tuple
+    from typing import Union
+
+    VERSION_TUPLE = Tuple[Union[int, str], ...]
+    COMMIT_ID = Union[str, None]
+else:
+    VERSION_TUPLE = object
+    COMMIT_ID = object
+
+version: str
+__version__: str
+__version_tuple__: VERSION_TUPLE
+version_tuple: VERSION_TUPLE
+commit_id: COMMIT_ID
+__commit_id__: COMMIT_ID
+
+__version__ = version = '0.7'
+__version_tuple__ = version_tuple = (0, 7)
+
+__commit_id__ = commit_id = 'g96ad968d8'

{kcai_data_sampling-0.0.6 → kcai_data_sampling-0.7}/src/kcai_data_sampling/adversarial_run.py

@@ -21,7 +21,7 @@ from cleverhans.torch.attacks.fast_gradient_method import fast_gradient_method
 from cleverhans.torch.attacks.projected_gradient_descent import projected_gradient_descent # PGD
 from cleverhans.torch.attacks.carlini_wagner_l2 import carlini_wagner_l2         # C&W (L2)
 # AutoAttack / APGD
-from autoattack import AutoAttack
+from pyautoattack import AutoAttack
 
 # def _seed32(*parts, base_seed: int = 0) -> int:
 #     h = hashlib.blake2b(digest_size=8)
@@ -196,11 +196,12 @@ def run_apgd(model, x, y, eps, n_restarts=1):
 def run_pipeline(cfg: Dict[str, Any]) -> str:
     ap = cfg["adversarial_pipeline"]
 
-    # model 
+    # model & préproc
     mcfg = ap["model"]
     resize_hw = mcfg.get("resize")
     mean = mcfg.get("mean", [0.0,0.0,0.0])
     std  = mcfg.get("std",  [1.0,1.0,1.0])
+
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     if device.type == "cuda":
         try:
@@ -276,7 +277,7 @@ def run_pipeline(cfg: Dict[str, Any]) -> str:
 
     attacks = ap["attacks"]
     all_records: List[Dict[str, Any]] = []  # Pour les métriques finales
-    
+
     # Calcul des bornes de clipping dans l'espace normalisé
     # Si input_range est [0,1] et on applique (x - mean) / std :
     # clip_min = (0 - mean) / std, clip_max = (1 - mean) / std
@@ -297,24 +298,24 @@ def run_pipeline(cfg: Dict[str, Any]) -> str:
     n = len(full_df)
     num_batches = (n + batch_size - 1) // batch_size
     total_operations = num_batches * len(attacks)
-    
+
     print(f"\n{'='*60}")
     print(f" --- Starting adversarial pipeline ---")
     print(f" 📊 {n} images | {len(attacks)} attacks | {num_batches} batches")
     print(f" ⚙️  Total: {total_operations} operations")
     print(f"{'='*60}\n")
-    
+
     pbar = tqdm(total=total_operations, desc="Global pipeline", unit="op", ncols=100)
-    
+
     # loop on attacks (for incremental writing)
     for attack_idx, a_cfg in enumerate(attacks):
         alias = a_cfg.get("alias", a_cfg["name"])
         aid = int(a_cfg.get("id", 0))
-        
+
         print(f"\n🎯 Attack {attack_idx+1}/{len(attacks)}: {alias} (id={aid})")
-        
+
         records: List[Dict[str, Any]] = []  # Records for this attack
-        
+
         # loop on batch for this attack
         for start in range(0, n, batch_size):
             part = full_df.iloc[start:start+batch_size]
@@ -371,7 +372,7 @@ def run_pipeline(cfg: Dict[str, Any]) -> str:
                 lr_cw = float(params.get("lr", 5e-3))
                 # C&W expects normalized input and works in the normalized space
                 x_adv_n = run_cw(model, x_in, y, steps=steps, c=c, k=k, lr=lr_cw, clip_min=clip_min, clip_max=clip_max)
-            elif alias.startswith("apgd") and lib == "autoattack":
+            elif alias.startswith("apgd") and lib == "pyautoattack":
                 eps = float(params["eps"])
                 n_restarts = int(params.get("n_restarts", 1))
                 # AutoAttack returns already preprocessed pixels for the model → here it's in the normalized space
@@ -512,13 +513,3 @@ def run_pipeline(cfg: Dict[str, Any]) -> str:
         else: mdf.to_parquet(mp, index=False, compression=mc)
 
     return out["table_output"]["path"]
-
-if __name__ == "__main__":
-    import argparse
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--config", required=True, help="Path to adversarial_pipeline.yaml")
-    args = parser.parse_args()
-    with open(args.config, "r") as f:
-        cfg = yaml.safe_load(f)
-    out_path = run_pipeline(cfg)
-    print("Wrote:", out_path)

{kcai_data_sampling-0.0.6 → kcai_data_sampling-0.7}/src/kcai_data_sampling/augment_run.py

@@ -48,7 +48,7 @@ def _predict_class(model, img_arr: np.ndarray, device, mean, std, resize):
     t = _to_tensor_for_model(img_arr, resize)
     t = t.unsqueeze(0).to(device)  # Add batch dimension
     t = _normalize_tensor(t, mean, std)
-    
+
     with torch.no_grad():
         logits = model(t)
         pred = logits.argmax(1).item()
@@ -795,13 +795,3 @@ def run_pipeline(cfg: Dict[str, Any]) -> str:
 
     mon.finish_all()
     return table_out_path
-
-if __name__ == "__main__":
-    import argparse
-    
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--config", required=True, help="Path to augment_pipeline.yaml")
-    args = parser.parse_args()
-    with open(args.config, "r") as f:
-        cfg = yaml.safe_load(f)
-    out_path = run_pipeline(cfg)

{kcai_data_sampling-0.0.6 → kcai_data_sampling-0.7}/src/kcai_data_sampling/kc_inference.py

@@ -14,7 +14,7 @@ from PIL import Image
 from tqdm import tqdm
 import yaml
 
-from augment_run import _to_tensor_for_model, _normalize_tensor
+from .augment_run import _to_tensor_for_model, _normalize_tensor
 
 
 def _pil_from_bytes(b: bytes) -> Image.Image:
@@ -351,7 +351,3 @@ def main():
     print("\nDone!")
     print(f"Output columns: {list(df_out.columns)}")
 
-
-if __name__ == "__main__":
-    main()
-

{kcai_data_sampling-0.0.6 → kcai_data_sampling-0.7}/src/kcai_data_sampling/run_augment_pipeline.py

@@ -646,13 +646,3 @@ def run_pipeline(cfg: Dict[str, Any]) -> str:
 
     mon.finish_all()
     return table_out_path
-
-if __name__ == "__main__":
-    import argparse
-    
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--config", required=True, help="Path to augment_pipeline.yaml")
-    args = parser.parse_args()
-    with open(args.config, "r") as f:
-        cfg = yaml.safe_load(f)
-    out_path = run_pipeline(cfg)

kcai_data_sampling-0.7/src/kcai_data_sampling.egg-info/PKG-INFO

@@ -0,0 +1,27 @@
+Metadata-Version: 2.4
+Name: kcai-data-sampling
+Version: 0.7
+Summary: Samples generation utilities managed with uv
+Author-email: Safenai <support@safenai.io>
+License-Expression: Apache-2.0
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: pyarrow>=22.0.0
+Requires-Dist: numpy>=1.20.0
+Requires-Dist: pillow>=8.0.0
+Requires-Dist: scikit-learn>=1.0.0
+Requires-Dist: torch>=2.8.0
+Requires-Dist: ipykernel>=6.29.5
+Requires-Dist: torchvision>=0.20.1
+Requires-Dist: captum==0.7.0
+Requires-Dist: torch_pruning>=1.3.7
+Requires-Dist: tqdm==4.66.5
+Requires-Dist: click==8.1.7
+Requires-Dist: pyyaml
+Requires-Dist: numpy<2
+Requires-Dist: neural-compressor==3.1.1
+Requires-Dist: thop
+Requires-Dist: tabulate
+Requires-Dist: albumentations>=2.0.8
+Requires-Dist: pyautoattack>=0.1.1
+Requires-Dist: boto3>=1.40.51

{kcai_data_sampling-0.0.6 → kcai_data_sampling-0.7}/src/kcai_data_sampling.egg-info/SOURCES.txt

@@ -1,7 +1,10 @@
-README.md
+.gitignore
 pyproject.toml
 requirements.txt
+test.ipynb
+src/kcai_data_sampling/Description_artefacts.md
 src/kcai_data_sampling/__init__.py
+src/kcai_data_sampling/_version_.py
 src/kcai_data_sampling/adversarial_run.py
 src/kcai_data_sampling/augment_run.py
 src/kcai_data_sampling/kc_inference.py

{kcai_data_sampling-0.0.6 → kcai_data_sampling-0.7}/src/kcai_data_sampling.egg-info/requires.txt

@@ -1,14 +1,12 @@
-pyarrow>=10.0.0
+pyarrow>=22.0.0
 numpy>=1.20.0
 pillow>=8.0.0
 scikit-learn>=1.0.0
-torch>=1.10.0
+torch>=2.8.0
 ipykernel>=6.29.5
 torchvision>=0.20.1
-torch==2.5.1
-torchvision==0.20.1
 captum==0.7.0
-torch_pruning==1.3.7
+torch_pruning>=1.3.7
 tqdm==4.66.5
 click==8.1.7
 pyyaml
@@ -16,8 +14,6 @@ numpy<2
 neural-compressor==3.1.1
 thop
 tabulate
-pyarrow==17.0.0
 albumentations>=2.0.8
-autoattack>=0.1
-cleverhans>=4.0.0
+pyautoattack>=0.1.1
 boto3>=1.40.51