colliderml 0.1.1__tar.gz

colliderml-0.1.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 ColliderML Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE. 

colliderml-0.1.1/MANIFEST.in

@@ -0,0 +1,8 @@
+include LICENSE
+include README.md
+include setup.cfg
+recursive-include tests *
+recursive-include docs *
+global-exclude *.py[cod]
+global-exclude __pycache__
+global-exclude *.so 

colliderml-0.1.1/PKG-INFO

@@ -0,0 +1,138 @@
+Metadata-Version: 2.2
+Name: colliderml
+Version: 0.1.1
+Summary: A modern machine learning library for high-energy physics data analysis
+Home-page: https://github.com/murnanedaniel/colliderml
+Author: Daniel Murnane
+Author-email: dtmurnane@lbl.gov
+Project-URL: Documentation, https://murnanedaniel.github.io/colliderml
+Project-URL: Source, https://github.com/murnanedaniel/colliderml
+Project-URL: Issues, https://github.com/murnanedaniel/colliderml/issues
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Scientific/Engineering :: Physics
+Requires-Python: >=3.10,<3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: requests>=2.31.0
+Requires-Dist: tqdm>=4.66.0
+Requires-Dist: numpy>=1.24.0
+Requires-Dist: pydantic>=2.5.0
+Requires-Dist: h5py>=3.10.0
+Provides-Extra: dev
+Requires-Dist: pytest>=7.4.0; extra == "dev"
+Requires-Dist: pytest-cov>=4.1.0; extra == "dev"
+Requires-Dist: black>=23.11.0; extra == "dev"
+Requires-Dist: ruff>=0.1.6; extra == "dev"
+Requires-Dist: mypy>=1.7.0; extra == "dev"
+Requires-Dist: mkdocs-material>=9.4.0; extra == "dev"
+Requires-Dist: mkdocstrings[python]>=0.24.0; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: project-url
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# ColliderML
+
+[![Tests](https://github.com/murnanedaniel/colliderml/actions/workflows/tests.yml/badge.svg)](https://github.com/murnanedaniel/colliderml/actions/workflows/tests.yml)
+![Coverage](./coverage.svg)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+A modern machine learning library for high-energy physics data analysis.
+
+## Features
+
+- Efficient parallel data downloading with resume capability
+- Support for common HEP data formats
+- Machine learning utilities for particle physics
+- Visualization tools for physics data
+
+## Installation
+
+### For Users
+```bash
+# Create and activate environment
+conda create -n collider-env python=3.11  # 3.10 or 3.11 recommended
+conda activate collider-env
+
+# Install package
+pip install colliderml
+```
+
+### For Developers
+```bash
+# Create and activate environment
+conda create -n collider-dev python=3.11  # 3.10 or 3.11 recommended
+conda activate collider-dev
+
+# Clone repository
+git clone https://github.com/murnanedaniel/colliderml.git
+cd colliderml
+
+# Install in development mode with extra dependencies
+pip install -e ".[dev]"
+```
+
+## Quick Start
+
+```python
+from colliderml.core.data.manifest import ManifestClient
+from colliderml.core.io import DataDownloader
+
+manifest = ManifestClient()
+files = manifest.select_files(campaign=None, datasets=["ttbar"], objects=["tracks"], max_events=1000)
+
+downloader = DataDownloader()
+results = downloader.download_files([f.path for f in files], local_dir="data", max_workers=4, resume=True)
+
+for path, result in results.items():
+    print(path, result.success, result.error)
+```
+
+### Features
+
+- **Manifest-driven**: Always selects files from the latest portal manifest
+- **Parallel Downloads**: Download multiple files concurrently
+- **Resume Capability**: Optionally resume interrupted downloads
+- **Progress Tracking**: Real-time progress bars
+- **Clear Errors**: Helpful failure messages and HEAD checks
+
+## Development
+
+1. Activate your environment:
+   ```bash
+   conda activate collider-dev
+   ```
+
+2. Run tests:
+   ```bash
+   # Run unit tests only
+   pytest -v -m "not integration"
+   
+   # Run all tests including integration tests
+   pytest -v
+   
+   # Run with coverage report
+   pytest --cov=colliderml
+   ```
+
+3. Build documentation:
+   ```bash
+   mkdocs build
+   mkdocs serve  # View at http://127.0.0.1:8000
+   ```
+
+## License
+
+[MIT License](LICENSE) 

colliderml-0.1.1/README.md

@@ -0,0 +1,94 @@
+# ColliderML
+
+[![Tests](https://github.com/murnanedaniel/colliderml/actions/workflows/tests.yml/badge.svg)](https://github.com/murnanedaniel/colliderml/actions/workflows/tests.yml)
+![Coverage](./coverage.svg)
+[![Python 3.10+](https://img.shields.io/badge/python-3.10%2B-blue.svg)](https://www.python.org/downloads/)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+A modern machine learning library for high-energy physics data analysis.
+
+## Features
+
+- Efficient parallel data downloading with resume capability
+- Support for common HEP data formats
+- Machine learning utilities for particle physics
+- Visualization tools for physics data
+
+## Installation
+
+### For Users
+```bash
+# Create and activate environment
+conda create -n collider-env python=3.11  # 3.10 or 3.11 recommended
+conda activate collider-env
+
+# Install package
+pip install colliderml
+```
+
+### For Developers
+```bash
+# Create and activate environment
+conda create -n collider-dev python=3.11  # 3.10 or 3.11 recommended
+conda activate collider-dev
+
+# Clone repository
+git clone https://github.com/murnanedaniel/colliderml.git
+cd colliderml
+
+# Install in development mode with extra dependencies
+pip install -e ".[dev]"
+```
+
+## Quick Start
+
+```python
+from colliderml.core.data.manifest import ManifestClient
+from colliderml.core.io import DataDownloader
+
+manifest = ManifestClient()
+files = manifest.select_files(campaign=None, datasets=["ttbar"], objects=["tracks"], max_events=1000)
+
+downloader = DataDownloader()
+results = downloader.download_files([f.path for f in files], local_dir="data", max_workers=4, resume=True)
+
+for path, result in results.items():
+    print(path, result.success, result.error)
+```
+
+### Features
+
+- **Manifest-driven**: Always selects files from the latest portal manifest
+- **Parallel Downloads**: Download multiple files concurrently
+- **Resume Capability**: Optionally resume interrupted downloads
+- **Progress Tracking**: Real-time progress bars
+- **Clear Errors**: Helpful failure messages and HEAD checks
+
+## Development
+
+1. Activate your environment:
+   ```bash
+   conda activate collider-dev
+   ```
+
+2. Run tests:
+   ```bash
+   # Run unit tests only
+   pytest -v -m "not integration"
+   
+   # Run all tests including integration tests
+   pytest -v
+   
+   # Run with coverage report
+   pytest --cov=colliderml
+   ```
+
+3. Build documentation:
+   ```bash
+   mkdocs build
+   mkdocs serve  # View at http://127.0.0.1:8000
+   ```
+
+## License
+
+[MIT License](LICENSE) 

colliderml-0.1.1/colliderml/__init__.py

@@ -0,0 +1,8 @@
+"""ColliderML: A modern machine learning library for high-energy physics data analysis."""
+
+__version__ = "0.1.1"
+
+from . import core
+from . import utils
+
+__all__ = ["core", "utils"] 

colliderml-0.1.1/colliderml/cli.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+"""ColliderML command line interface."""
+
+import argparse
+import sys
+from pathlib import Path
+from colliderml.core.io import DataDownloader
+from colliderml.core.data.manifest import ManifestClient
+
+
+def get(args):
+    """Handle the get command (manifest-driven)."""
+    downloader = DataDownloader()
+    manifest = ManifestClient()
+
+    # Determine campaign
+    campaign = args.campaign if args.campaign and args.campaign != "default" else None
+
+    # Parse lists
+    datasets = args.datasets.split(',') if args.datasets else None
+    objects = args.objects.split(',') if args.objects else None
+
+    # Select files
+    try:
+        files = manifest.select_files(
+            campaign=campaign,
+            datasets=datasets,
+            objects=objects,
+            max_events=args.events,
+            version=args.version,
+        )
+    except Exception as e:
+        print(f"\nError reading manifest: {e}")
+        sys.exit(1)
+
+    if not files:
+        print("No files matched the selection from the manifest.")
+        sys.exit(0)
+
+    print("\nGet Configuration:")
+    print(f"Campaign: {args.campaign or 'default'}")
+    print(f"Version: {args.version or 'dataset defaults'}")
+    print(f"Datasets: {', '.join(datasets) if datasets else 'ALL'}")
+    print(f"Objects: {', '.join(objects) if objects else 'ALL'}")
+    print(f"Requested events: {args.events if args.events else 'ALL'}")
+    print(f"Output directory: {args.output_dir}")
+
+    # Download
+    remote_paths = [f.path for f in files]
+    results = downloader.download_files(
+        remote_paths=remote_paths,
+        local_dir=args.output_dir,
+        max_workers=args.workers,
+        resume=not args.no_resume,
+    )
+
+    successful = [r for r in results.values() if r.success]
+    failed = [r for r in results.values() if not r.success]
+
+    print("\nGet Summary:")
+    print(f"Total files: {len(results)}")
+    print(f"Successful: {len(successful)}")
+    print(f"Failed: {len(failed)}")
+    if failed:
+        print("\nFailed downloads:")
+        for path, result in results.items():
+            if not result.success:
+                print(f"✗ {path}: {result.error}")
+        sys.exit(1)
+
+
+def main():
+    """Main entry point."""
+    parser = argparse.ArgumentParser(description="ColliderML command line interface")
+    subparsers = parser.add_subparsers(dest='command', help='Command to run')
+
+    # Get command (manifest-driven)
+    get_parser = subparsers.add_parser('get', help='Get files using manifest selection')
+    get_parser.add_argument('-c', '--campaign', type=str, default='default',
+                            help='Campaign name (or "default" to use manifest default)')
+    get_parser.add_argument('-d', '--datasets', type=str,
+                            help='Comma-separated list of datasets (e.g. ttbar,qcd)')
+    get_parser.add_argument('-o', '--objects', type=str,
+                            help='Comma-separated list of objects (e.g. tracks,hits)')
+    get_parser.add_argument('-e', '--events', type=int, default=None,
+                            help='Max number of events to download (across selection)')
+    get_parser.add_argument('-O', '--output-dir', '--output_dir', dest='output_dir', type=str, default='data',
+                            help='Directory to save downloaded files')
+    get_parser.add_argument('-w', '--workers', type=int, default=4,
+                            help='Number of parallel downloads')
+    get_parser.add_argument('--no-resume', '--no_resume', dest='no_resume', action='store_true',
+                            help='Disable resuming partial downloads')
+    get_parser.add_argument('-v', '--version', type=str, default=None,
+                            help='Dataset version to use (overrides dataset default_version)')
+
+    args = parser.parse_args()
+
+    if args.command == 'get':
+        get(args)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()
+
+

colliderml-0.1.1/colliderml/core/__init__.py

@@ -0,0 +1,7 @@
+"""Core functionality for ColliderML."""
+
+from . import io
+from . import data
+from .get import get
+
+__all__ = ["io", "data", "get"]

colliderml-0.1.1/colliderml/core/data/__init__.py

@@ -0,0 +1,21 @@
+"""Data handling functionality for ColliderML."""
+
+from .dataset import Dataset
+from .config import (
+    PileupLevel,
+    DataType,
+    OBJECT_CONFIGS,
+    VALID_PROCESSES,
+    BASE_URL,
+    MANIFEST_URL,
+)
+
+__all__ = [
+    "Dataset",
+    "PileupLevel",
+    "DataType",
+    "OBJECT_CONFIGS",
+    "VALID_PROCESSES",
+    "BASE_URL",
+    "MANIFEST_URL",
+]

colliderml-0.1.1/colliderml/core/data/config.py

@@ -0,0 +1,104 @@
+"""Configuration and lightweight enums/constants for ColliderML.
+
+This module provides base URLs and minimal legacy constants while we migrate
+to a manifest-driven data selection model.
+"""
+
+from __future__ import annotations
+
+from enum import Enum
+from typing import Dict, Set
+
+
+# Base URL configuration
+BASE_URL: str = "https://portal.nersc.gov/cfs/m4958/ColliderML/"
+MANIFEST_URL: str = f"{BASE_URL}manifest.json"
+
+# Downloader base URLs list (kept for backward compatibility)
+DEFAULT_URLS = [BASE_URL]
+
+
+# Legacy-style enums (kept to avoid breaking existing imports/tests)
+class PileupLevel(str, Enum):
+    SINGLE = "single-particle"
+    LOW = "pileup-10"
+    HIGH = "pileup-200"
+
+
+class DataType(str, Enum):
+    RECO = "reco"
+    TRUTH = "truth"
+    MEASUREMENTS = "measurements"
+
+
+# Minimal object configuration placeholder to satisfy existing imports
+class _ObjectConfig:
+    def __init__(self, data_type: DataType):
+        self.data_type = data_type
+
+
+# Representative objects; the authoritative list comes from the manifest
+OBJECT_CONFIGS: Dict[str, _ObjectConfig] = {
+    "tracks": _ObjectConfig(DataType.RECO),
+    "particle_flow": _ObjectConfig(DataType.RECO),
+    "particles": _ObjectConfig(DataType.TRUTH),
+    "tracker_hits": _ObjectConfig(DataType.MEASUREMENTS),
+}
+
+
+# Representative physics processes; authoritative set comes from the manifest
+VALID_PROCESSES: Set[str] = {"ttbar", "wjets", "zjets", "susy", "higgs", "qcd", "exotics"}
+
+
+# Legacy constants used by older CLI/tests; superseded by manifest data
+EVENTS_PER_FILE: int = 1000
+
+# Optional dataset sizes for summary planning (will be superseded by manifest totals)
+DATASET_SIZES: Dict[str, int] = {
+    "ttbar": 100_000,
+    "qcd": 100_000,
+}
+
+
+def get_object_path(
+    pileup: PileupLevel,
+    process: str,
+    object_name: str,
+    start_event: int,
+    end_event: int,
+) -> str:
+    """Build a legacy-style relative path for an object file.
+
+    This is maintained only for backward compatibility with existing tests and
+    scripts. New code should select paths from the manifest instead.
+    """
+
+    # Very lightweight validation to match test expectations
+    if process not in VALID_PROCESSES:
+        raise ValueError("Invalid process")
+    if object_name not in OBJECT_CONFIGS:
+        raise ValueError("Invalid object type")
+
+    # Use a fixed version segment to satisfy test path format
+    version = "v1"
+    data_type = OBJECT_CONFIGS[object_name].data_type.value
+    pileup_str = pileup.value if isinstance(pileup, PileupLevel) else str(pileup)
+    filename = f"{pileup_str}.{process}.{version}.{data_type}.{object_name}.events{start_event}-{end_event}.h5"
+    rel_path = f"{pileup_str}/{process}/{version}/{data_type}/{object_name}/{filename}"
+    return rel_path
+
+
+__all__ = [
+    "BASE_URL",
+    "MANIFEST_URL",
+    "DEFAULT_URLS",
+    "PileupLevel",
+    "DataType",
+    "OBJECT_CONFIGS",
+    "VALID_PROCESSES",
+    "EVENTS_PER_FILE",
+    "DATASET_SIZES",
+    "get_object_path",
+]
+
+

colliderml-0.1.1/colliderml/core/data/dataset.py

@@ -0,0 +1,45 @@
+"""Dataset class for handling HEP data."""
+
+from typing import Optional, List, Dict, Any
+from pathlib import Path
+
+class Dataset:
+    """Base class for handling HEP datasets."""
+    
+    def __init__(self, name: str, files: List[str]):
+        """Initialize a dataset.
+        
+        Args:
+            name: Name of the dataset.
+            files: List of file paths in the dataset.
+        """
+        self.name = name
+        self.files = files
+        self._metadata: Dict[str, Any] = {}
+        
+    def __len__(self) -> int:
+        """Get the number of files in the dataset."""
+        return len(self.files)
+        
+    def add_metadata(self, key: str, value: Any) -> None:
+        """Add metadata to the dataset.
+        
+        Args:
+            key: Metadata key.
+            value: Metadata value.
+        """
+        self._metadata[key] = value
+        
+    def get_metadata(self, key: str) -> Any:
+        """Get metadata from the dataset.
+        
+        Args:
+            key: Metadata key.
+            
+        Returns:
+            The metadata value.
+            
+        Raises:
+            KeyError: If the key doesn't exist.
+        """
+        return self._metadata[key]