lightray 0.2.3__tar.gz → 0.2.4__tar.gz

lightray-0.2.4/.flake8

@@ -0,0 +1,11 @@
+[flake8]
+max-line-length = 79
+max-complexity = 18
+select = B,C,E,F,W,T4,B9
+ignore = W503, E203  # ignore for consistency with black
+
+# ignore asterisk imports and unused
+# import errors in __init__ files
+per-file-ignores = 
+    **/__init__.py:F401,F403,E402
+    

lightray-0.2.4/.github/workflows/charts.yaml

@@ -0,0 +1,32 @@
+name: Release Charts
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+        - 'charts/**'
+
+jobs:
+  release:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config user.name "$GITHUB_ACTOR"
+          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+
+      - name: Run chart-releaser
+        uses: helm/chart-releaser-action@v1.6.0
+        env:
+          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

lightray-0.2.4/.github/workflows/pre-commit.yaml

@@ -0,0 +1,17 @@
+name: pre-commit
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      with:
+        python-version: "3.9"
+    - uses: pre-commit/action@v2.0.3

lightray-0.2.4/.github/workflows/publish.yaml

@@ -0,0 +1,27 @@
+name: Publish to PyPI
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    environment: PyPI
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: uv.lock
+
+      - name: Set up Python
+        run: uv python install 3.12
+
+      - name: Build
+        run: uv build
+
+      - name: Publish
+        run: uv publish -t ${{ secrets.PYPI_TOKEN }}

lightray-0.2.4/.github/workflows/unit-tests.yaml

@@ -0,0 +1,34 @@
+name: unit-tests
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.9', '3.10', '3.11']
+    steps:
+    - uses: actions/checkout@v2
+
+    # set up our environment
+    - 
+      name: Install Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    -
+      name: Install package
+      run: |
+        python -m pip install .
+        python -m pip install pytest
+
+    # run the library's tests
+    -
+      name: run tests
+      run: python -m pytest tests/

lightray-0.2.4/.gitignore

@@ -0,0 +1,2 @@
+**/__pycache__
+**/id_rsa

lightray-0.2.4/.pre-commit-config.yaml

@@ -0,0 +1,26 @@
+repos:
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.9.6
+  hooks:
+    - id: ruff
+      args: [ --fix ]
+    - id: ruff-format
+- repo: https://github.com/astral-sh/uv-pre-commit
+  # uv version.
+  rev: 0.6.0
+  hooks:
+    - id: uv-lock
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v2.3.0
+  hooks:
+  - id: end-of-file-fixer
+  - id: check-yaml
+    exclude: ^charts/raycluster/templates/
+- repo: https://github.com/gruntwork-io/pre-commit
+  rev: v0.1.15
+  hooks:
+    - id: helmlint
+- repo: https://github.com/dhruvmanila/remove-print-statements
+  rev: v0.5.2  
+  hooks:
+  - id: remove-print-statements

{lightray-0.2.3 → lightray-0.2.4}/PKG-INFO

@@ -1,25 +1,17 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: lightray
-Version: 0.2.3
+Version: 0.2.4
 Summary: Distribute a LightningCLI hyperparameter search with Ray Tune
-License: MIT
-Author: Ethan Jacob Marx
-Author-email: ethan.marx@ligo.org
-Requires-Python: >=3.9,<4.0
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
-Classifier: Programming Language :: Python :: 3.10
-Classifier: Programming Language :: Python :: 3.11
-Classifier: Programming Language :: Python :: 3.12
-Classifier: Programming Language :: Python :: 3.13
-Requires-Dist: jsonargparse[signatures] (>=4.33.2,<5.0.0)
-Requires-Dist: lightning (<=2.4.0)
-Requires-Dist: omegaconf (>=2.3.0,<3.0.0)
-Requires-Dist: pyarrow (>=17.0.0,<18.0.0)
-Requires-Dist: ray[default,tune] (>=2.8.0,<3.0.0)
-Requires-Dist: s3fs (>=2024.9.0,<2025.0.0)
-Requires-Dist: wandb (>=0.18.5,<0.19.0)
+Author-email: Ethan Jacob Marx <ethan.marx@ligo.org>
+License-Expression: MIT
+Requires-Python: ~=3.9
+Requires-Dist: jsonargparse[signatures]<5,>=4.33.2
+Requires-Dist: lightning>=2.4.0
+Requires-Dist: omegaconf<3,>=2.3.0
+Requires-Dist: pyarrow<18,>=17.0.0
+Requires-Dist: ray[default,tune]<3,>=2.8.0
+Requires-Dist: s3fs<2025,>=2024.9.0
+Requires-Dist: wandb<0.19,>=0.18.5
 Description-Content-Type: text/markdown
 
 # lightray
@@ -146,4 +138,3 @@ To connect to a remote ray cluster, pass the ip address (using port `10001`) to
 ```console
 lightray --config tune.yaml --ray_init.address ray://{ip}:10001 -- --config cli.yaml
 ```
-

lightray-0.2.4/charts/raycluster/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

lightray-0.2.4/charts/raycluster/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: ray-cluster
+description: Deploy static ray clusters with Kubernetes.
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.6
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

lightray-0.2.4/charts/raycluster/templates/external-service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-head-loadbalancer
+spec:
+  ports:
+  - name: client
+    port: 10001
+    protocol: TCP
+    targetPort: 10001
+  selector:
+    app: {{ .Chart.Name }}-head
+  type: LoadBalancer

lightray-0.2.4/charts/raycluster/templates/head-deployment.yaml

@@ -0,0 +1,102 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-head
+  labels:
+    app: {{ .Chart.Name }}-head
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      component: {{ .Chart.Name }}-head
+      type: ray
+      app: {{ .Chart.Name }}-head
+  template:
+    metadata:
+      labels:
+        component: {{ .Chart.Name }}-head
+        type: ray
+        app: {{ .Chart.Name }}-head
+    spec:
+      restartPolicy: Always
+      volumes:
+      - name: dshm
+        emptyDir:
+          medium: Memory
+      - name: {{ .Values.gitRepo.name }}-volume
+        emptyDir: {}
+      - name: {{ .Release.Name }}-git-secret
+        secret:
+          defaultMode: 0400
+          secretName: {{ .Release.Name }}-git-creds
+      securityContext:
+        fsGroup: 65533
+    {{- if .Values.dev }}
+      initContainers:
+        - name: git-sync
+          image: registry.k8s.io/git-sync/git-sync:v4.2.1
+          env:
+          - name: GITSYNC_REPO
+            value: {{ .Values.gitRepo.url }}
+          - name: GITSYNC_REF
+            value: {{ .Values.gitRepo.ref }} 
+          - name: GITSYNC_ROOT
+            value: {{ .Values.gitRepo.mountPath }}
+          - name: GITSYNC_LINK
+            value: {{ .Values.gitRepo.name }}
+          - name: GITSYNC_ONE_TIME
+            value: "true"
+          - name: GITSYNC_SSH_KNOWN_HOSTS
+            value: "false"
+          - name: GITSYNC_SUBMODULES
+            value: "recursive"
+          - name: GITSYNC_ADD_USER
+            value: "true"
+          - name: GITSYNC_SYNC_TIMEOUT
+            value: "360s"
+          volumeMounts:
+            - name: {{ .Values.gitRepo.name }}-volume
+              mountPath: {{ .Values.gitRepo.mountPath }}
+            - name: {{ .Release.Name }}-git-secret
+              mountPath: /etc/git-secret
+              readOnly: true
+          securityContext:
+            runAsUser: 65533
+    {{- end }}
+      containers:
+        - name: ray-head
+          image: {{ .Values.image }}
+          imagePullPolicy: {{ .Values.imagePullPolicy }}
+          command: [ "/bin/bash", "-c", "--" ]
+          args:
+            - "ray start --head --port=6380 --num-cpus=$MY_CPU_REQUEST --dashboard-host=0.0.0.0 --object-manager-port=8076 --node-manager-port=8077 --dashboard-agent-grpc-port=8078 --dashboard-agent-listen-port=52365 --block"
+          ports:
+            - containerPort: 6380
+            - containerPort: 10001 
+            - containerPort: 8265 
+          volumeMounts:
+            - mountPath: /dev/shm
+              name: dshm
+            - mountPath: {{ .Values.gitRepo.mountPath }}
+              name: {{ .Values.gitRepo.name }}-volume
+          env:
+          - name: TUNE_MAX_PENDING_TRIALS_PG
+            value: "64"
+          - name: MY_CPU_REQUEST
+            valueFrom:
+              resourceFieldRef:
+                resource: requests.cpu
+          - name: AWS_ENDPOINT_URL
+            value: {{ .Values.awsUrl }} 
+          envFrom:
+          - secretRef:
+              name: {{ .Release.Name }}-credentials
+          resources:
+            limits:
+              cpu: {{ .Values.head.cpu | quote }}
+              memory: {{ .Values.head.memory | quote }}
+              ephemeral-storage: "100Gi"
+            requests:
+              cpu: {{ .Values.head.cpu | quote }}
+              memory: {{ .Values.head.memory | quote }}
+              ephemeral-storage: "100Gi"

lightray-0.2.4/charts/raycluster/templates/internal-service.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-service
+  labels:
+    app: {{ .Chart.Name }}-head
+spec:
+  clusterIP: None
+  ports:
+  - name: client
+    protocol: TCP
+    port: 10001
+    targetPort: 10001
+  - name: dashboard
+    protocol: TCP
+    port: 8265
+    targetPort: 8265
+  - name: gcs-server
+    protocol: TCP
+    port: 6380
+    targetPort: 6380
+  selector:
+    app: {{ .Chart.Name }}-head

lightray-0.2.4/charts/raycluster/templates/secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-credentials
+type: Opaque
+stringData:
+  WANDB_API_KEY: {{ .Values.secret.wandbKey }}
+  AWS_ACCESS_KEY_ID: {{ .Values.secret.awsId }}
+  AWS_SECRET_ACCESS_KEY: {{ .Values.secret.awsKey }}

lightray-0.2.4/charts/raycluster/templates/ssh-secret.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.dev }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-git-creds
+type: Opaque
+data:
+  ssh: {{ .Values.gitRepo.sshKey | b64enc }}
+  known_hosts: {{ .Values.gitRepo.known_hosts | quote | b64enc | indent 2 }}
+{{- end }}

lightray-0.2.4/charts/raycluster/templates/worker-deployment.yaml

@@ -0,0 +1,109 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-worker
+  labels:
+    app: {{ .Chart.Name }}-worker
+spec:
+  replicas: {{ .Values.worker.replicas }}
+  selector:
+    matchLabels:
+      component: {{ .Chart.Name }}-worker
+      type: ray
+      app: {{ .Chart.Name }}-worker
+  template:
+    metadata:
+      labels:
+        component: {{ .Chart.Name }}-worker
+        type: ray
+        app: {{ .Chart.Name }}-worker
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: nvidia.com/gpu.memory
+                operator: Gt
+                values:
+                - {{ .Values.worker.min_gpu_memory | quote }}
+      restartPolicy: Always
+      volumes:
+      - name: dshm
+        emptyDir:
+          medium: Memory
+      - name: {{ .Values.gitRepo.name }}-volume
+        emptyDir: {}
+      - name: {{ .Release.Name }}-git-secret
+        secret:
+          defaultMode: 0400
+          secretName: {{ .Release.Name }}-git-creds
+      securityContext:
+        fsGroup: 65533
+    {{- if .Values.dev }}
+      initContainers:
+        - name: git-sync
+          image: registry.k8s.io/git-sync/git-sync:v4.2.1
+          env:
+          - name: GITSYNC_REPO
+            value: {{ .Values.gitRepo.url }}
+          - name: GITSYNC_REF
+            value: {{ .Values.gitRepo.ref }}
+          - name: GITSYNC_ROOT
+            value: {{ .Values.gitRepo.mountPath }}
+          - name: GITSYNC_LINK
+            value: {{ .Values.gitRepo.name }}
+          - name: GITSYNC_ONE_TIME
+            value: "true"
+          - name: GITSYNC_SSH_KNOWN_HOSTS
+            value: "false"
+          - name: GITSYNC_SUBMODULES
+            value: "recursive"
+          - name: GITSYNC_ADD_USER
+            value: "true"
+          - name: GITSYNC_SYNC_TIMEOUT
+            value: "360s"
+          volumeMounts:
+            - name: {{ .Values.gitRepo.name }}-volume
+              mountPath: {{ .Values.gitRepo.mountPath }}
+            - name: {{ .Release.Name }}-git-secret
+              mountPath: /etc/git-secret
+              readOnly: true
+          securityContext:
+            runAsUser: 65533
+    {{- end }}
+      containers:
+      - name: ray-worker
+        image: {{ .Values.image  }}
+        imagePullPolicy: {{ .Values.imagePullPolicy  }} 
+        command: ["/bin/bash", "-c", "--"]
+        args:
+          - "ray start --num-cpus=$MY_CPU_REQUEST --address={{ .Release.Name }}-service:6380 --object-manager-port=8076 --node-manager-port=8077 --dashboard-agent-grpc-port=8078 --dashboard-agent-listen-port=52365 --block"
+        volumeMounts:
+          - mountPath: /dev/shm
+            name: dshm
+        {{- if .Values.dev }}
+          - mountPath: {{ .Values.gitRepo.mountPath }}
+            name: {{ .Values.gitRepo.name }}-volume
+        {{- end }}
+        env:
+        - name: MY_CPU_REQUEST
+          valueFrom:
+            resourceFieldRef:
+              resource: requests.cpu
+        - name: AWS_ENDPOINT_URL
+          value: {{ .Values.awsUrl }}
+        envFrom:
+        - secretRef:
+            name: {{ .Release.Name }}-credentials
+        resources:
+            limits:
+              nvidia.com/gpu: {{ .Values.worker.gpu | quote }}
+              cpu: {{ .Values.worker.cpu | quote }}
+              ephemeral-storage: 110Gi
+              memory: {{ .Values.worker.memory | quote }}
+            requests:
+              nvidia.com/gpu: {{ .Values.worker.gpu | quote }}
+              cpu: {{ .Values.worker.cpu | quote }}
+              ephemeral-storage: 100Gi
+              memory: {{ .Values.worker.memory | quote }}

lightray-0.2.4/charts/raycluster/values.yaml

@@ -0,0 +1,53 @@
+# release name
+name: ray-cluster
+# image for worker and head pods
+image: rayproject/ray:2.3.0
+# whether or not to always pull image from remote repo
+imagePullPolicy: Always
+
+# dev specifies if remote code should be mounted  
+dev: false 
+
+# if dev is true, the repository that will be mounted
+gitRepo:
+  # git url for repo
+  url: git@github.com:ML4GW/aframev2.git
+  # location where git-sync will store data
+  mountPath: /opt
+  # name relative to mountPath where code will be cloned
+  # and, identifier for git repo in yaml
+  name: aframe
+  # branch or commit on git repo
+  ref: main
+  # ssh key that will be mapped into container for cloning 
+  sshKey: ""
+
+head:
+  # number of cpus for the head pod
+  cpu: "32"
+  # memory for the head pod
+  memory: "32G"
+
+worker:
+  # number of worker pods to create
+  replicas: 1
+  # number of cpus per work pod
+  cpu: "32"
+  # memory per worker pod
+  memory: "32G"
+  # gpus per worker pod
+  gpu: "1"
+  # minimum amount of gpu memory
+  min_gpu_memory: "15000"
+
+# url for s3 cluster
+awsUrl: http://rook-ceph-rgw-nautiluss3.rook
+
+# secret that will be used with worker pods
+secret:
+  # value of WANDB_API_KEY environment variable
+  wandbKey: "" 
+  # value of AWS_ACCESS_KEY_ID environment variable
+  awsId: "" 
+  # value of AWS_SECRET_ACCESS_KEY environment variable
+  awsKey: "" 

lightray-0.2.4/example/cli.yaml

@@ -0,0 +1,20 @@
+trainer:
+  max_epochs: 100
+  logger: 
+    class_path: lightning.pytorch.loggers.CSVLogger
+    init_args:
+      save_dir: "logs/"
+
+data:
+  class_path: example.lightning.SimpleDataModule
+  init_args:
+    data_dim: 10
+    num_samples: 1000
+    batch_size: 32
+
+model:
+  class_path: example.lightning.SimpleModel
+  init_args:
+    data_dim: 10
+    hidden_size: 32
+    learning_rate: 0.01

lightray-0.2.4/example/example.yaml

@@ -0,0 +1,54 @@
+# ray.tune.TuneConfig
+tune_config:
+  mode: "min"
+  metric: "val_loss"
+  scheduler: 
+    class_path: ray.tune.schedulers.ASHAScheduler
+    init_args:
+      max_t: 200
+      grace_period: 21
+      reduction_factor: 2
+  num_samples: 8
+  reuse_actors: true
+
+# ray.train.RunConfig
+run_config:
+  name: "my-first-run"
+  storage_path: s3://aframe-test/new-tune/
+  failure_config:
+    class_path: ray.train.FailureConfig
+    init_args:
+      max_failures: 1
+  checkpoint_config:
+    class_path: ray.train.CheckpointConfig
+    init_args:
+      num_to_keep: 5
+      checkpoint_score_attribute: "val_loss"
+      checkpoint_score_order: "min"
+  verbose: null
+
+# ray.train.SyncConfig
+sync_config:
+  sync_period: 1000
+
+# ray.init
+ray_init:
+  address: null
+  
+# tune.Tune.param_space
+param_space:
+  model.learning_rate: tune.loguniform(1e-3, 4)
+
+# ray.tune.TuneCallback
+tune_callback:
+  class_path: lightray.callbacks.LightRayReportCheckpointCallback
+  init_args:
+    'on': "validation_end"
+    checkpoint_every: 10
+
+# resources per trial
+cpus_per_trial: 2
+gpus_per_trial: 1
+
+# lightning cli cls
+lightning_cli_cls: example.lightning.Cli