particleflow 3.1.0__py3-none-any.whl

habana/gaudi-pod-python-v19-1hpu.yaml

@@ -0,0 +1,101 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mlpf-hpu-strategy-v19-1hpu-hvd-constbatch-bm2
+spec:
+  completions: 1
+  parallelism: 1
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: jduarte
+      nodeSelector:
+        brightcomputing.com/node-category: "gaudi"
+      hostNetwork: false
+      volumes:
+        - name: home
+          hostPath:
+            path: /home/jduarte
+            type: Directory
+        - name: ceph
+          hostPath:
+            path: /voyager/ceph/users/jduarte
+            type: Directory
+        - name: scratch
+          emptyDir: {}
+      imagePullSecrets:
+        - name: registry-credentials
+      containers:
+        - name: htf2110-190-580-20230327-ubuntu2004
+          image: jmduarte/particleflow:habana_v19
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 48
+              memory: 384Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 256Gi
+            limits:
+              cpu: 96
+              memory: 396Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 512Gi
+          volumeMounts:
+            - name: home
+              mountPath: /home/jduarte
+            - name: ceph
+              mountPath: /voyager/ceph/users/jduarte
+            - name: scratch
+              mountPath: /scratch
+          env:
+            - name: POD_NAME_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NODE_HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HOME
+              value: "/home/jduarte"
+            - name: CEPH
+              value: "/voyager/ceph/users/jduarte"
+            - name: LOCAL_SCRATCH_DIR
+              value: "/scratch"
+            - name: MPI_ROOT
+              value: "/opt/amazon/openmpi"
+            - name: TFDS_DATA_DIR
+              value: "/voyager/ceph/users/jduarte/tensorflow_datasets"
+          workingDir: /home/jduarte/particleflow
+          command: ["/bin/bash", "-c"]
+          args:
+            - >-
+              declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')";
+              declare -xir UNIX_TIME="$(date +'%s')";
+
+              declare -xr VGR_POD_ID="${POD_NAME_ID}.$(date +'%s').${RANDOM}.${POD_NODE_HOSTNAME}";
+
+              declare -xr K8S_JOB_YAML_FILE="${PWD}/gaudi-pod-python-v19.yaml";
+              declare -xr K8S_JOB_YAML_MD5SUM="$(md5sum ${K8S_JOB_YAML_FILE})";
+
+              echo "${UNIX_TIME} ${VGR_POD_ID} ${K8S_JOB_YAML_MD5SUM}";
+              echo "";
+
+              cat "${K8S_JOB_YAML_FILE}";
+
+              printenv;
+
+              cat /etc/os-release;
+              lscpu;
+              free -h;
+              cat /proc/meminfo;
+              lsblk --output-all;
+              cat /etc/fstab;
+              lspci -vvv;
+              hl-smi;
+              hl-smi -q;
+
+              time -p mpirun -n 1 --allow-run-as-root --prefix "${MPI_ROOT}" -x "${VGR_POD_ID}" python3 -u mlpf/pipeline.py train -g -m -c parameters/clic-test.yaml --plot-freq 0 --batch-multiplier 2 --ntrain 50000 --ntest 50000 --nepochs 11 --benchmark_dir exp_dir;

habana/gaudi-pod-python-v19-2hpu.yaml

@@ -0,0 +1,101 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mlpf-hpu-strategy-v19-2hpu-constbatch-bm2
+spec:
+  completions: 1
+  parallelism: 1
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: jduarte
+      nodeSelector:
+        brightcomputing.com/node-category: "gaudi"
+      hostNetwork: false
+      volumes:
+        - name: home
+          hostPath:
+            path: /home/jduarte
+            type: Directory
+        - name: ceph
+          hostPath:
+            path: /voyager/ceph/users/jduarte
+            type: Directory
+        - name: scratch
+          emptyDir: {}
+      imagePullSecrets:
+        - name: registry-credentials
+      containers:
+        - name: htf2110-190-580-20230327-ubuntu2004
+          image: jmduarte/particleflow:habana_v19
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 48
+              memory: 384Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 256Gi
+            limits:
+              cpu: 96
+              memory: 396Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 512Gi
+          volumeMounts:
+            - name: home
+              mountPath: /home/jduarte
+            - name: ceph
+              mountPath: /voyager/ceph/users/jduarte
+            - name: scratch
+              mountPath: /scratch
+          env:
+            - name: POD_NAME_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NODE_HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HOME
+              value: "/home/jduarte"
+            - name: CEPH
+              value: "/voyager/ceph/users/jduarte"
+            - name: LOCAL_SCRATCH_DIR
+              value: "/scratch"
+            - name: MPI_ROOT
+              value: "/opt/amazon/openmpi"
+            - name: TFDS_DATA_DIR
+              value: "/voyager/ceph/users/jduarte/tensorflow_datasets"
+          workingDir: /home/jduarte/particleflow
+          command: ["/bin/bash", "-c"]
+          args:
+            - >-
+              declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')";
+              declare -xir UNIX_TIME="$(date +'%s')";
+
+              declare -xr VGR_POD_ID="${POD_NAME_ID}.$(date +'%s').${RANDOM}.${POD_NODE_HOSTNAME}";
+
+              declare -xr K8S_JOB_YAML_FILE="${PWD}/gaudi-pod-python-v19.yaml";
+              declare -xr K8S_JOB_YAML_MD5SUM="$(md5sum ${K8S_JOB_YAML_FILE})";
+
+              echo "${UNIX_TIME} ${VGR_POD_ID} ${K8S_JOB_YAML_MD5SUM}";
+              echo "";
+
+              cat "${K8S_JOB_YAML_FILE}";
+
+              printenv;
+
+              cat /etc/os-release;
+              lscpu;
+              free -h;
+              cat /proc/meminfo;
+              lsblk --output-all;
+              cat /etc/fstab;
+              lspci -vvv;
+              hl-smi;
+              hl-smi -q;
+
+              time -p mpirun -n 2 --allow-run-as-root --prefix "${MPI_ROOT}" -x "${VGR_POD_ID}" python3 -u mlpf/pipeline.py train -g -m -c parameters/clic-test.yaml --plot-freq 0 --batch-multiplier 2 --ntrain 50000 --ntest 50000 --nepochs 11 --benchmark_dir exp_dir;

habana/gaudi-pod-python-v19-3hpu.yaml

@@ -0,0 +1,101 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mlpf-hpu-strategy-v19-3hpu-constbatch-bm2
+spec:
+  completions: 1
+  parallelism: 1
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: jduarte
+      nodeSelector:
+        brightcomputing.com/node-category: "gaudi"
+      hostNetwork: false
+      volumes:
+        - name: home
+          hostPath:
+            path: /home/jduarte
+            type: Directory
+        - name: ceph
+          hostPath:
+            path: /voyager/ceph/users/jduarte
+            type: Directory
+        - name: scratch
+          emptyDir: {}
+      imagePullSecrets:
+        - name: registry-credentials
+      containers:
+        - name: htf2110-190-580-20230327-ubuntu2004
+          image: jmduarte/particleflow:habana_v19
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 48
+              memory: 384Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 256Gi
+            limits:
+              cpu: 96
+              memory: 396Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 512Gi
+          volumeMounts:
+            - name: home
+              mountPath: /home/jduarte
+            - name: ceph
+              mountPath: /voyager/ceph/users/jduarte
+            - name: scratch
+              mountPath: /scratch
+          env:
+            - name: POD_NAME_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NODE_HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HOME
+              value: "/home/jduarte"
+            - name: CEPH
+              value: "/voyager/ceph/users/jduarte"
+            - name: LOCAL_SCRATCH_DIR
+              value: "/scratch"
+            - name: MPI_ROOT
+              value: "/opt/amazon/openmpi"
+            - name: TFDS_DATA_DIR
+              value: "/voyager/ceph/users/jduarte/tensorflow_datasets"
+          workingDir: /home/jduarte/particleflow
+          command: ["/bin/bash", "-c"]
+          args:
+            - >-
+              declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')";
+              declare -xir UNIX_TIME="$(date +'%s')";
+
+              declare -xr VGR_POD_ID="${POD_NAME_ID}.$(date +'%s').${RANDOM}.${POD_NODE_HOSTNAME}";
+
+              declare -xr K8S_JOB_YAML_FILE="${PWD}/gaudi-pod-python-v19.yaml";
+              declare -xr K8S_JOB_YAML_MD5SUM="$(md5sum ${K8S_JOB_YAML_FILE})";
+
+              echo "${UNIX_TIME} ${VGR_POD_ID} ${K8S_JOB_YAML_MD5SUM}";
+              echo "";
+
+              cat "${K8S_JOB_YAML_FILE}";
+
+              printenv;
+
+              cat /etc/os-release;
+              lscpu;
+              free -h;
+              cat /proc/meminfo;
+              lsblk --output-all;
+              cat /etc/fstab;
+              lspci -vvv;
+              hl-smi;
+              hl-smi -q;
+
+              time -p mpirun -n 3 --allow-run-as-root --prefix "${MPI_ROOT}" -x "${VGR_POD_ID}" python3 -u mlpf/pipeline.py train -g -m -c parameters/clic-test.yaml --plot-freq 0 --batch-multiplier 2 --ntrain 50000 --ntest 50000 --nepochs 11 --benchmark_dir exp_dir;

habana/gaudi-pod-python-v19-4hpu.yaml

@@ -0,0 +1,101 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mlpf-hpu-strategy-v19-4hpu-constbatch-bm2
+spec:
+  completions: 1
+  parallelism: 1
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: jduarte
+      nodeSelector:
+        brightcomputing.com/node-category: "gaudi"
+      hostNetwork: false
+      volumes:
+        - name: home
+          hostPath:
+            path: /home/jduarte
+            type: Directory
+        - name: ceph
+          hostPath:
+            path: /voyager/ceph/users/jduarte
+            type: Directory
+        - name: scratch
+          emptyDir: {}
+      imagePullSecrets:
+        - name: registry-credentials
+      containers:
+        - name: htf2110-190-580-20230327-ubuntu2004
+          image: jmduarte/particleflow:habana_v19
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 48
+              memory: 384Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 256Gi
+            limits:
+              cpu: 96
+              memory: 396Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 512Gi
+          volumeMounts:
+            - name: home
+              mountPath: /home/jduarte
+            - name: ceph
+              mountPath: /voyager/ceph/users/jduarte
+            - name: scratch
+              mountPath: /scratch
+          env:
+            - name: POD_NAME_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NODE_HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HOME
+              value: "/home/jduarte"
+            - name: CEPH
+              value: "/voyager/ceph/users/jduarte"
+            - name: LOCAL_SCRATCH_DIR
+              value: "/scratch"
+            - name: MPI_ROOT
+              value: "/opt/amazon/openmpi"
+            - name: TFDS_DATA_DIR
+              value: "/voyager/ceph/users/jduarte/tensorflow_datasets"
+          workingDir: /home/jduarte/particleflow
+          command: ["/bin/bash", "-c"]
+          args:
+            - >-
+              declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')";
+              declare -xir UNIX_TIME="$(date +'%s')";
+
+              declare -xr VGR_POD_ID="${POD_NAME_ID}.$(date +'%s').${RANDOM}.${POD_NODE_HOSTNAME}";
+
+              declare -xr K8S_JOB_YAML_FILE="${PWD}/gaudi-pod-python-v19.yaml";
+              declare -xr K8S_JOB_YAML_MD5SUM="$(md5sum ${K8S_JOB_YAML_FILE})";
+
+              echo "${UNIX_TIME} ${VGR_POD_ID} ${K8S_JOB_YAML_MD5SUM}";
+              echo "";
+
+              cat "${K8S_JOB_YAML_FILE}";
+
+              printenv;
+
+              cat /etc/os-release;
+              lscpu;
+              free -h;
+              cat /proc/meminfo;
+              lsblk --output-all;
+              cat /etc/fstab;
+              lspci -vvv;
+              hl-smi;
+              hl-smi -q;
+
+              time -p mpirun -n 4 --allow-run-as-root --prefix "${MPI_ROOT}" -x "${VGR_POD_ID}" python3 -u mlpf/pipeline.py train -g -m -c parameters/clic-test.yaml --plot-freq 0 --batch-multiplier 2 --ntrain 50000 --ntest 50000 --nepochs 11 --benchmark_dir exp_dir;

habana/gaudi-pod-python-v19-5hpu.yaml

@@ -0,0 +1,101 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mlpf-hpu-strategy-v19-5hpu-constbatch-bm2
+spec:
+  completions: 1
+  parallelism: 1
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: jduarte
+      nodeSelector:
+        brightcomputing.com/node-category: "gaudi"
+      hostNetwork: false
+      volumes:
+        - name: home
+          hostPath:
+            path: /home/jduarte
+            type: Directory
+        - name: ceph
+          hostPath:
+            path: /voyager/ceph/users/jduarte
+            type: Directory
+        - name: scratch
+          emptyDir: {}
+      imagePullSecrets:
+        - name: registry-credentials
+      containers:
+        - name: htf2110-190-580-20230327-ubuntu2004
+          image: jmduarte/particleflow:habana_v19
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 48
+              memory: 384Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 256Gi
+            limits:
+              cpu: 96
+              memory: 396Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 512Gi
+          volumeMounts:
+            - name: home
+              mountPath: /home/jduarte
+            - name: ceph
+              mountPath: /voyager/ceph/users/jduarte
+            - name: scratch
+              mountPath: /scratch
+          env:
+            - name: POD_NAME_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NODE_HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HOME
+              value: "/home/jduarte"
+            - name: CEPH
+              value: "/voyager/ceph/users/jduarte"
+            - name: LOCAL_SCRATCH_DIR
+              value: "/scratch"
+            - name: MPI_ROOT
+              value: "/opt/amazon/openmpi"
+            - name: TFDS_DATA_DIR
+              value: "/voyager/ceph/users/jduarte/tensorflow_datasets"
+          workingDir: /home/jduarte/particleflow
+          command: ["/bin/bash", "-c"]
+          args:
+            - >-
+              declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')";
+              declare -xir UNIX_TIME="$(date +'%s')";
+
+              declare -xr VGR_POD_ID="${POD_NAME_ID}.$(date +'%s').${RANDOM}.${POD_NODE_HOSTNAME}";
+
+              declare -xr K8S_JOB_YAML_FILE="${PWD}/gaudi-pod-python-v19.yaml";
+              declare -xr K8S_JOB_YAML_MD5SUM="$(md5sum ${K8S_JOB_YAML_FILE})";
+
+              echo "${UNIX_TIME} ${VGR_POD_ID} ${K8S_JOB_YAML_MD5SUM}";
+              echo "";
+
+              cat "${K8S_JOB_YAML_FILE}";
+
+              printenv;
+
+              cat /etc/os-release;
+              lscpu;
+              free -h;
+              cat /proc/meminfo;
+              lsblk --output-all;
+              cat /etc/fstab;
+              lspci -vvv;
+              hl-smi;
+              hl-smi -q;
+
+              time -p mpirun -n 5 --allow-run-as-root --prefix "${MPI_ROOT}" -x "${VGR_POD_ID}" python3 -u mlpf/pipeline.py train -g -m -c parameters/clic-test.yaml --plot-freq 0 --batch-multiplier 2 --ntrain 50000 --ntest 50000 --nepochs 11 --benchmark_dir exp_dir;

habana/gaudi-pod-python-v19-6hpu.yaml

@@ -0,0 +1,101 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: mlpf-hpu-strategy-v19-6hpu-constbatch-bm2
+spec:
+  completions: 1
+  parallelism: 1
+  backoffLimit: 0
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: jduarte
+      nodeSelector:
+        brightcomputing.com/node-category: "gaudi"
+      hostNetwork: false
+      volumes:
+        - name: home
+          hostPath:
+            path: /home/jduarte
+            type: Directory
+        - name: ceph
+          hostPath:
+            path: /voyager/ceph/users/jduarte
+            type: Directory
+        - name: scratch
+          emptyDir: {}
+      imagePullSecrets:
+        - name: registry-credentials
+      containers:
+        - name: htf2110-190-580-20230327-ubuntu2004
+          image: jmduarte/particleflow:habana_v19
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: 48
+              memory: 384Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 256Gi
+            limits:
+              cpu: 96
+              memory: 396Gi
+              habana.ai/gaudi: 8
+              hugepages-2Mi: 96000Mi
+              ephemeral-storage: 512Gi
+          volumeMounts:
+            - name: home
+              mountPath: /home/jduarte
+            - name: ceph
+              mountPath: /voyager/ceph/users/jduarte
+            - name: scratch
+              mountPath: /scratch
+          env:
+            - name: POD_NAME_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NODE_HOSTNAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HOME
+              value: "/home/jduarte"
+            - name: CEPH
+              value: "/voyager/ceph/users/jduarte"
+            - name: LOCAL_SCRATCH_DIR
+              value: "/scratch"
+            - name: MPI_ROOT
+              value: "/opt/amazon/openmpi"
+            - name: TFDS_DATA_DIR
+              value: "/voyager/ceph/users/jduarte/tensorflow_datasets"
+          workingDir: /home/jduarte/particleflow
+          command: ["/bin/bash", "-c"]
+          args:
+            - >-
+              declare -xr LOCAL_TIME="$(date +'%Y%m%dT%H%M%S%z')";
+              declare -xir UNIX_TIME="$(date +'%s')";
+
+              declare -xr VGR_POD_ID="${POD_NAME_ID}.$(date +'%s').${RANDOM}.${POD_NODE_HOSTNAME}";
+
+              declare -xr K8S_JOB_YAML_FILE="${PWD}/gaudi-pod-python-v19.yaml";
+              declare -xr K8S_JOB_YAML_MD5SUM="$(md5sum ${K8S_JOB_YAML_FILE})";
+
+              echo "${UNIX_TIME} ${VGR_POD_ID} ${K8S_JOB_YAML_MD5SUM}";
+              echo "";
+
+              cat "${K8S_JOB_YAML_FILE}";
+
+              printenv;
+
+              cat /etc/os-release;
+              lscpu;
+              free -h;
+              cat /proc/meminfo;
+              lsblk --output-all;
+              cat /etc/fstab;
+              lspci -vvv;
+              hl-smi;
+              hl-smi -q;
+
+              time -p mpirun -n 6 --allow-run-as-root --prefix "${MPI_ROOT}" -x "${VGR_POD_ID}" python3 -u mlpf/pipeline.py train -g -m -c parameters/clic-test.yaml --plot-freq 0 --batch-multiplier 2 --ntrain 50000 --ntest 50000 --nepochs 11 --benchmark_dir exp_dir;