Bash Script to monitor Nvidia GPU Utilization

#!/bin/bash

# Slack webhook URL

SLACK_WEBHOOK_URL=”https://hooks.slack.com/triggers/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"

# Function to send alert to Slack

send_alert() {

. local pid=$1

. local mem_utilization=$2

. curl -X POST -H ‘Content-type: application/json’ \

. — data “{\”message\”: \”ALERT: High GPU memory utilization detected! Process ID: ${pid}, MEM: ${mem_utilization}%\”}” \

. $SLACK_WEBHOOK_URL

}

# Run nvidia-smi to get process information

output=$(nvidia-smi pmon -s u -c 1 | grep -vE ‘gpu|Idx’ | awk ‘{print $2 “ “ $5}’ | sed “s/%//g; s/-/0/g”)

# Process the output line by line

echo “$output” | while read -r line; do

. pid=$(echo “$line” | awk ‘{print $1}’) # Extract process ID (PID)

. mem=$(echo “$line” | awk ‘{print $2}’) # Extract memory utilization

. # Check if memory utilization exceeds 5%

. if [[ $mem -gt 5 ]]; then

. send_alert “$pid” “$mem”

. fi

done