Bash Script to monitor Nvidia GPU Utilization
#!/bin/bash
# Slack webhook URL
SLACK_WEBHOOK_URL=”https://hooks.slack.com/triggers/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"
# Function to send alert to Slack
send_alert() {
. local pid=$1
. local mem_utilization=$2
. curl -X POST -H ‘Content-type: application/json’ \
. — data “{\”message\”: \”ALERT: High GPU memory utilization detected! Process ID: ${pid}, MEM: ${mem_utilization}%\”}” \
. $SLACK_WEBHOOK_URL
}
# Run nvidia-smi to get process information
output=$(nvidia-smi pmon -s u -c 1 | grep -vE ‘gpu|Idx’ | awk ‘{print $2 “ “ $5}’ | sed “s/%//g; s/-/0/g”)
# Process the output line by line
echo “$output” | while read -r line; do
. pid=$(echo “$line” | awk ‘{print $1}’) # Extract process ID (PID)
. mem=$(echo “$line” | awk ‘{print $2}’) # Extract memory utilization
. # Check if memory utilization exceeds 5%
. if [[ $mem -gt 5 ]]; then
. send_alert “$pid” “$mem”
. fi
done