From 354088b2453499d958ce6c8954bf3a507c4020c8 Mon Sep 17 00:00:00 2001
From: Trey Blancher <trey@blancher.net>
Date: Tue, 8 Aug 2023 18:20:04 -0400
Subject: [PATCH] Final working version, for now

---
 psi-alerts.sh | 55 ++++++++++++++++++++++++++-------------------------
 1 file changed, 28 insertions(+), 27 deletions(-)
 mode change 100644 => 100755 psi-alerts.sh

diff --git a/psi-alerts.sh b/psi-alerts.sh
old mode 100644
new mode 100755
index 86186c0..5a79546
--- a/psi-alerts.sh
+++ b/psi-alerts.sh
@@ -44,6 +44,8 @@ sms_dst="${SMS_DST}"
 sms_domain="$(awk -F@ '{print $NF}' <<< ${SMS_DST})"
 ssh_port="${SSH_PORT}"
 ssh_host="${SSH_HOST}"
+ssh_user="${SSH_USER}"
+ssh_id_path="${SSH_ID_PATH}"
 clear_threshold="${CLEAR_THRESHOLD}"
 notification_cmd="${NOTIFICATION_CMD}"
 notification_opts="${NOTIFICATION_OPTS}"
@@ -51,32 +53,31 @@ id_idx="${NOTIFICATION_IDX}"
 
 print_psi () {
     local psi_file="${1}"
-    cat "${(P)psi_file}"
+    cat "${(P)$(tr '[[:upper:]]' '[[:lower:]]' <<< "${psi_file}")}"
 }
 
 print_pidstat () {
     local psi_type="${1}"
-    local opts="-l --human"
 
     case "${psi_type}" in
         CPU)
-            opts="-u ${opts}"
+            pidstat -ul --human
             ;;
         IO)
-            opts="-d ${opts}"
+            pidstat -dl --human
             ;;
         MEM)
-            opts="-r ${opts}"
+            pidstat -rl --human
             ;;
         *)
             print "Invalid psi_type:  ${psi_type}" >&2
             ;;
     esac
-
-    pidstat "${opts}"
 }
 
+
 send_notice () {
+    #set -x
     local psi_type="${1}"
     shift
     
@@ -96,7 +97,7 @@ send_notice () {
     esac
 
     integer notification_id
-    if ! notification_id=$(ssh -q "${ssh_host}" -p ${ssh_port} \
+    if ! notification_id=$(ssh -q -i "${ssh_id_path}" "${ssh_user}@${ssh_host}" -p ${ssh_port} \
         "${notification_cmd} ${notification_opts} '${host}:  PSI ${psi_type} triggered!' '${psi}'"); then
             print "Connection to notification daemon failed!" >&2
             false
@@ -104,6 +105,7 @@ send_notice () {
         echo ${notification_id}
         true
     fi
+    #set +x
 }
 
 send () {
@@ -128,29 +130,28 @@ send () {
 
     local psi="$(print_psi $(tr '[[:upper:]]' '[[:lower:]]' <<< ${psi_type}))"
 
-    local subj="PSI on deltachunk ${psi_type} triggered!"
-
-
-    local body="Pressure Stall Information triggered on ${psi_type} at $(date +'%FT%T %Z')"
-    if [[ -n "${current_alarms}" ]]; then
-        body="${body}\nMultiple alarms triggered:  ${current_alarms}"
-        # if this is not an SMS, include pidstat info
-        if [[ ! "${dst}" =~ "${sms_domain}" ]]; then
-            for p in $(tr '|' ' ' <<< "${current_alarms}"); do
-                body="${body}\n\n$(print_pidstat ${p})"
-            done
-        fi
-    fi
-
 
     local email=$(mktemp /tmp/psi.eml.XXXX)
+    local subj
+    printf "Pressure Stall Information triggered on ${psi_type} at $(date +'%FT%T %Z')\n\n" > ${email}
+    if [[ -n "${current_alarms}" ]]; then
+        subj="PSI on deltachunk ${current_alarms} triggered!"
+        printf "Multiple alarms triggered:  ${current_alarms}\n\n" >> ${email}
+    else
+        subj="PSI on deltachunk ${psi_type} triggered!"
+        current_alarms="${psi_type}"
+    fi
 
-    cat <<-EOF > ${email}
-        ${body}
-
-EOF
-
+    # is this an email or SMS?
+    if [[ ! "${dst}" =~ "${sms_domain}" ]]; then
+        for p in $(tr '|' ' ' <<< "${current_alarms}"); do
+            printf "\npidstat info for ${p}\n\n" >> ${email}
+            print_pidstat "${p}" >> ${email}
+            printf "\n\n" >> ${email}
+        done
+    fi
 
+    # send the message
     /usr/bin/mail --resource-files=/ \
                   --subject="${subj}" \
                   --end-options \