Initial commit of .gitignore

.gitignore

@@ -0,0 +1,9 @@
+*
+!psi-alerts.sh
+!psi-alerts@.service
+!psi-monitor.service
+!psi-by-example
+!.gitignore
+!CONFIGURE.md
+!INSTALL.md
+!README.md

CONFIGURE.md

@@ -25,6 +25,7 @@ Environment=EMAIL_TO="email@domain.tld"
 Environment=SMS_DST="phone_number@sms.domain.tld"
 Environment=NOTIFICATION_CMD="dunstify"
 Environment=NOTIFICATION_OPTS="--timeout=0 --printid --urgency=critical --icon=/usr/share/icons/breeze-dark/emblems/16/emblem-warning.svg"
+Environment=NOTIFICATION_HIST_CMD="dunstctl history"
 Environment=NOTIFICATION_IDX=15
 Environment=SSH_USER="username"
 Environment=SSH_HOST="localhost"
@@ -49,3 +50,40 @@ Environment=CLEAR_THRESHOLD="5.0"
 # WantedBy=multi-user.target
 ```
 
+All of these are required except where noted, there are no default options
+(defaults may be added in the future).  A brief description of each:
+* **EMAIL_TO**:  the email address the notification should be sent to.  The
+  output of `pidstat` will be included in the body of this email, for each
+  triggered resource type (CPU, I/O, Memory), at the time the monitor alerted.
+* **SMS_DST**:  the email-to-SMS address, as defined by your mobile carrier.
+  Please review your mobile carrier's documentation.  For Google Fi, based in
+  the US, the format is
+  `<mobile_number_without_country_code>@msg.fi.google.com`.  This email address
+  does **not** get the output of `pidstat` in the body of the message.
+* **NOTIFICATION_CMD**:   The command on the remote host to run to display
+  notifications, e.g. `notify-send` or `dunstify`.
+* **NOTIFICATION_OPTS**:   Options for the `${NOTIFICATION_CMD}`.  Should
+  include `--print-id` if supported by the command.
+* **NOTIFICATION_HIST_CMD**:  The command to display the notification history
+  (e.g. `dunst history`).
+* **NOTIFICATION_IDX**:  The index if the JSON structure that contains the
+  notification ID.  `dunst`, as of version 1.9.2-1, displays its history as a
+  JSON structure.  For other notification daemons, some other history mechanism
+  will likely be required;  patches needed and welcome!
+* **SSH_USER**:  The SSH username to connect to the remote host that will
+  display the notifications to the system administrator.
+* **SSH_HOST**:  The SSH host to connect to.  This is where
+  `${NOTIFICATION_CMD} ${NOTIFICATION_OPTS}` and `${NOTIFICATION_HIST_CMD}`
+  will run.
+* **SSH_PORT**:  The SSH port to connect to.
+* **SSH_ID_PATH**:  The path to the SSH id (private key file) to use for
+  authenticating to the remote host.  This can be exluded if the local user
+  already has an ssh-agent running, with the necessary key and passphrase
+  entered.  If ssh-agent is not desired, then this SSH id (private key file)
+  should have an empty passphrase (i.e., no passphrase).  Not having this
+  environment variable, and no ssh-agent will disable the desktop notifications
+  (SMS and email will still work, as they don't use SSH)
+* **CLEAR_THRESHOLD**:  The percentage threshold the some avg300 threshold
+  should be below before considering the alert cleared.  This will depend
+  highly on the workload running on 
+

README.md

@@ -17,6 +17,18 @@ near real time.
 * psi-by-example (a modified version of this is included in this project as a
   submodule)
 * a libnotify-compatible desktop notification system
+    * any notification program should use the `--print-id` parameter if
+      possible
+        * both `notify-send` and `dunstify` (part of
+          [dunst](https://dunst-project.org/)) support this
+    * note, this has only been tested with `dunst`, since it has the capability
+      of showing notification history
+        * `notify-send` specifically does not appear to retain a history, so the
+          `check_dunst_id_is_visible` function won't work with it (and the logic to
+          skip sending a new notification if one is already sent will be broken).  
+            * since I don't use `notify-send`, I'm not sure how to solve this
+            * patches welcome!
+* jq (for the aformentioned `dunst` integration)
 
 ## History
 
@@ -98,6 +110,7 @@ below the configurable threshold for at least five minutes).
     * consider reworking this for a user service, not a system service
         * this could make desktop notifications simpler, and not having to use
           SSH keys without passphrases
+          * possibly learn how to connect to an existing ssh-agent
         * need to become much more familiar with user services
 * consider reworking all code in a compiled language (other than C)
     * time to learn Go

psi-alerts.sh

@@ -1,8 +1,9 @@
 #!/usr/bin/env zsh
 
+################################################################################
 # Send alerts when Pressure Stall Information is high
 # 
-#    Copyright © 2023 Trey Blancher
+#    Copyright © 2023 Trey Blancher $(base64 -d <<< dHJleUBibGFuY2hlci5uZXQK)
 #
 #    This program is free software: you can redistribute it and/or modify it
 #    under the terms of the GNU General Public License as published by the Free
@@ -17,6 +18,9 @@
 #    You should have received a copy of the GNU General Public License along
 #    with this program.  If not, see <https://www.gnu.org/licenses/>.
 #
+#    Submodules may be distributed under a separate software license;  see the
+#    LICENSE file within each submodule.
+# 
 # This script monitors the systemd journal, specifically the
 # `psi-monitor.service` and waits for Pressure State Information monitor events
 # to be logged.  The monitor program is shipped in the psi-by-example submodule
@@ -32,7 +36,7 @@
 # systemd psi-alerts@<user>.service overrides (will be placed in
 # /etc/systemd/system/psi-alerts@<user>.service.d/override.conf. See the
 # README.md for details.
-
+################################################################################
 
 svc="psi-monitor.service"
 cpu="/proc/pressure/cpu"
@@ -48,9 +52,25 @@ ssh_user="${SSH_USER}"
 ssh_id_path="${SSH_ID_PATH}"
 clear_threshold="${CLEAR_THRESHOLD}"
 notification_cmd="${NOTIFICATION_CMD}"
+notification_hist_cmd="${NOTIFICATION_HIST_CMD}"
 notification_opts="${NOTIFICATION_OPTS}"
 id_idx="${NOTIFICATION_IDX}"
 
+get_ssh_agent () {
+    for dir in /tmp/ssh-*; do
+        if [[ -O ${dir} ]]; then
+            # only choose the last agent
+            export SSH_AUTH_SOCK=$(ls ${dir}/agent.* | tail -1)
+        fi
+    done
+
+    if [[ -S ${SSH_AUTH_SOCK} ]]; then
+        # we found an ssh_agent socket
+        true
+    else
+        false
+    fi
+}
 print_psi () {
     local psi_file="${1}"
     cat "${(P)$(tr '[[:upper:]]' '[[:lower:]]' <<< "${psi_file}")}"
@@ -97,6 +117,16 @@ send_notice () {
     esac
 
     integer notification_id
+    if get_ssh_agent && [[ -S ${SSH_AUTH_SOCK} ]]; then
+        if ! notification_id=$(ssh -q "${ssh_user}@${ssh_host}" -p ${ssh_port} \
+            "${notification_cmd} ${notification_opts} '${host}:  PSI ${psi_type} triggered!' '${psi}'"); then
+                print "Connection to notification daemon failed!" >&2
+                false
+        else
+            echo ${notification_id}
+            true
+        fi
+    elif [[ -n "${ssh_id_path}" ]]; then
         if ! notification_id=$(ssh -q -i "${ssh_id_path}" "${ssh_user}@${ssh_host}" -p ${ssh_port} \
             "${notification_cmd} ${notification_opts} '${host}:  PSI ${psi_type} triggered!' '${psi}'"); then
                 print "Connection to notification daemon failed!" >&2
@@ -105,6 +135,10 @@ send_notice () {
             echo ${notification_id}
             true
         fi
+    else
+        echo "No SSH notifications configured.  Returning." >&2
+        false
+    fi
     #set +x
 }
 
@@ -167,11 +201,11 @@ is_clear () {
     local psi_type="${1}"
     local psi_file="$(tr '[[:upper:]]' '[[:lower:]]' <<< "${psi_type}")"
 
-    local avg10=$(grep some "${(P)psi_file}" | awk '{print $2}' | awk -F= '{print $2}')
+    local avg300=$(grep some "${(P)psi_file}" | awk '{print $4}' | awk -F= '{print $2}')
     local prev=$(grep some "${(P)psi_file}" | awk '{print $5}' | awk -F= '{print $2}')
 
     
-    if [[ ${avg10} -lt ${clear_threshold} ]]; then
+    if [[ ${avg300} -lt ${clear_threshold} ]]; then
         true
     else
         false
@@ -209,35 +243,69 @@ check_dunst_id_is_visible () {
     local dunst_id="${1}"
 
     typeset -a ids
-    if ids=$(ssh -q "${ssh_host}" -p ${ssh_port} \
+    if ! ids=$(ssh -q "${ssh_host}" -p ${ssh_port} -l "${ssh_user}" \
-        "dunstctl history | jq '.data[0][][${id_idx}].data'"); then
+        "${notification_hist_cmd} | jq '.data[0][].id.data'"); then
+        if ! ids=$(ssh -qi "${ssh_id_path}" -p ${ssh_port} -l "${ssh_user}" \
+            "${ssh_host}" "${notification_hist_cmd} | jq '.data[0][].id.data'"); then
                 echo "Connection to dunst failed!" >&2
                 return 2
         fi
+    fi
 
+    # if the alert is visible, it's not in the dunst history
     if grep -qP "\b${dunst_id}\b" <<< "${ids}"; then
-        true
-    else
         false
+    else
+        true
     fi
 }
-#set -x
+
 local current_alarm=""
 local last_alarm=""
 typeset -A notice_sent
 typeset -A secs
 integer last_dunst_id=-1
+local last_line=""
 
-journalctl -b 0 -fn 3 -u "${svc}"  | \
+set -x
-while read line; do
+while true; do
+    local line=$(journalctl -u ${svc} -n1)
+    if [[ "${last_line}" == "${line}" ]]; then
+        # line hasn't changed since last run, do nothing
+        sleep 1
+        continue
+    fi
+    last_line="${line}"
+    local now=$(date +%s)
+    local last_timestamp=$(date -d $(awk '{print $1" "$2" "$3}' <<< "${line}") +%s)
+    local time_diff=$(( now - last_timestamp ))
+    if (( time_diff >= 3 )); then
+        # haven't seen a monitor alert for 3 seconds, see if we can clear them
+        if [[ -n "${current_alarms}" ]]; then
+            typeset -a alarms=( $(tr '|' ' ' <<< "$current_alarms") )
+            for alarm in ${alarms}; do
+                integer elapsed=$(( now - ${secs[${alarm}]} ))
+                if is_clear "${alarm}" && (( elapsed >= 300 )); then
+                    current_alarms=$(sed -E "s/${alarm}\|?//" <<< "${current_alarms}")
+                    last_alarm=$(awk -F'|' '{print $NF}' <<< "${current_alarms}")
+                    unset "notice_sent[${alarm}]"
+                    unset "secs[${alarm}]"
+                fi
+            done
+        fi
+        sleep 1
+        continue
+    fi
     local psi_type="$(grep -Eo "(CPU|MEM|IO) PSI event" <<< "${line}" | grep -Eo "CPU|MEM|IO")"
     if [[ -n "${psi_type}" ]]; then
-        secs+=(${psi_type} $(date +%s))
+        secs+=(${psi_type} ${now})
         if [[ "${psi_type}" != "${last_alarm}" ]]; then
             if [[ ! ${notice_sent[${psi_type}]} ]]; then
                 last_dunst_id=$(exec_notices "${psi_type}" "${current_alarms}")
                 notice_sent+=(${psi_type} true)
-            elif (( last_dunst_id >= 0 )) && ! check_dunst_id_is_visible "${last_dunst_id}"; then
+            elif (( last_dunst_id >= 0 )) && check_dunst_id_is_visible "${last_dunst_id}"; then
+                last_alarm="${psi_type}"
+                sleep 1
                 continue
             fi
         fi
@@ -249,16 +317,8 @@ while read line; do
                 current_alarms="${current_alarms}|${psi_type}"
             fi
         fi
-    else
-        typeset -a alarms=( $(tr '|' ' ' <<< "$current_alarms") )
-        for alarm in ${alarms}; do
-            integer elapsed=$(( $(date +%s) - ${secs[${alarm}]} ))
-            if is_clear "${alarm}" && (( elapsed > 300 )); then
-                current_alarms=$(sed -E "s/${alarm}\|?//" <<< "${current_alarms}")
-                last_alarm=$(awk -F'|' '{print $NF}' <<< "${current_alarms}")
-            fi
-        done
     fi
+    sleep 1
 done
-#set +x
+set +x