22

✏️ Editing: imrt2-diag.sh

#!/bin/bash
# imrt2-diag — imunify-realtime-av (im-rt2) incident data collector.
#
# Read-only: gathers diagnostic state into a tarball for support; changes
# nothing on the host. Run as root. Usage:
#
#   imrt2-diag.sh [reason]
#
#   reason (optional, free text) is recorded in meta.txt — e.g. "wedged",
#   "stop-hang", "post-trip-saturation".
#
# Run when: (a) a host stays saturated AFTER a spike-guard trip, or
# (b) `systemctl stop imunify-realtime-av` hangs (run from a second shell
# WHILE it hangs), or (c) the journal shows "fanotify reader appears WEDGED".
#
# History: grew out of the one-time collector shared with a customer in
# DEF-44978; productized (DEF-51333) after the web1119 tarball needed manual
# follow-up for the since-start journal, feature-flag state, resolved failover
# mode, audit rules, and host-wide D-state processes — all captured here now.
set -u

REASON="${1:-unspecified}"
UNIT=imunify-realtime-av
FF_FILE=/var/imunify360/feature_flags.json
CONFIG=/etc/imunify-realtime-av/config.yaml
STATS_LOG=/var/log/imunify-realtime-av-stats.log
# Caps keep the tarball attachable to a ticket: the deduplicated journal is
# bounded, and a flood of identical warnings collapses to one line + count.
JOURNAL_CAP_BYTES=$((20 * 1024 * 1024))
JOURNAL_HEAD_LINES=2000

OUT=/root/imrt2-diag-$(hostname -s)-$(date +%Y%m%d-%H%M%S)
mkdir -p "$OUT"

{
  echo "captured_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  echo "reason=$REASON"
  echo "collector_version=2 (DEF-51333)"
  rpm -q imunify-realtime-av imunify-realtime-av-imrt2 2>/dev/null
  dpkg -l 2>/dev/null | awk '/imunify-realtime-av/{print $2"="$3}'
} > "$OUT/meta.txt"

# --- Process / thread state -------------------------------------------------
PID=$(systemctl show -p MainPID "$UNIT" 2>/dev/null | cut -d= -f2)
{ [ -z "$PID" ] || [ "$PID" = "0" ]; } && PID=$(pgrep -o -f 'imunify-realtime-av( |$)')
ps -eLo pid,tid,state,wchan:34,pcpu,comm --no-headers | awk -v p="${PID:-0}" '$1==p || $6 ~ /imunify|im-rt2/' > "$OUT/threads.txt" 2>/dev/null
if [ -n "${PID:-}" ]; then
  for t in /proc/"$PID"/task/*; do
    tid=$(basename "$t")
    { echo "=== tid $tid state=$(awk '{print $3}' "$t/stat" 2>/dev/null) wchan=$(cat "$t/wchan" 2>/dev/null)"
      cat "$t/stack" 2>/dev/null; echo; } >> "$OUT/kernel-stacks.txt"
  done
  cat /proc/"$PID"/status > "$OUT/status.txt" 2>/dev/null
  ls -la /proc/"$PID"/fd > "$OUT/fds.txt" 2>/dev/null
  grep -r . /proc/"$PID"/fdinfo/ 2>/dev/null > "$OUT/fdinfo.txt"
fi

# Host-wide D-state census (DEF-51333): the wedge signature is uninterruptible
# kernel waits, and OUR threads are often only some of the victims — capture
# every D-state process plus its kernel stack so storms driven by the workload
# (mysqld, php-fpm, backup agents) are attributable from the tarball alone.
ps -eo pid,stat,wchan:32,comm --no-headers 2>/dev/null | awk '$2 ~ /D/' > "$OUT/dstate-procs.txt"
while read -r dpid _; do
  [ -n "$dpid" ] || continue
  { echo "=== pid $dpid ($(cat /proc/"$dpid"/comm 2>/dev/null))"
    cat /proc/"$dpid"/stack 2>/dev/null; echo; } >> "$OUT/dstate-stacks.txt"
done < "$OUT/dstate-procs.txt"

# --- Host pressure ----------------------------------------------------------
cat /proc/pressure/cpu /proc/pressure/io /proc/pressure/memory > "$OUT/psi.txt" 2>/dev/null
vmstat 1 5 > "$OUT/vmstat.txt" 2>&1
top -bn1 -o %CPU 2>/dev/null | head -45 > "$OUT/top.txt"
dmesg -T 2>/dev/null | tail -300 > "$OUT/dmesg.txt"
{ uname -a; nproc; } > "$OUT/uname.txt"
grep -E 'Slab|SReclaimable|SUnreclaim|MemFree|MemAvailable|Dirty' /proc/meminfo > "$OUT/meminfo.txt" 2>/dev/null

# --- Daemon telemetry + config ----------------------------------------------
tail -c 300000 "$STATS_LOG" > "$OUT/stats.log" 2>/dev/null
cp "$CONFIG" "$OUT/config.yaml" 2>/dev/null
systemctl status "$UNIT" --no-pager > "$OUT/unit-status.txt" 2>&1
systemctl show "$UNIT" --no-pager > "$OUT/unit-show.txt" 2>&1

# Feature flags (DEF-51333): the guard arm/disarm root cause lives here — the
# imrt2_* entries decide the effective failover mode. Capture entries + mtime.
if [ -f "$FF_FILE" ]; then
  { echo "# mtime: $(stat -c %y "$FF_FILE" 2>/dev/null)"
    tr ',' '\n' < "$FF_FILE" | grep -i 'imrt2\|realtime'
  } > "$OUT/feature-flags.txt" 2>/dev/null
fi

# --- Journal since service start (DEF-51333) ---------------------------------
# The web1119 capture held only ~3h of journal, flooded by repeated warnings —
# the boot-time mode-resolution lines were lost. Capture since the unit's
# ActiveEnterTimestamp, collapse consecutive repeats of the same message body
# to one line + a repeat count, and cap the result (keep head = boot lines,
# plus the newest tail) so the tarball stays attachable.
SINCE=$(systemctl show -p ActiveEnterTimestamp "$UNIT" 2>/dev/null | cut -d= -f2-)
[ -z "$SINCE" ] && SINCE="-48 hours"
journalctl -u "$UNIT" --since "$SINCE" --no-pager 2>/dev/null | awk '
  { body = $0; sub(/^.*]: /, "", body) }
  body == prev { rep++; next }
  { if (rep > 1) printf "    [previous line repeated %d times]\n", rep - 1
    rep = 1; prev = body; print }
  END { if (rep > 1) printf "    [previous line repeated %d times]\n", rep - 1 }
' > "$OUT/journal.txt"
JSIZE=$(wc -c < "$OUT/journal.txt")
if [ "$JSIZE" -gt "$JOURNAL_CAP_BYTES" ]; then
  # Head is capped by BYTES as well as lines so the cap is a hard invariant:
  # head half + tail half + one marker line <= JOURNAL_CAP_BYTES regardless
  # of line lengths.
  head -n "$JOURNAL_HEAD_LINES" "$OUT/journal.txt" | head -c $((JOURNAL_CAP_BYTES / 2)) > "$OUT/journal.capped"
  echo "    [... capped: $JSIZE bytes deduplicated journal, head+tail kept ...]" >> "$OUT/journal.capped"
  tail -c $((JOURNAL_CAP_BYTES / 2 - 128)) "$OUT/journal.txt" >> "$OUT/journal.capped"
  mv "$OUT/journal.capped" "$OUT/journal.txt"
fi

# --- Resolved failover mode (DEF-51333) ---------------------------------------
# One synthesized answer to "was the guard armed, and why": local yaml mode,
# spike_guard override block, imrt2_* feature flags, the boot-time resolution
# lines, the latest telemetry mode/breaker fields, and recent breaker activity.
{
  echo "## yaml (local config)"
  grep -E '^ebpf_failover_mode' "$CONFIG" 2>/dev/null || echo "ebpf_failover_mode: (unset)"
  # Print the spike_guard block up to (excluding) the next top-level key.
  # awk instead of sed-range + drop-last-line: when spike_guard is the FINAL
  # section, the last line is guard content, not a sibling key.
  awk '/^spike_guard:/{inblock=1; print; next} inblock && /^[^[:space:]]/{exit} inblock{print}' "$CONFIG" 2>/dev/null
  echo
  echo "## feature flags (imrt2_*)"
  if [ -f "$FF_FILE" ]; then
    tr ',' '\n' < "$FF_FILE" | grep imrt2 || echo "(feature flags file present, no imrt2_* entries)"
  else
    echo "(no feature flags file)"
  fi
  echo
  echo "## boot-time mode resolution (journal)"
  grep -E 'ebpf_failover_mode|Circuit breaker active|failover' "$OUT/journal.txt" 2>/dev/null | head -20
  echo
  echo "## latest telemetry (stats.log)"
  echo "# realtime_mode: 0=off 1=on/shadow 2=auto 3=primary | active_source: 0=fanotify 1=vfs | breaker_state: 0=clear 1=tripped 2=probing"
  # stats.log entries are multi-line indented JSON (json.MarshalIndent), so
  # take the LAST occurrence of each field in the newest ~64 KB of the file.
  if [ -r "$STATS_LOG" ]; then
    for field in realtime_mode active_source breaker_state breaker_trip_count \
                 breaker_tripped_seconds breaker_config_version \
                 fanotify_queue_depth_bytes slow_reads; do
      tail -c 65536 "$STATS_LOG" | grep -oE "\"$field\": *[0-9.]+" | tail -1
    done
  else
    echo "(no stats.log)"
  fi
  echo
  echo "## recent breaker/wedge activity (journal)"
  grep -iE 'circuit breaker|WEDGED' "$OUT/journal.txt" 2>/dev/null | tail -50
} > "$OUT/resolved-mode.txt"

# --- Linux audit state (DEF-51333) --------------------------------------------
# The web1119 kernel stack showed audit_copy_inode → get_vfs_caps_from_disk →
# ext4_sb_bread: ANY audit exit rule (even a `never` rule) makes every path
# lookup pay an on-disk xattr read on 4.18 kernels — a storm amplifier that
# lives outside our process. Best-effort: auditctl may be absent.
{
  auditctl -s 2>&1
  echo "---"
  auditctl -l 2>&1
  echo "---"
  ls -la /etc/audit/rules.d/ 2>&1
} > "$OUT/audit.txt"

if tar czf "$OUT.tar.gz" -C "$(dirname "$OUT")" "$(basename "$OUT")"; then
  rm -rf "$OUT"
  echo "DONE: attach $OUT.tar.gz to the ticket"
else
  echo "ERROR: failed to create $OUT.tar.gz (disk full?); raw data left in $OUT" >&2
  exit 1
fi

MR SKB