From d603b260c4d7b3d6dd4319cb53d8d2832dcf2692 Mon Sep 17 00:00:00 2001 From: James Peru Date: Fri, 1 May 2026 03:08:35 +0300 Subject: [PATCH 1/2] KVM: make storage heartbeat fence action configurable The KVM agent's storage heartbeat scripts (kvmheartbeat.sh and kvmspheartbeat.sh) hard-code an immediate kernel-level reboot via 'echo b > /proc/sysrq-trigger' when a heartbeat write to primary storage times out. This bypasses all OS-level shutdown protections, drops every running VM on the host instantly, and triggers HA cascades onto surviving hosts. For NFS shared storage the binary "heartbeat-write-failed = host-is-dead" heuristic is reasonable. For LINSTOR/DRBD or other replicated local storage, the same disk serves application I/O, replication I/O and heartbeat I/O simultaneously - so a transient I/O contention spike can time out the heartbeat write without the host actually being unhealthy. The result is false-positive sysrq fencing. Adds a new agent.properties option: kvm.heartbeat.fence.action = reboot | graceful-reboot | restart-agent | log-only Default value is "reboot" so existing deployments keep their current behavior. Operators on replicated storage backends can choose a less destructive action: - graceful-reboot: 'systemctl reboot' instead of sysrq, allowing VMs a chance to shut down cleanly - restart-agent: restart cloudstack-agent only, preserving running VMs - log-only: log + alert, no automatic action The existing 'reboot.host.and.alert.management.on.heartbeat.timeout' boolean continues to function as a complete Java-side bypass. Refs: https://github.com/apache/cloudstack/issues/13089 --- agent/conf/agent.properties | 16 +++++++ .../agent/properties/AgentProperties.java | 19 +++++++++ scripts/vm/hypervisor/kvm/kvmheartbeat.sh | 42 ++++++++++++++++--- scripts/vm/hypervisor/kvm/kvmspheartbeat.sh | 42 ++++++++++++++++--- 4 files changed, 109 insertions(+), 10 deletions(-) diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 0dc5b8211e0d..964ed1e7c2b0 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -310,6 +310,22 @@ iscsi.session.cleanup.enabled=false # This parameter specifies if the host must be rebooted when something goes wrong with the heartbeat. #reboot.host.and.alert.management.on.heartbeat.timeout=true +# Action taken by kvmheartbeat.sh / kvmspheartbeat.sh when a storage heartbeat +# write fails persistently. Supersedes the legacy binary +# 'reboot.host.and.alert.management.on.heartbeat.timeout' when set to a non-default value. +# +# Allowed values: +# reboot - immediate sysrq-trigger reboot (default; original behavior) +# graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly +# restart-agent - restart cloudstack-agent only; running VMs are preserved +# log-only - log + alert; take no automatic action (admin must investigate) +# +# The 'graceful-reboot', 'restart-agent', and 'log-only' actions are recommended +# for setups using LINSTOR/DRBD or any local storage with replication, where +# transient I/O contention can cause a heartbeat write to time out without the +# host actually being unhealthy. +#kvm.heartbeat.fence.action=reboot + # Enables manually setting CPU's topology on KVM's VM. #enable.manually.setting.cpu.topology.on.kvm.vm=true diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 3364f9708cf5..169f3d15834e 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -598,6 +598,25 @@ public class AgentProperties{ public static final Property REBOOT_HOST_AND_ALERT_MANAGEMENT_ON_HEARTBEAT_TIMEOUT = new Property<>("reboot.host.and.alert.management.on.heartbeat.timeout", true); + /** + * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh / kvmspheartbeat.sh) + * when a heartbeat write fails persistently. Allowed values: + * + * The non-default values are recommended for setups using LINSTOR/DRBD or other replicated + * local storage, where transient I/O contention can cause a heartbeat write to time out + * without the host actually being unhealthy.
+ * Read by the heartbeat shell scripts directly from agent.properties.
+ * Data type: String.
+ * Default value: {@code reboot} + */ + public static final Property KVM_HEARTBEAT_FENCE_ACTION + = new Property<>("kvm.heartbeat.fence.action", "reboot"); + /** * Enables manually setting CPU's topology on KVM's VM.
* Data type: Boolean.
diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh index 9b7eadada69f..4f8056922656 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh @@ -156,11 +156,43 @@ then exit 0 elif [ "$cflag" == "1" ] then - /usr/bin/logger -t heartbeat "kvmheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? + # Read fence action from agent.properties (default: reboot for backward compatibility). + # Allowed values: reboot | graceful-reboot | restart-agent | log-only + AGENT_PROPS="/etc/cloudstack/agent/agent.properties" + FENCE_ACTION="reboot" + if [ -r "$AGENT_PROPS" ]; then + val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') + [ -n "$val" ] && FENCE_ACTION="$val" + fi + + case "$FENCE_ACTION" in + log-only) + /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." + exit 0 + ;; + restart-agent) + /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." + sync & + sleep 2 + systemctl restart cloudstack-agent + exit $? + ;; + graceful-reboot) + /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." + sync & + sleep 5 + systemctl reboot + exit $? + ;; + reboot|*) + # Original behavior: immediate kernel-level reboot via sysrq-trigger + /usr/bin/logger -t heartbeat "kvmheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." + sync & + sleep 5 + echo b > /proc/sysrq-trigger + exit $? + ;; + esac else write_hbLog exit $? diff --git a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh index 3cb459e3e854..bc0294cfdfb7 100755 --- a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh @@ -58,9 +58,41 @@ deleteVMs() { if [ "$cflag" == "1" ] then - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? + # Read fence action from agent.properties (default: reboot for backward compatibility). + # Allowed values: reboot | graceful-reboot | restart-agent | log-only + AGENT_PROPS="/etc/cloudstack/agent/agent.properties" + FENCE_ACTION="reboot" + if [ -r "$AGENT_PROPS" ]; then + val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') + [ -n "$val" ] && FENCE_ACTION="$val" + fi + + case "$FENCE_ACTION" in + log-only) + /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." + exit 0 + ;; + restart-agent) + /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." + sync & + sleep 2 + systemctl restart cloudstack-agent + exit $? + ;; + graceful-reboot) + /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." + sync & + sleep 5 + systemctl reboot + exit $? + ;; + reboot|*) + # Original behavior: immediate kernel-level reboot via sysrq-trigger + /usr/bin/logger -t heartbeat "kvmspheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." + sync & + sleep 5 + echo b > /proc/sysrq-trigger + exit $? + ;; + esac fi From f5f3db5ea31edfb95caddd3283d4ba5ee7730044 Mon Sep 17 00:00:00 2001 From: jmsperu Date: Sat, 9 May 2026 06:50:28 +0300 Subject: [PATCH 2/2] kvm-heartbeat: extract shared fence helper; add 'custom' action; rename to hard-reboot Per review on PR #13090: - Refactor common fence-action case into kvmha-fence.sh sourced by both kvmheartbeat.sh and kvmspheartbeat.sh (per @sureshanaparti). - Add 'custom' fence action that invokes an operator-supplied script (kvm.heartbeat.fence.custom.script, default /etc/cloudstack/agent/ heartbeat-fence-custom.sh) with the heartbeat script name as arg; falls back to hard-reboot if the script is missing/non-executable (per @NuxRo). - Rename canonical action 'reboot' -> 'hard-reboot' for clarity; keep 'reboot' accepted as alias so existing deployments don't break (per @DaanHoogland). Default behavior unchanged: sysrq-trigger reboot, required where a stale NFSv3 mount blocks systemctl reboot. --- agent/conf/agent.properties | 27 ++++-- .../agent/properties/AgentProperties.java | 31 +++++-- scripts/vm/hypervisor/kvm/kvmha-fence.sh | 85 +++++++++++++++++++ scripts/vm/hypervisor/kvm/kvmheartbeat.sh | 40 +-------- scripts/vm/hypervisor/kvm/kvmspheartbeat.sh | 40 +-------- 5 files changed, 134 insertions(+), 89 deletions(-) create mode 100755 scripts/vm/hypervisor/kvm/kvmha-fence.sh diff --git a/agent/conf/agent.properties b/agent/conf/agent.properties index 964ed1e7c2b0..a8e3b73e8dd9 100644 --- a/agent/conf/agent.properties +++ b/agent/conf/agent.properties @@ -315,16 +315,25 @@ iscsi.session.cleanup.enabled=false # 'reboot.host.and.alert.management.on.heartbeat.timeout' when set to a non-default value. # # Allowed values: -# reboot - immediate sysrq-trigger reboot (default; original behavior) -# graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly -# restart-agent - restart cloudstack-agent only; running VMs are preserved -# log-only - log + alert; take no automatic action (admin must investigate) +# hard-reboot - immediate sysrq-trigger reboot (default; 'reboot' kept as alias). +# Required default for setups where a stale NFSv3 mount can prevent +# a graceful shutdown from completing. +# graceful-reboot - 'systemctl reboot' instead of sysrq; allows VMs to stop cleanly. +# Use only if a stale storage mount cannot block shutdown. +# restart-agent - restart cloudstack-agent only; running VMs are preserved. +# log-only - log + alert; take no automatic action (admin must investigate). +# custom - invoke the script at 'kvm.heartbeat.fence.custom.script' (see below). +# Script is called with one positional arg: the heartbeat script name +# (e.g. 'kvmheartbeat.sh'). Falls back to hard-reboot if missing or +# not executable. # -# The 'graceful-reboot', 'restart-agent', and 'log-only' actions are recommended -# for setups using LINSTOR/DRBD or any local storage with replication, where -# transient I/O contention can cause a heartbeat write to time out without the -# host actually being unhealthy. -#kvm.heartbeat.fence.action=reboot +# The non-default values are recommended for setups using LINSTOR/DRBD or any local +# storage with replication, where transient I/O contention can cause a heartbeat +# write to time out without the host actually being unhealthy. +#kvm.heartbeat.fence.action=hard-reboot + +# Path to the operator-supplied script invoked when kvm.heartbeat.fence.action=custom. +#kvm.heartbeat.fence.custom.script=/etc/cloudstack/agent/heartbeat-fence-custom.sh # Enables manually setting CPU's topology on KVM's VM. #enable.manually.setting.cpu.topology.on.kvm.vm=true diff --git a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java index 169f3d15834e..7951e37b035a 100644 --- a/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java +++ b/agent/src/main/java/com/cloud/agent/properties/AgentProperties.java @@ -602,20 +602,39 @@ public class AgentProperties{ * Action taken by the KVM agent's storage heartbeat scripts (kvmheartbeat.sh / kvmspheartbeat.sh) * when a heartbeat write fails persistently. Allowed values: *
    - *
  • {@code reboot} (default) — immediate sysrq-trigger reboot; original behavior
    • - *
  • {@code graceful-reboot} — {@code systemctl reboot} instead of sysrq, lets VMs stop cleanly
    • - *
  • {@code restart-agent} — restart cloudstack-agent only; running VMs preserved
    • - *
  • {@code log-only} — log + alert, no automatic action
    • + *
  • {@code hard-reboot} (default; {@code reboot} accepted as alias) — immediate + * sysrq-trigger reboot. Required default for setups where a stale NFSv3 mount can + * prevent a graceful shutdown from completing.
    • + *
  • {@code graceful-reboot} — {@code systemctl reboot} instead of sysrq; allows VMs + * to stop cleanly. Use only if a stale storage mount cannot block shutdown.
    • + *
  • {@code restart-agent} — restart cloudstack-agent only; running VMs preserved.
    • + *
  • {@code log-only} — log + alert; take no automatic action (admin must investigate).
    • + *
  • {@code custom} — invoke the script at {@link #KVM_HEARTBEAT_FENCE_CUSTOM_SCRIPT} + * (default {@code /etc/cloudstack/agent/heartbeat-fence-custom.sh}). The script is + * called with one argument: the heartbeat script name (e.g. {@code kvmheartbeat.sh}). + * If the script is missing or not executable, falls back to {@code hard-reboot}.
    • *
* The non-default values are recommended for setups using LINSTOR/DRBD or other replicated * local storage, where transient I/O contention can cause a heartbeat write to time out * without the host actually being unhealthy.
* Read by the heartbeat shell scripts directly from agent.properties.
* Data type: String.
- * Default value: {@code reboot} + * Default value: {@code hard-reboot} */ public static final Property KVM_HEARTBEAT_FENCE_ACTION - = new Property<>("kvm.heartbeat.fence.action", "reboot"); + = new Property<>("kvm.heartbeat.fence.action", "hard-reboot"); + + /** + * Path to the operator-supplied script invoked when + * {@link #KVM_HEARTBEAT_FENCE_ACTION} is set to {@code custom}. The script must be + * executable and is called with a single positional argument: the heartbeat script name + * that triggered the fence (e.g. {@code kvmheartbeat.sh}). Read by the heartbeat shell + * scripts directly from agent.properties.
+ * Data type: String.
+ * Default value: {@code /etc/cloudstack/agent/heartbeat-fence-custom.sh} + */ + public static final Property KVM_HEARTBEAT_FENCE_CUSTOM_SCRIPT + = new Property<>("kvm.heartbeat.fence.custom.script", "/etc/cloudstack/agent/heartbeat-fence-custom.sh"); /** * Enables manually setting CPU's topology on KVM's VM.
diff --git a/scripts/vm/hypervisor/kvm/kvmha-fence.sh b/scripts/vm/hypervisor/kvm/kvmha-fence.sh new file mode 100755 index 000000000000..5f3006f71d7b --- /dev/null +++ b/scripts/vm/hypervisor/kvm/kvmha-fence.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Shared fence-action helper for kvmheartbeat.sh and kvmspheartbeat.sh. +# Sourced by both scripts; do not invoke directly. +# +# Usage from caller: +# source "$(dirname "$0")/kvmha-fence.sh" +# fence_action "kvmheartbeat.sh" # script name passed for log tagging + +AGENT_PROPS="${AGENT_PROPS:-/etc/cloudstack/agent/agent.properties}" + +fence_action() { + local source_script="${1:-kvmha}" + local FENCE_ACTION="hard-reboot" + local CUSTOM_SCRIPT="/etc/cloudstack/agent/heartbeat-fence-custom.sh" + + if [ -r "$AGENT_PROPS" ]; then + local val + val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') + [ -n "$val" ] && FENCE_ACTION="$val" + local cval + cval=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.custom\.script[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + [ -n "$cval" ] && CUSTOM_SCRIPT="$cval" + fi + + case "$FENCE_ACTION" in + log-only) + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." + exit 0 + ;; + restart-agent) + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." + sync & + sleep 2 + systemctl restart cloudstack-agent + exit $? + ;; + graceful-reboot) + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." + sync & + sleep 5 + systemctl reboot + exit $? + ;; + custom) + if [ -x "$CUSTOM_SCRIPT" ]; then + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'custom' — running ${CUSTOM_SCRIPT}." + sync & + sleep 2 + "$CUSTOM_SCRIPT" "$source_script" + exit $? + else + /usr/bin/logger -t heartbeat "${source_script}: heartbeat write to storage failed; fence action 'custom' selected but ${CUSTOM_SCRIPT} is missing or not executable — falling back to hard-reboot." + sync & + sleep 5 + echo b > /proc/sysrq-trigger + exit $? + fi + ;; + hard-reboot|reboot|*) + # 'reboot' kept as alias for back-compat with pre-existing deployments. + /usr/bin/logger -t heartbeat "${source_script} will reboot system because it was unable to write the heartbeat to the storage." + sync & + sleep 5 + echo b > /proc/sysrq-trigger + exit $? + ;; + esac +} diff --git a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh index 4f8056922656..0947798dcfcd 100755 --- a/scripts/vm/hypervisor/kvm/kvmheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmheartbeat.sh @@ -156,43 +156,9 @@ then exit 0 elif [ "$cflag" == "1" ] then - # Read fence action from agent.properties (default: reboot for backward compatibility). - # Allowed values: reboot | graceful-reboot | restart-agent | log-only - AGENT_PROPS="/etc/cloudstack/agent/agent.properties" - FENCE_ACTION="reboot" - if [ -r "$AGENT_PROPS" ]; then - val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') - [ -n "$val" ] && FENCE_ACTION="$val" - fi - - case "$FENCE_ACTION" in - log-only) - /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." - exit 0 - ;; - restart-agent) - /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." - sync & - sleep 2 - systemctl restart cloudstack-agent - exit $? - ;; - graceful-reboot) - /usr/bin/logger -t heartbeat "kvmheartbeat.sh: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." - sync & - sleep 5 - systemctl reboot - exit $? - ;; - reboot|*) - # Original behavior: immediate kernel-level reboot via sysrq-trigger - /usr/bin/logger -t heartbeat "kvmheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? - ;; - esac + # shellcheck disable=SC1091 + . "$(dirname "$0")/kvmha-fence.sh" + fence_action "kvmheartbeat.sh" else write_hbLog exit $? diff --git a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh index bc0294cfdfb7..1a5b2a24dd9f 100755 --- a/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh +++ b/scripts/vm/hypervisor/kvm/kvmspheartbeat.sh @@ -58,41 +58,7 @@ deleteVMs() { if [ "$cflag" == "1" ] then - # Read fence action from agent.properties (default: reboot for backward compatibility). - # Allowed values: reboot | graceful-reboot | restart-agent | log-only - AGENT_PROPS="/etc/cloudstack/agent/agent.properties" - FENCE_ACTION="reboot" - if [ -r "$AGENT_PROPS" ]; then - val=$(grep -E '^[[:space:]]*kvm\.heartbeat\.fence\.action[[:space:]]*=' "$AGENT_PROPS" | tail -n 1 | cut -d= -f2- | tr -d '[:space:]') - [ -n "$val" ] && FENCE_ACTION="$val" - fi - - case "$FENCE_ACTION" in - log-only) - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'log-only' selected — taking no automatic action. Operator must investigate." - exit 0 - ;; - restart-agent) - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'restart-agent' — restarting cloudstack-agent (running VMs preserved)." - sync & - sleep 2 - systemctl restart cloudstack-agent - exit $? - ;; - graceful-reboot) - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh: heartbeat write to storage failed; fence action 'graceful-reboot' — rebooting via systemctl (allows running VMs to stop cleanly)." - sync & - sleep 5 - systemctl reboot - exit $? - ;; - reboot|*) - # Original behavior: immediate kernel-level reboot via sysrq-trigger - /usr/bin/logger -t heartbeat "kvmspheartbeat.sh will reboot system because it was unable to write the heartbeat to the storage." - sync & - sleep 5 - echo b > /proc/sysrq-trigger - exit $? - ;; - esac + # shellcheck disable=SC1091 + . "$(dirname "$0")/kvmha-fence.sh" + fence_action "kvmspheartbeat.sh" fi