Pacemaker & Corosync High Availability

Pacemaker and Corosync integrate with PiVirt to provide automated cluster quorum management, split-brain protection, and high-availability daemon supervision across all nodes.

Architecture

flowchart TB
    subgraph Pacemaker["Pacemaker Cluster"]
        CORO["Corosync (Messaging)"]
        CRM["Pacemaker (CRM)"]
        RA["Resource Agents (OCF / LSB)"]
    end

    subgraph Node1["Node 1"]
        P1["pivirtd / iscsiman"]
    end
    subgraph Node2["Node 2"]
        P2["pivirtd / iscsiman"]
    end
    subgraph Node3["Node 3"]
        P3["pivirtd / iscsiman"]
    end

    Pacemaker --> Node1
    Pacemaker --> Node2
    Pacemaker --> Node3

Install Pacemaker

Install Packages

# On all nodes
apt-get install -y pacemaker corosync fence-agents resource-agents

Configure Corosync

# Generate authkey
corosync-keygen

# Copy to all nodes
scp /etc/corosync/authkey node-02:/etc/corosync/
scp /etc/corosync/authkey node-03:/etc/corosync/

# Configure corosync
cat > /etc/corosync/corosync.conf <<EOF
totem {
    version: 2
    cluster_name: pivirt-cluster
    transport: knet
    crypto_cipher: aes256
    crypto_hash: sha256
}

nodelist {
    node {
        ring0_addr: 10.0.1.10
        name: node-01
        nodeid: 1
    }
    node {
        ring0_addr: 10.0.1.11
        name: node-02
        nodeid: 2
    }
    node {
        ring0_addr: 10.0.1.12
        name: node-03
        nodeid: 3
    }
}

quorum {
    provider: corosync_votequorum
    expected_votes: 3
    two_node: 0
}

logging {
    to_logfile: yes
    logfile: /var/log/corosync/corosync.log
    to_syslog: yes
}
EOF

# Copy to all nodes
scp /etc/corosync/corosync.conf node-02:/etc/corosync/
scp /etc/corosync/corosync.conf node-03:/etc/corosync/

Start Services

# On all nodes
systemctl enable --now corosync
systemctl enable --now pacemaker

# Verify
systemctl status corosync
systemctl status pacemaker

Configure Fencing

SBD Fencing

# Install SBD
apt-get install -y sbd

# Configure SBD
cat > /etc/default/sbd <<EOF
SBD_DEVICE="/dev/disk/by-id/scsi-360050764008181c60000000000000002"
SBD_PACEMAKER=yes
SBD_STARTMODE=always
EOF

# Initialize SBD
sbd -d /dev/disk/by-id/scsi-360050764008181c60000000000000002 create

# Enable SBD
systemctl enable sbd

IPMI Fencing

# Configure IPMI fencing
pcs stonith create ipmi-fence fence_ipmi_lan \
  ipaddr=10.0.0.11 \
  login=admin \
  passwd=password \
  lanplus=1 \
  pcmk_host_list=node-01

# Add fencing for other nodes
pcs stonith create ipmi-fence-02 fence_ipmi_lan \
  ipaddr=10.0.0.12 \
  login=admin \
  passwd=password \
  lanplus=1 \
  pcmk_host_list=node-02

Configure Resources

iSCSI Target Resource

# Create iSCSI target resource
pcs resource create iscsi-target ocf:pilab:iscsi-target \
  iqn="iqn.2024-01.hu.pilab:vm-storage" \
  op monitor interval=30s

# Create clone (run on all nodes)
pcs resource clone iscsi-target

OCFS2 Resource

# Create OCFS2 resource
pcs resource create ocfs2-vm-shared ocf:heartbeat:Filesystem \
  device="/dev/disk/by-id/scsi-360050764008181c60000000000000001" \
  directory="/var/lib/pivirtd/shared" \
  fstype="ocfs2" \
  options="_netdev,nointr" \
  op monitor interval=30s

# Create clone
pcs resource clone ocfs2-vm-shared

pivirtd Resource

# Create pivirtd resource
pcs resource create pivirtd systemd:pivirtd \
  op monitor interval=30s

# Create clone
pcs resource clone pivirtd

Configure Constraints

Colocation Constraints

# Ensure OCFS2 runs with iSCSI
pcs constraint colocation add ocfs2-vm-shared-clone with iscsi-target-clone INFINITY

# Ensure pivirtd runs with OCFS2
pcs constraint colocation add pivirtd-clone with ocfs2-vm-shared-clone INFINITY

Ordering Constraints

# Start iSCSI before OCFS2
pcs constraint order iscsi-target-clone then ocfs2-vm-shared-clone

# Start OCFS2 before pivirtd
pcs constraint order ocfs2-vm-shared-clone then pivirtd-clone

Monitoring

Check Cluster Status

# Check cluster status
pcs status

# Check resource status
pcs resource status

# Check node status
pcs node status

Check Logs

# Check Pacemaker logs
journalctl -u pacemaker -f

# Check Corosync logs
journalctl -u corosync -f

Troubleshooting

Cluster Not Forming

# Check Corosync
corosync-cfgtool -s

# Check quorum
corosync-quorumtool -s

# Check logs
journalctl -u corosync -f

Resource Failures

# Check resource status
pcs resource debug-start <resource-name>

# Check resource logs
journalctl -u pacemaker -f | grep <resource-name>

# Check resource agent
/usr/lib/ocf/resource.d/pilab/iscsi-target meta-data

Fencing Issues

# Test fencing
pcs stonith fence node-02

# Check fencing logs
journalctl -u pacemaker -f | grep -i fence

Related

• Clustering Guide
• High Availability
• iscsiman Guide