OCFS2 Multi-Node Cluster Setup

This guide provides a step-by-step procedure for configuring the O2CB cluster stack, configuring nodemanager daemons, formatting shared LUNs, and mounting OCFS2 across all PiVirt cluster nodes.

Prerequisites

# Install OCFS2 tools
apt-get install -y ocfs2-tools

# Load kernel modules
modprobe ocfs2
modprobe ocfs2_dlmfs
modprobe ocfs2_nodemanager

Cluster Configuration

Configure O2CB Cluster

# Create cluster configuration
cat > /etc/ocfs2/cluster.conf <<EOF
cluster:
    node_count = 3
    name = pivirt-cluster

node:
    ip_port = 7777
    ip_address = 10.0.1.10
    number = 0
    name = node-01
    cluster = pivirt-cluster

node:
    ip_port = 7777
    ip_address = 10.0.1.11
    number = 1
    name = node-02
    cluster = pivirt-cluster

node:
    ip_port = 7777
    ip_address = 10.0.1.12
    number = 2
    name = node-03
    cluster = pivirt-cluster
EOF

Start O2CB Service

# Start O2CB
systemctl start o2cb

# Enable on boot
systemctl enable o2cb

# Check status
systemctl status o2cb

Format OCFS2 Volume

Using pvcli

pvcli storage ocfs2 create \
  --name vm-shared \
  --device /dev/disk/by-id/scsi-360050764008181c60000000000000001 \
  --cluster pivirt-cluster \
  --label vm-shared

Using mkfs.ocfs2

# Format volume
mkfs.ocfs2 \
  --cluster-stack=pcmk \
  --cluster-name=pivirt-cluster \
  -N 3 \
  -L vm-shared \
  /dev/disk/by-id/scsi-360050764008181c60000000000000001

Mount OCFS2

Manual Mount

# Create mount point
mkdir -p /var/lib/pivirtd/shared

# Mount OCFS2
mount -t ocfs2 -o _netdev,nointr \
  /dev/disk/by-id/scsi-360050764008181c60000000000000001 \
  /var/lib/pivirtd/shared

Using pvcli

pvcli storage ocfs2 mount \
  --name vm-shared \
  --mount-point /var/lib/pivirtd/shared \
  --all-nodes

Fstab Entry

# Add to /etc/fstab
/dev/disk/by-id/scsi-360050764008181c60000000000000001 /var/lib/pivirtd/shared ocfs2 _netdev,nointr 0 0

Fencing Configuration

SBD Fencing

# Install SBD
apt-get install -y sbd

# Configure SBD
cat > /etc/default/sbd <<EOF
SBD_DEVICE="/dev/disk/by-id/scsi-360050764008181c60000000000000002"
SBD_PACEMAKER=yes
SBD_STARTMODE=always
SBD_DELAY_START=no
EOF

# Initialize SBD
sbd -d /dev/disk/by-id/scsi-360050764008181c60000000000000002 create

IPMI Fencing

/etc/iscsiman/config.yaml

cluster:
  fencing:
    enabled: true
    agent: "ipmi"
    devices:
      - name: "node-01-fence"
        ip: "10.0.0.11"
        user: "admin"
        password_ref: "vault:secret/data/bmc/node-01#password"
        lanplus: true

Monitoring

Check Cluster Status

# Check O2CB status
o2cb cluster-status

# Check node status
o2cb node-status

# Check mounted filesystems
mounted.ocfs2 -f

Check DLM Status

# Check DLM
dlm_tool ls

# Check lockspace
dlm_tool lockdebug pivirt-dlm

Troubleshooting

Mount Fails

# Check cluster status
o2cb cluster-status

# Check DLM
dlm_tool ls

# Check logs
journalctl -u o2cb -f
dmesg | grep -i ocfs2

Quorum Lost

# Check node connectivity
ping 10.0.1.11
ping 10.0.1.12

# Check O2CB logs
journalctl -u o2cb -f

# Restart O2CB
systemctl restart o2cb

Filesystem Corruption

# Unmount filesystem
umount /var/lib/pivirtd/shared

# Check filesystem
fsck.ocfs2 -f /dev/disk/by-id/scsi-360050764008181c60000000000000001

# Mount again
mount -t ocfs2 -o _netdev,nointr \
  /dev/disk/by-id/scsi-360050764008181c60000000000000001 \
  /var/lib/pivirtd/shared

Related

• Storage Guide
• iscsiman Guide
• Clustering Guide