High Availability Architecture

PiVirt provides built-in high availability across clustered hypervisor nodes with shared storage, automated node health probing, split-brain fencing, and rapid VM restart failover.

Architecture

flowchart TB
    LB["Load Balancer<br/>(agent endpoints)"]

    subgraph Nodes["Cluster Nodes"]
        subgraph N1["Node 1"]
            P1[pivirtd]
            A1[agent]
        end
        subgraph N2["Node 2"]
            P2[pivirtd]
            A2[agent]
        end
        subgraph N3["Node 3"]
            P3[pivirtd]
            A3[agent]
        end
    end

    STORAGE[("Shared Storage<br/>(iSCSI / OCFS2)")]

    LB --> N1
    LB --> N2
    LB --> N3

    N1 --> STORAGE
    N2 --> STORAGE
    N3 --> STORAGE

Control Plane HA

etcd Cluster

# Initialize etcd cluster
pvcli cluster init --name prod-cluster \
  --etcd-members 3 \
  --etcd-nodes node-01,node-02,node-03

# Verify etcd health
etcdctl endpoint health \
  --endpoints=https://node-01:2379,https://node-02:2379,https://node-03:2379

PiTrust CA Server HA

# PiTrust CA server configuration
server:
  addr: "0.0.0.0:8080"
  ca:
    root_cert_path: "/etc/pitrust/certs/PiRootR1.pem"
    root_key_path: "/etc/pitrust/certs/PiRootR1.key"
    intermediate_cert_path: "/etc/pitrust/certs/PiCA.pem"
    intermediate_key_path: "/etc/pitrust/certs/PiCA.key"
  database:
    path: "/var/lib/pitrust/certs.db"
  replication:
    enabled: true
    peers:
      - "https://ca-01.pivirt.internal:8080"
      - "https://ca-02.pivirt.internal:8080"

Agent Load Balancing

# Load balancer configuration (HAProxy)
frontend pivirt-agent
  bind *:8443
  mode tcp
  default_backend pivirt-agents

backend pivirt-agents
  mode tcp
  balance roundrobin
  option ssl-hello-chk
  server node-01 10.0.1.10:8443 check
  server node-02 10.0.1.11:8443 check
  server node-03 10.0.1.12:8443 check

Storage HA

iSCSI Multipath

/etc/multipath.conf

defaults {
  user_friendly_names yes
  find_multipaths smart
  path_checker tur
  path_grouping_policy group_by_prio
  prio alua
  failback immediate
}

devices {
  device {
    vendor "LIO-ORG"
    product "*"
    path_checker tur
    hardware_handler "1 alua"
  }
}

OCFS2 Cluster

cluster:
  name: "pivirt-cluster"
  o2cb:
    stack: "o2cb"
    nodes:
      - name: "node-01"
        ip: "10.0.1.10"
        number: 0
      - name: "node-02"
        ip: "10.0.1.11"
        number: 1
      - name: "node-03"
        ip: "10.0.1.12"
        number: 2
  dlm:
    lockspace: "pivirt-dlm"
    recovery_timeout: "30s"

Fencing

cluster:
  fencing:
    enabled: true
    agent: "ipmi"
    devices:
      - name: "node-01-fence"
        ip: "10.0.0.11"
        user: "admin"
        password_ref: "vault:secret/data/bmc/node-01#password"
        lanplus: true
      - name: "node-02-fence"
        ip: "10.0.0.12"
        user: "admin"
        password_ref: "vault:secret/data/bmc/node-02#password"
        lanplus: true
      - name: "node-03-fence"
        ip: "10.0.0.13"
        user: "admin"
        password_ref: "vault:secret/data/bmc/node-03#password"
        lanplus: true
    action: "reboot"
    timeout: "60s"

VM HA

Automatic Restart

apiVersion: pivirt.pilab.hu/v1
kind: VirtualMachine
metadata:
  name: web-01
  namespace: production
spec:
  # ... VM spec ...
  ha:
    enabled: true
    restart_policy: "Always"
    max_restarts: 3
    restart_delay: "30s"

Anti-Affinity

apiVersion: pivirt.pilab.hu/v1
kind: VirtualMachine
metadata:
  name: web-01
  namespace: production
spec:
  # ... VM spec ...
  affinity:
    podAntiAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchLabels:
              app: web
          topologyKey: "kubernetes.io/hostname"

Live Migration

# Live migrate VM
pvcli vm migrate web-01 --target node-02 --live

# Automatic migration on node drain
pvcli cluster node drain node-01 --migrate-vms

Monitoring HA

Health Checks

# Check cluster health
pvcli cluster health

# Check etcd health
etcdctl endpoint health \
  --endpoints=https://node-01:2379,https://node-02:2379,https://node-03:2379

# Check PiTrust CA health
curl -f http://ca-01.pivirt.internal:8080/healthz

Prometheus Alerts

prometheus-rules.yml

groups:
  - name: pivirt.ha.rules
    rules:
      - alert: PivirtClusterQuorumLost
        expr: count(pivirt_cluster_node_count{status="ready"}) < 3
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Cluster quorum lost"

      - alert: PivirtEtcdUnhealthy
        expr: etcd_cluster_health == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "etcd cluster unhealthy"

      - alert: PivirtStorageUnavailable
        expr: iscsiman_pool_available_bytes == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Storage pool unavailable"

Disaster Recovery

Backup Strategy

# Backup etcd
pvcli cluster backup create --name backup-2024-01-15

# Backup VMs
pvcli vm backup create web-01 \
  --destination s3://backups.pilab.hu/vms/

# Backup storage
pvcli storage volume backup create data-01 \
  --destination s3://backups.pilab.hu/volumes/

Recovery Procedure

# Restore etcd
pvcli cluster backup restore --name backup-2024-01-15

# Restore VM
pvcli vm backup restore \
  --source s3://backups.pilab.hu/vms/web-01-backup \
  --name web-01-restored

# Restore storage
pvcli storage volume backup restore \
  --source s3://backups.pilab.hu/volumes/data-01-backup \
  --name data-01-restored

Best Practices

1. Use Odd Number of Nodes

• 3 or 5 nodes for etcd quorum
• Prevents split-brain scenarios

2. Distribute Across Failure Domains

• Different racks, switches, power sources
• Geographic distribution for DR

3. Regular Testing

• Test failover procedures
• Test backup/restore
• Test live migration

4. Monitor Proactively

• Set up alerts for quorum loss
• Monitor storage capacity
• Monitor network latency

5. Document Procedures

• Document failover procedures
• Document recovery procedures
• Keep runbooks up to date

Related

• Clustering Concepts
• Live Migration
• Storage Replication
• Backup & Restore