Service Discovery & Registration

PiVirt utilizes a distributed service discovery and dynamic registration mechanism driven by an etcd cluster, enabling nodes and client SDKs to resolve active daemon endpoints in real time.

Architecture

flowchart TB
    subgraph Cluster["etcd Cluster"]
        E0["etcd-0"]
        E1["etcd-1"]
        E2["etcd-2"]
    end

    subgraph Services["PiVirt Services"]
        P["pivirtd (Service)"]
        A["agent (Service)"]
        I["iscsiman (Service)"]
    end

    P ---|"Register / Watch"| Cluster
    A ---|"Register / Watch"| Cluster
    I ---|"Register / Watch"| Cluster

Service Registration

Registration Flow

sequenceDiagram
    participant Service
    participant Agent
    participant etcd

    Service->>Agent: Register(service_info)
    Agent->>etcd: PUT /pivirt/services/{service_id}
    etcd-->>Agent: OK
    Agent-->>Service: Registered
    
    loop Heartbeat
        Service->>Agent: Heartbeat
        Agent->>etcd: PUT /pivirt/services/{service_id}/heartbeat
    end

Service Info

{
  "id": "pivirtd-node-01",
  "name": "pivirtd",
  "node": "node-01",
  "address": "10.0.1.10",
  "port": 9001,
  "metadata": {
    "version": "v0.0.89",
    "region": "eu-central-1",
    "zone": "eu-central-1a"
  },
  "health": {
    "status": "healthy",
    "last_check": "2024-01-15T10:30:00Z"
  },
  "tags": ["production", "hypervisor"]
}

Service Discovery

Discovery Flow

sequenceDiagram
    participant Client
    participant Agent
    participant etcd

    Client->>Agent: Discover(service_name)
    Agent->>etcd: GET /pivirt/services?prefix=pivirtd
    etcd-->>Agent: Service list
    Agent-->>Client: Endpoints
    
    loop Watch
        etcd->>Agent: Service changed
        Agent->>Client: Update endpoints
    end

Discovery API

# Discover all pivirtd services
curl -X GET "https://agent.pilab.hu:8443/pivirt.discovery.v1.DiscoveryService/Discover" \
  -H "Content-Type: application/json" \
  -d '{"service_name": "pivirtd"}'

# Response
{
  "endpoints": [
    {
      "id": "pivirtd-node-01",
      "address": "10.0.1.10",
      "port": 9001,
      "metadata": {"version": "v0.0.89"}
    },
    {
      "id": "pivirtd-node-02",
      "address": "10.0.1.11",
      "port": 9001,
      "metadata": {"version": "v0.0.89"}
    }
  ]
}

Load Balancing

Client-Side Load Balancing

import (
    "go.pilab.hu/cloud/virtpb/pilab/agent/v1"
    "go.pilab.hu/cloud/pibase/pkg/discovery"
)

func main() {
    // Create discovery client
    disc := discovery.NewClient("https://agent.pilab.hu:8443")

    // Create load balancer
    lb := discovery.NewRoundRobinLB(disc, "pivirtd")

    // Create service client with load balancer
    client := vmv1connect.NewVMServiceClient(
        http.DefaultClient,
        "https://agent.pilab.hu:8443",
        connect.WithInterceptors(lb.Interceptor()),
    )
}

Load Balancing Strategies

Strategy	Description	Use Case
Round Robin	Distribute evenly	General purpose
Least Connections	Route to least busy	Long-running operations
Random	Random selection	Simple distribution
Weighted	Based on weights	Heterogeneous clusters

Health Checking

Health Check Configuration

# agent config
service_discovery:
  health_check:
    enabled: true
    interval: "10s"
    timeout: "5s"
    unhealthy_threshold: 3
    healthy_threshold: 2

Health Check Flow

sequenceDiagram
    participant Agent
    participant Service
    participant etcd

    loop Every 10s
        Agent->>Service: Health check
        alt Healthy
            Service-->>Agent: OK
            Agent->>etcd: Update health status
        else Unhealthy
            Service-->>Agent: Error
            Agent->>etcd: Mark unhealthy
            Note over Agent,etcd: After 3 failures
            Agent->>etcd: Deregister service
        end
    end

Service Mesh Integration

PiTrust PKI Identity

# Service identity with PiTrust DNS SAN
service:
  name: "pivirtd"
  domain: "pivirtd.pivirt.internal"

mTLS Configuration

# agent config
service_discovery:
  tls:
    enabled: true
    cert_file: "/etc/pivirt-agent/tls/server.crt"
    key_file: "/etc/pivirt-agent/tls/server.key"
    ca_file: "/etc/pivirt-agent/tls/ca.crt"

Monitoring

Discovery Metrics

# View discovery metrics
curl http://localhost:8444/metrics | grep discovery

# Key metrics:
# pivirt_discovery_services_registered{service="pivirtd"} 3
# pivirt_discovery_health_checks_total{status="healthy"} 1000
# pivirt_discovery_health_checks_total{status="unhealthy"} 5

Prometheus Alerts

prometheus-rules.yml

groups:
  - name: pivirt.discovery.rules
    rules:
      - alert: PivirtServiceDown
        expr: pivirt_discovery_services_registered{service="pivirtd"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "No pivirtd services registered"

      - alert: PivirtServiceUnhealthy
        expr: pivirt_discovery_health_checks_total{status="unhealthy"} > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Service health checks failing"

Troubleshooting

Service Not Discovered

# Check etcd connectivity
etcdctl endpoint health

# Check registered services
etcdctl get /pivirt/services --prefix

# Check agent logs
journalctl -u pivirt-agent -f | grep -i discovery

Load Balancing Issues

# Check service endpoints
curl -X GET "https://agent.pilab.hu:8443/pivirt.discovery.v1.DiscoveryService/Discover" \
  -d '{"service_name": "pivirtd"}'

# Check load balancer metrics
curl http://localhost:8444/metrics | grep load_balancer

Related

• Architecture
• agent Guide
• Clustering Guide