Satellite Heartbeat System

Satellite (Every 30s)          Backend
     │                            │
     │─── POST /heartbeat ───────▶│
     │    {                        │
     │      system_metrics,        │
     │      mcp_status,            │
     │      processes_by_team      │
     │    }                        │
     │                            │
     │◀── { success: true } ──────│
     │                            │
     │                            │─── Update satellites.last_heartbeat
     │                            │─── Insert satelliteHeartbeats record
     │                            │─── Store mcp_status JSON

interface HeartbeatPayload {
  status: 'active' | 'degraded' | 'error';
  system_metrics: SystemMetrics;
  processes: ProcessInfo[];  // Legacy HTTP proxy processes
  processes_by_team: Record<string, StdioProcessMetric[]>;
  normalized_data?: NormalizedHeartbeatData;  // Large-scale deployments
  mcp_status?: McpStatusSnapshot;  // MCP server runtime status
  error_count: number;
  version: string;
}

interface SystemMetrics {
  cpu_usage_percent: number;    // 0-100
  memory_usage_mb: number;       // Heap memory in MB
  disk_usage_percent: number;    // 0-100
  uptime_seconds: number;        // Process uptime
}

interface McpStatusSnapshot {
  summary: {
    total_instances: number;      // Total running instances
    active_instances: number;     // Instances in 'running' state
    dormant_instances: number;    // Idle processes (cached for respawn)
    total_tools: number;          // Total discovered tools
    online_servers: number;       // Servers in 'online' state
    offline_servers: number;      // Servers in 'offline' state
    error_servers: number;        // Servers in 'error' state
  };
  instances: Array<{
    installation_id: string;      // Installation UUID
    installation_name: string;    // {server}-{team}-{user}-{id}
    instance_id: string;          // Per-user instance identifier
    user_id: string;              // User owning this instance
    team_id: string;              // Team identifier
    status: string;               // 'starting' | 'running' | 'failed'
    transport_type: string;       // 'stdio' | 'http' | 'sse'
    uptime_seconds: number;       // Instance uptime
    message_count: number;        // Total messages sent
    error_count: number;          // Total errors
    health_status: string;        // 'healthy' | 'unhealthy' | 'unknown'
    pid?: number;                 // Process ID (stdio only)
  }>;
  tool_names: Array<{
    namespaced_name: string;      // {server-slug}:{tool-name}
    server_slug: string;          // Server identifier
    transport: string;            // 'stdio' | 'http' | 'sse'
  }>;
  server_status_counts: {
    online: number;
    offline: number;
    error: number;
    requires_reauth: number;
  };
}

UPDATE satellites
SET
  last_heartbeat = NOW(),
  status = 'active',
  mcp_status = :mcp_status_json
WHERE id = :satelliteId;

INSERT INTO satelliteHeartbeats (
  id,
  satellite_id,
  status,
  system_metrics,     -- JSON string
  process_count,
  healthy_process_count,
  error_count,
  uptime_seconds,
  version,
  timestamp
) VALUES (...);

export const satellites = pgTable('satellites', {
  id: text('id').primaryKey(),
  name: text('name').notNull(),
  satellite_type: text('satellite_type', { enum: ['global', 'team'] }).notNull(),
  team_id: text('team_id'),
  status: text('status', { enum: ['active', 'inactive', 'maintenance', 'error'] }).notNull(),
  last_heartbeat: timestamp('last_heartbeat', { withTimezone: true }),
  mcp_status: text('mcp_status'),  // JSON: Latest MCP status (updated on each heartbeat)
  satellite_url: text('satellite_url').notNull(),
  region: text('region'),
  // ... other fields
});

export const satelliteHeartbeats = pgTable('satelliteHeartbeats', {
  id: text('id').primaryKey(),
  satellite_id: text('satellite_id').notNull(),
  status: text('status', { enum: ['active', 'degraded', 'error'] }).notNull(),
  system_metrics: text('system_metrics').notNull(),  // JSON
  process_count: integer('process_count').notNull().default(0),
  healthy_process_count: integer('healthy_process_count').notNull().default(0),
  error_count: integer('error_count').notNull().default(0),
  response_time_ms: integer('response_time_ms'),
  uptime_seconds: integer('uptime_seconds'),
  version: text('version'),
  timestamp: timestamp('timestamp', { withTimezone: true }).notNull().defaultNow(),
});

SELECT
  id,
  name,
  status,
  last_heartbeat,
  mcp_status::json AS mcp_status
FROM satellites
WHERE id = 'sat-xxx';

SELECT
  id,
  name,
  mcp_status::json->'summary'->>'total_instances' AS total_instances,
  mcp_status::json->'summary'->>'active_instances' AS active_instances,
  mcp_status::json->'summary'->>'total_tools' AS total_tools,
  mcp_status::json->'summary'->>'online_servers' AS online_servers
FROM satellites
WHERE id = 'sat-xxx';

SELECT
  id,
  name,
  jsonb_array_elements(mcp_status::jsonb->'instances') AS instance
FROM satellites
WHERE id = 'sat-xxx';

SELECT
  timestamp,
  system_metrics::json->'memory_usage_mb' AS memory_mb,
  system_metrics::json->'uptime_seconds' AS uptime,
  process_count,
  healthy_process_count
FROM satelliteHeartbeats
WHERE satellite_id = 'sat-xxx'
ORDER BY timestamp DESC
LIMIT 100;

SELECT
  id,
  name,
  last_heartbeat,
  NOW() - last_heartbeat AS lag,
  status
FROM satellites
WHERE status = 'active'
ORDER BY lag DESC;

cd services/satellite
LOG_LEVEL=debug npm run dev

{
  "level": "debug",
  "operation": "mcp_status_snapshot_collected",
  "total_instances": 5,
  "total_tools": 42,
  "msg": "Collected MCP status snapshot for heartbeat"
}

{
  "level": "debug",
  "satelliteId": "sat-xxx",
  "msg": "Heartbeat sent successfully"
}

{
  "level": "error",
  "error": "Request timeout",
  "msg": "Failed to send heartbeat"
}