High Availability Deployment

                        ┌──────────────────────┐
                        │   Terminal Clients    │
                        │  (Tauri desktop app)  │
                        └──────────┬───────────┘
                                   │
                              HTTPS / WSS
                                   │
                        ┌──────────▼───────────┐
                        │   Nginx Load Balancer │
                        │   (least_conn, TLS    │
                        │    termination)        │
                        └────┬────────────┬─────┘
                             │            │
                    ┌────────▼──┐    ┌────▼────────┐
                    │Controller │    │ Controller   │
                    │Instance 1 │    │ Instance 2   │
                    │  :3000    │    │   :3000      │
                    └──┬────┬───┘    └──┬────┬──────┘
                       │    │           │    │
              ┌────────┘    └─────┬─────┘    └────────┐
              │                   │                    │
    ┌─────────▼─────────┐  ┌─────▼──────────┐  ┌─────▼──────────┐
    │    PostgreSQL 16   │  │ Valkey Primary  │  │ Network Devices│
    │   (pgvector ext)   │  │   + Replicas    │  │ (SSH/Telnet)   │
    │                    │  │                 │  │                │
    │  Users, devices,   │  │ Session state,  │  └────────────────┘
    │  credentials,      │  │ instance reg,   │
    │  audit logs,       │  │ heartbeats      │
    │  certificates      │  │                 │
    └────────────────────┘  └────────┬────────┘
                                     │
                            ┌────────▼────────┐
                            │ Valkey Sentinel  │
                            │  (3 instances)   │
                            │                  │
                            │ Monitors primary,│
                            │ auto-promotes    │
                            │ replica on fail  │
                            └─────────────────┘

# Generate the vault master key (64 hex characters = 32 bytes)
# This encrypts all credentials, SSH CA keys, and sensitive data in the database
export VAULT_MASTER_KEY=$(openssl rand -hex 32)
echo "VAULT_MASTER_KEY=$VAULT_MASTER_KEY"

# Generate the JWT signing secret (64 hex characters = 32 bytes)
# This signs all authentication tokens — if changed, all users are logged out
export JWT_SECRET=$(openssl rand -hex 32)
echo "JWT_SECRET=$JWT_SECRET"

# Generate the database password
export DB_PASSWORD=$(openssl rand -base64 24)
echo "DB_PASSWORD=$DB_PASSWORD"

# Save these values securely — you will need them for the docker-compose file
# and for any additional controller instances in HA mode

version: "3.8"

services:
  postgres:
    image: pgvector/pgvector:pg16
    restart: unless-stopped
    environment:
      POSTGRES_USER: netstacks
      POSTGRES_PASSWORD: ${DB_PASSWORD}
      POSTGRES_DB: netstacks
    volumes:
      - postgres-data:/var/lib/postgresql/data
    ports:
      - "127.0.0.1:5432:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U netstacks"]
      interval: 5s
      timeout: 5s
      retries: 5

  controller:
    image: ghcr.io/netstacks/controller:latest
    restart: unless-stopped
    depends_on:
      postgres:
        condition: service_healthy
    ports:
      - "3000:3000"
    environment:
      # Database
      DATABASE_URL: postgres://netstacks:${DB_PASSWORD}@postgres:5432/netstacks

      # Security — generate with: openssl rand -hex 32
      VAULT_MASTER_KEY: ${VAULT_MASTER_KEY}
      JWT_SECRET: ${JWT_SECRET}

      # TLS — auto-generates a self-signed CA on first boot
      # Replace with your own certs for production (see TLS section)
      TLS_CERT_PATH: /data/tls/cert.pem
      TLS_KEY_PATH: /data/tls/key.pem
      TLS_AUTO_GENERATE: "true"

      # Instance naming (optional, shown in Admin UI)
      NETSTACKS_INSTANCE_NAME: "controller-1"

      # Logging
      RUST_LOG: "info,netstacks=debug"
    volumes:
      - controller-data:/data

volumes:
  postgres-data:
  controller-data:

# Create a .env file with your generated secrets
cat > .env << 'EOF'
VAULT_MASTER_KEY=<your-64-hex-char-key>
JWT_SECRET=<your-64-hex-char-jwt-secret>
DB_PASSWORD=<your-database-password>
EOF

# Start the stack
docker compose up -d

# Watch the logs for the initial admin password
docker compose logs -f controller

[INFO] Created default admin user
[INFO] Admin password: Xk9mP2vL8nQwR4tY
[INFO] Change this password immediately after first login

# Check the health endpoint (use -k for self-signed TLS)
curl -k https://localhost:3000/api/v1/health

# Expected response:
# {
#   "status": "healthy",
#   "instance_id": "ctrl-a1b2c3d4",
#   "uptime_secs": 42,
#   "valkey": "not_configured",
#   "database": "connected",
#   "cluster": {
#     "registered_instances": 1,
#     "healthy_instances": 1,
#     "total_sessions": 0,
#     "local_sessions": 0
#   }
# }

# Log in and get a token
curl -k -X POST https://localhost:3000/api/auth/login \
  -H "Content-Type: application/json" \
  -d '{"username": "admin", "password": "<password-from-logs>"}'

version: "3.8"

services:
  # ─────────────────────────────────────────────
  # PostgreSQL (use external DB for production)
  # ─────────────────────────────────────────────
  postgres:
    image: pgvector/pgvector:pg16
    restart: unless-stopped
    environment:
      POSTGRES_USER: netstacks
      POSTGRES_PASSWORD: ${DB_PASSWORD}
      POSTGRES_DB: netstacks
    volumes:
      - postgres-data:/var/lib/postgresql/data
    ports:
      - "127.0.0.1:5432:5432"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U netstacks"]
      interval: 5s
      timeout: 5s
      retries: 5

  # ─────────────────────────────────────────────
  # Valkey Primary
  # ─────────────────────────────────────────────
  valkey-primary:
    image: valkey/valkey:8-alpine
    restart: unless-stopped
    command: >
      valkey-server
      --appendonly yes
      --appendfsync everysec
      --maxmemory 512mb
      --maxmemory-policy noeviction
    volumes:
      - valkey-primary-data:/data
    ports:
      - "127.0.0.1:6379:6379"
    healthcheck:
      test: ["CMD", "valkey-cli", "ping"]
      interval: 5s
      timeout: 3s
      retries: 5

  # ─────────────────────────────────────────────
  # Valkey Replicas
  # ─────────────────────────────────────────────
  valkey-replica-1:
    image: valkey/valkey:8-alpine
    restart: unless-stopped
    command: >
      valkey-server
      --appendonly yes
      --appendfsync everysec
      --replicaof valkey-primary 6379
    volumes:
      - valkey-replica1-data:/data
    depends_on:
      valkey-primary:
        condition: service_healthy

  valkey-replica-2:
    image: valkey/valkey:8-alpine
    restart: unless-stopped
    command: >
      valkey-server
      --appendonly yes
      --appendfsync everysec
      --replicaof valkey-primary 6379
    volumes:
      - valkey-replica2-data:/data
    depends_on:
      valkey-primary:
        condition: service_healthy

  # ─────────────────────────────────────────────
  # Valkey Sentinels (3 for quorum)
  # ─────────────────────────────────────────────
  valkey-sentinel-1:
    image: valkey/valkey:8-alpine
    restart: unless-stopped
    command: >
      valkey-sentinel /etc/valkey/sentinel.conf
    volumes:
      - ./sentinel.conf:/etc/valkey/sentinel.conf:ro
    depends_on:
      valkey-primary:
        condition: service_healthy

  valkey-sentinel-2:
    image: valkey/valkey:8-alpine
    restart: unless-stopped
    command: >
      valkey-sentinel /etc/valkey/sentinel.conf
    volumes:
      - ./sentinel.conf:/etc/valkey/sentinel.conf:ro
    depends_on:
      valkey-primary:
        condition: service_healthy

  valkey-sentinel-3:
    image: valkey/valkey:8-alpine
    restart: unless-stopped
    command: >
      valkey-sentinel /etc/valkey/sentinel.conf
    volumes:
      - ./sentinel.conf:/etc/valkey/sentinel.conf:ro
    depends_on:
      valkey-primary:
        condition: service_healthy

  # ─────────────────────────────────────────────
  # Controller Instance 1
  # ─────────────────────────────────────────────
  controller-1:
    image: ghcr.io/netstacks/controller:latest
    restart: unless-stopped
    depends_on:
      postgres:
        condition: service_healthy
      valkey-primary:
        condition: service_healthy
    environment:
      DATABASE_URL: postgres://netstacks:${DB_PASSWORD}@postgres:5432/netstacks
      VAULT_MASTER_KEY: ${VAULT_MASTER_KEY}
      JWT_SECRET: ${JWT_SECRET}
      NETSTACKS_INSTANCE_NAME: "controller-1"
      RUST_LOG: "info,netstacks=debug"

      # Valkey Sentinel — controllers discover the primary automatically
      VALKEY_SENTINEL_URL: "redis://valkey-sentinel-1:26379,valkey-sentinel-2:26379,valkey-sentinel-3:26379"
      VALKEY_SENTINEL_MASTER_NAME: "mymaster"

      # HA tuning
      HA_HEARTBEAT_INTERVAL_SECS: "5"
      HA_STALE_INSTANCE_TTL_SECS: "30"

      # TLS
      TLS_CERT_PATH: /data/tls/cert.pem
      TLS_KEY_PATH: /data/tls/key.pem
      TLS_AUTO_GENERATE: "true"
    volumes:
      - controller1-data:/data

  # ─────────────────────────────────────────────
  # Controller Instance 2
  # ─────────────────────────────────────────────
  controller-2:
    image: ghcr.io/netstacks/controller:latest
    restart: unless-stopped
    depends_on:
      postgres:
        condition: service_healthy
      valkey-primary:
        condition: service_healthy
    environment:
      DATABASE_URL: postgres://netstacks:${DB_PASSWORD}@postgres:5432/netstacks
      VAULT_MASTER_KEY: ${VAULT_MASTER_KEY}
      JWT_SECRET: ${JWT_SECRET}
      NETSTACKS_INSTANCE_NAME: "controller-2"
      RUST_LOG: "info,netstacks=debug"

      # Same Sentinel config — both instances discover the same Valkey primary
      VALKEY_SENTINEL_URL: "redis://valkey-sentinel-1:26379,valkey-sentinel-2:26379,valkey-sentinel-3:26379"
      VALKEY_SENTINEL_MASTER_NAME: "mymaster"

      HA_HEARTBEAT_INTERVAL_SECS: "5"
      HA_STALE_INSTANCE_TTL_SECS: "30"

      TLS_CERT_PATH: /data/tls/cert.pem
      TLS_KEY_PATH: /data/tls/key.pem
      TLS_AUTO_GENERATE: "true"
    volumes:
      - controller2-data:/data

  # ─────────────────────────────────────────────
  # Nginx Load Balancer
  # ─────────────────────────────────────────────
  nginx:
    image: nginx:1.27-alpine
    restart: unless-stopped
    ports:
      - "443:443"
      - "80:80"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      - ./certs:/etc/nginx/certs:ro
    depends_on:
      - controller-1
      - controller-2

volumes:
  postgres-data:
  valkey-primary-data:
  valkey-replica1-data:
  valkey-replica2-data:
  controller1-data:
  controller2-data:

# sentinel.conf
port 26379

# Monitor the Valkey primary
# Format: sentinel monitor <master-name> <host> <port> <quorum>
sentinel monitor mymaster valkey-primary 6379 2

# Time in ms before a non-responding primary is considered down
sentinel down-after-milliseconds mymaster 5000

# Failover timeout in ms
sentinel failover-timeout mymaster 10000

# How many replicas can sync from the new primary simultaneously
sentinel parallel-syncs mymaster 1

# Deny external writes to sentinel config (security)
sentinel deny-scripts-reconfig yes

worker_processes auto;

events {
    worker_connections 2048;
}

http {
    # Upstream — controller instances
    upstream netstacks_api {
        least_conn;
        server controller-1:3000;
        server controller-2:3000;
    }

    # Connection upgrade map for WebSocket support
    map $http_upgrade $connection_upgrade {
        default upgrade;
        ''      close;
    }

    # Redirect HTTP to HTTPS
    server {
        listen 80;
        server_name _;
        return 301 https://$host$request_uri;
    }

    # Main HTTPS server
    server {
        listen 443 ssl;
        server_name netstacks.example.net;

        # TLS certificates
        ssl_certificate     /etc/nginx/certs/fullchain.pem;
        ssl_certificate_key /etc/nginx/certs/privkey.pem;
        ssl_protocols       TLSv1.2 TLSv1.3;
        ssl_ciphers         HIGH:!aNULL:!MD5;
        ssl_prefer_server_ciphers on;

        # Security headers
        add_header Strict-Transport-Security "max-age=63072000; includeSubDomains" always;
        add_header X-Content-Type-Options "nosniff" always;
        add_header X-Frame-Options "DENY" always;

        # Proxy settings
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;

        # WebSocket support — critical for terminal sessions
        proxy_set_header Upgrade $http_upgrade;
        proxy_set_header Connection $connection_upgrade;

        # Long timeout for terminal sessions (1 hour)
        proxy_read_timeout 3600s;
        proxy_send_timeout 3600s;

        # All traffic to the controller API
        location / {
            proxy_pass https://netstacks_api;

            # TLS to backend (controller uses self-signed certs)
            proxy_ssl_verify off;
        }

        # Health check endpoint for monitoring (no auth required)
        location /api/v1/health {
            proxy_pass https://netstacks_api;
            proxy_ssl_verify off;
            access_log off;
        }
    }
}

# Ensure your .env file has all required secrets
cat .env
# VAULT_MASTER_KEY=<64-hex-chars>
# JWT_SECRET=<64-hex-chars>
# DB_PASSWORD=<database-password>

# Start the full stack
docker compose -f docker-compose-ha.yml up -d

# Verify all containers are running
docker compose -f docker-compose-ha.yml ps

# Check cluster health from the load balancer
curl -k https://localhost/api/v1/health

# The response should show 2 registered instances:
# {
#   "status": "healthy",
#   "cluster": {
#     "registered_instances": 2,
#     "healthy_instances": 2,
#     ...
#   }
# }

# Verify both instances individually
docker compose -f docker-compose-ha.yml exec controller-1 \
  curl -sk https://localhost:3000/api/v1/health | jq .instance_id

docker compose -f docker-compose-ha.yml exec controller-2 \
  curl -sk https://localhost:3000/api/v1/health | jq .instance_id

# Connect to PostgreSQL as superuser
psql -h db.example.net -U postgres

# Create the netstacks database and user
CREATE USER netstacks WITH PASSWORD 'your-secure-password';
CREATE DATABASE netstacks OWNER netstacks;

# Connect to the netstacks database
\c netstacks

# Install the pgvector extension (required for AI knowledge base)
CREATE EXTENSION IF NOT EXISTS vector;

# Verify the extension is installed
SELECT extname, extversion FROM pg_extension WHERE extname = 'vector';
#  extname | extversion
# ---------+------------
#  vector  | 0.7.4

# Grant schema permissions
GRANT ALL PRIVILEGES ON DATABASE netstacks TO netstacks;
GRANT ALL PRIVILEGES ON SCHEMA public TO netstacks;

# pgbouncer.ini
[databases]
netstacks = host=db.example.net port=5432 dbname=netstacks

[pgbouncer]
listen_addr = 0.0.0.0
listen_port = 6432
auth_type = md5
auth_file = /etc/pgbouncer/userlist.txt

# Transaction pooling is recommended for NetStacks
pool_mode = transaction
max_client_conn = 200
default_pool_size = 25
min_pool_size = 5
reserve_pool_size = 5

# Timeouts
server_idle_timeout = 600
client_idle_timeout = 0
server_connect_timeout = 15

# userlist.txt — PgBouncer auth file
# Format: "username" "password"
"netstacks" "your-secure-password"

# Add PgBouncer to your docker-compose-ha.yml
  pgbouncer:
    image: edoburu/pgbouncer:1.23.1-p2
    restart: unless-stopped
    volumes:
      - ./pgbouncer.ini:/etc/pgbouncer/pgbouncer.ini:ro
      - ./userlist.txt:/etc/pgbouncer/userlist.txt:ro
    ports:
      - "127.0.0.1:6432:6432"
    depends_on:
      - postgres

# Then update DATABASE_URL on each controller instance:
# DATABASE_URL: postgres://netstacks:<password>@pgbouncer:6432/netstacks

# Controller environment (already in the compose examples above)
environment:
  TLS_CERT_PATH: /data/tls/cert.pem
  TLS_KEY_PATH: /data/tls/key.pem
  TLS_AUTO_GENERATE: "true"

# Generate a CSR and private key (if using an internal CA)
openssl req -new -newkey rsa:4096 -nodes \
  -keyout netstacks.key \
  -out netstacks.csr \
  -subj "/CN=netstacks.example.net/O=Example Corp"

# After your CA signs the CSR, you'll have:
# - netstacks.key     (private key)
# - netstacks.crt     (signed certificate)
# - ca-chain.crt      (CA certificate chain)

# Create a full chain file
cat netstacks.crt ca-chain.crt > fullchain.pem

# Verify the certificate
openssl x509 -in fullchain.pem -text -noout | head -20

# Mount your certificates into the controller container
  controller-1:
    image: ghcr.io/netstacks/controller:latest
    environment:
      TLS_CERT_PATH: /certs/fullchain.pem
      TLS_KEY_PATH: /certs/netstacks.key
      TLS_AUTO_GENERATE: "false"
    volumes:
      - ./certs/fullchain.pem:/certs/fullchain.pem:ro
      - ./certs/netstacks.key:/certs/netstacks.key:ro

# Install certbot
sudo apt-get update && sudo apt-get install -y certbot

# Obtain a certificate (standalone mode — stop Nginx first)
sudo certbot certonly --standalone \
  -d netstacks.example.net \
  --agree-tos \
  --email admin@example.net \
  --non-interactive

# Certificate files are at:
# /etc/letsencrypt/live/netstacks.example.net/fullchain.pem
# /etc/letsencrypt/live/netstacks.example.net/privkey.pem

# Set up automatic renewal (runs twice daily by default)
sudo systemctl enable --now certbot.timer

# Test renewal
sudo certbot renew --dry-run

# Update nginx volumes to mount Let's Encrypt certificates
  nginx:
    image: nginx:1.27-alpine
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      - /etc/letsencrypt/live/netstacks.example.net:/etc/nginx/certs:ro
      - /etc/letsencrypt/archive/netstacks.example.net:/etc/nginx/certs-archive:ro

# In nginx.conf, reference the certificates:
#   ssl_certificate     /etc/nginx/certs/fullchain.pem;
#   ssl_certificate_key /etc/nginx/certs/privkey.pem;

# Example: AWS Route 53 health-checked failover
# Primary record (us-east-1)
netstacks.example.net  A  203.0.113.10  (failover: PRIMARY, health-check: /api/v1/health)

# Secondary record (eu-west-1)
netstacks.example.net  A  198.51.100.20 (failover: SECONDARY, health-check: /api/v1/health)

# Alternative: latency-based routing (active-active geo)
netstacks.example.net  A  203.0.113.10  (region: us-east-1, health-check: enabled)
netstacks.example.net  A  198.51.100.20 (region: eu-west-1, health-check: enabled)

# Basic health check (no auth required)
curl -k https://netstacks.example.net/api/v1/health

{
  "status": "healthy",
  "instance_id": "ctrl-a1b2c3d4",
  "instance_name": "controller-1",
  "uptime_secs": 86400,
  "version": "0.1.0",
  "valkey": "connected",
  "database": "connected",
  "cluster": {
    "registered_instances": 2,
    "healthy_instances": 2,
    "total_sessions": 42,
    "local_sessions": 21
  }
}

# prometheus.yml — scrape NetStacks health via json_exporter
scrape_configs:
  - job_name: 'netstacks-health'
    metrics_path: /probe
    params:
      module: [netstacks]
    static_configs:
      - targets:
          - https://netstacks.example.net/api/v1/health
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: json-exporter:7979

---
# json_exporter config (config.yml)
modules:
  netstacks:
    metrics:
      - name: netstacks_up
        path: '{ .status }'
        help: "Controller health status"
        values:
          healthy: 1
          degraded: 0
      - name: netstacks_uptime_seconds
        path: '{ .uptime_secs }'
        help: "Controller uptime in seconds"
      - name: netstacks_cluster_instances
        path: '{ .cluster.registered_instances }'
        help: "Number of registered controller instances"
      - name: netstacks_cluster_healthy_instances
        path: '{ .cluster.healthy_instances }'
        help: "Number of healthy controller instances"
      - name: netstacks_sessions_total
        path: '{ .cluster.total_sessions }'
        help: "Total active sessions across the cluster"
      - name: netstacks_sessions_local
        path: '{ .cluster.local_sessions }'
        help: "Active sessions on this instance"

# Check cluster health from a monitoring script
#!/bin/bash
RESPONSE=$(curl -sk https://netstacks.example.net/api/v1/health)
STATUS=$(echo "$RESPONSE" | jq -r '.status')
HEALTHY=$(echo "$RESPONSE" | jq -r '.cluster.healthy_instances')
TOTAL=$(echo "$RESPONSE" | jq -r '.cluster.registered_instances')

if [ "$STATUS" != "healthy" ]; then
  echo "CRITICAL: NetStacks controller is $STATUS"
  exit 2
fi

if [ "$HEALTHY" -lt "$TOTAL" ]; then
  echo "WARNING: $HEALTHY/$TOTAL instances healthy"
  exit 1
fi

echo "OK: $HEALTHY/$TOTAL instances healthy, $STATUS"
exit 0

# Backup the full database (compressed)
pg_dump -h db.example.net -U netstacks -d netstacks \
  --format=custom \
  --compress=9 \
  --file=netstacks-backup-$(date +%Y%m%d-%H%M%S).dump

# Backup only the schema (for documentation or migration reference)
pg_dump -h db.example.net -U netstacks -d netstacks \
  --schema-only \
  --file=netstacks-schema-$(date +%Y%m%d).sql

# Backup specific tables (e.g., just users and devices)
pg_dump -h db.example.net -U netstacks -d netstacks \
  --format=custom \
  --table=users --table=devices --table=credentials \
  --file=netstacks-core-$(date +%Y%m%d-%H%M%S).dump

# Add to your docker-compose-ha.yml
  db-backup:
    image: prodrigestivill/postgres-backup-local:16
    restart: unless-stopped
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      POSTGRES_HOST: postgres
      POSTGRES_DB: netstacks
      POSTGRES_USER: netstacks
      POSTGRES_PASSWORD: ${DB_PASSWORD}
      POSTGRES_EXTRA_OPTS: "--format=custom --compress=9"

      # Schedule: daily at 2:00 AM
      SCHEDULE: "0 2 * * *"

      # Retention policy
      BACKUP_KEEP_DAYS: 7      # Keep daily backups for 7 days
      BACKUP_KEEP_WEEKS: 4     # Keep weekly backups for 4 weeks
      BACKUP_KEEP_MONTHS: 6    # Keep monthly backups for 6 months

      # Healthcheck URL (optional, for monitoring)
      HEALTHCHECK_PORT: 8080
    volumes:
      - db-backups:/backups

volumes:
  db-backups:
    driver: local
    driver_opts:
      type: none
      o: bind
      device: /opt/netstacks/backups

# Stop the controller instances first
docker compose -f docker-compose-ha.yml stop controller-1 controller-2

# Restore from a custom-format backup
pg_restore -h db.example.net -U netstacks -d netstacks \
  --clean \
  --if-exists \
  --no-owner \
  --no-privileges \
  netstacks-backup-20260410-020000.dump

# Verify the restore
psql -h db.example.net -U netstacks -d netstacks \
  -c "SELECT count(*) FROM users; SELECT count(*) FROM devices;"

# Restart the controller instances
docker compose -f docker-compose-ha.yml start controller-1 controller-2

# Backup controller data volumes
docker run --rm \
  -v netstacks_controller1-data:/source:ro \
  -v /opt/netstacks/backups:/backup \
  alpine tar czf /backup/controller1-data-$(date +%Y%m%d).tar.gz -C /source .

docker run --rm \
  -v netstacks_controller2-data:/source:ro \
  -v /opt/netstacks/backups:/backup \
  alpine tar czf /backup/controller2-data-$(date +%Y%m%d).tar.gz -C /source .

# Pull the latest image
docker compose pull controller

# Restart with the new image
docker compose up -d controller

# Watch the logs for migration output and successful startup
docker compose logs -f controller

# Verify health
curl -k https://localhost:3000/api/v1/health

# Pull the latest image on all hosts
docker compose -f docker-compose-ha.yml pull

# Step 1: Upgrade controller-1
# Stop controller-1 — nginx health check removes it from the pool
docker compose -f docker-compose-ha.yml stop controller-1

# Start controller-1 with the new image
docker compose -f docker-compose-ha.yml up -d controller-1

# Wait for controller-1 to become healthy
until curl -sk https://localhost/api/v1/health | grep -q '"healthy_instances": 2'; do
  echo "Waiting for controller-1 to rejoin cluster..."
  sleep 5
done
echo "controller-1 is healthy"

# Step 2: Upgrade controller-2
docker compose -f docker-compose-ha.yml stop controller-2
docker compose -f docker-compose-ha.yml up -d controller-2

# Wait for controller-2 to become healthy
until curl -sk https://localhost/api/v1/health | grep -q '"healthy_instances": 2'; do
  echo "Waiting for controller-2 to rejoin cluster..."
  sleep 5
done
echo "Upgrade complete — both instances healthy"

# Verify the new version
curl -sk https://localhost/api/v1/health | jq '.version'

# Check the currently running version
docker compose -f docker-compose-ha.yml exec controller-1 \
  curl -sk https://localhost:3000/api/v1/health | jq '.version'

# Roll back to a specific version
# Edit docker-compose-ha.yml (or set the image tag):
#   controller-1:
#     image: ghcr.io/netstacks/controller:0.0.8  # previous version

# Or use an environment variable override
export CONTROLLER_IMAGE=ghcr.io/netstacks/controller:0.0.8

docker compose -f docker-compose-ha.yml stop controller-1 controller-2

# Start with the previous version
docker compose -f docker-compose-ha.yml up -d controller-1
sleep 10
docker compose -f docker-compose-ha.yml up -d controller-2

# Verify the rollback
curl -sk https://localhost/api/v1/health | jq '.version'