使用 lavik-ctl 部署主从高可用集群

./lavik --version
./lavik-meta --version
./lavik-ctl --version

# Run from the extracted v0.1.0-beta.1 package directory.
export LAVIK_BIN_DIR="$(pwd -P)"
export LAVIK_ROOT="$LAVIK_BIN_DIR/ha-beta1"
# Initialization is for a new, empty directory only.
test ! -e "$LAVIK_ROOT" || { echo "Already exists: $LAVIK_ROOT" >&2; exit 1; }
umask 077
mkdir -p "$LAVIK_ROOT"
for i in 1 2 3; do mkdir "$LAVIK_ROOT/meta-$i"; done
for i in 1 2; do
  mkdir "$LAVIK_ROOT/data-$i"
  fallocate -l 512M "$LAVIK_ROOT/data-$i/lavik.data"
done
cat > "$LAVIK_ROOT/cluster.toml" <<'TOML'
schema_version = 1
slot_strategy = "contiguous-even"

[[meta_members]]
id = 1
raft_endpoint = "tcp://127.0.0.1:7101"
data_control_endpoint = "tcp://127.0.0.1:7301"
ctl_endpoint = "tcp://127.0.0.1:7201"

[[meta_members]]
id = 2
raft_endpoint = "tcp://127.0.0.1:7102"
data_control_endpoint = "tcp://127.0.0.1:7302"
ctl_endpoint = "tcp://127.0.0.1:7202"

[[meta_members]]
id = 3
raft_endpoint = "tcp://127.0.0.1:7103"
data_control_endpoint = "tcp://127.0.0.1:7303"
ctl_endpoint = "tcp://127.0.0.1:7203"

[[data_nodes]]
id = "1111111111111111111111111111111111111111"
client_endpoint = "tcp://127.0.0.1:6371"

[[data_nodes]]
id = "2222222222222222222222222222222222222222"
client_endpoint = "tcp://127.0.0.1:6372"

[[groups]]
id = "group-1"
primary = "1111111111111111111111111111111111111111"
replicas = ["2222222222222222222222222222222222222222"]
TOML

# Reuse these paths after a restart; do not initialize the directory again.
: "${LAVIK_BIN_DIR:?Set the absolute package directory}"
: "${LAVIK_ROOT:?Set the existing state directory}"
test -f "$LAVIK_ROOT/cluster.toml"
for pidfile in "$LAVIK_ROOT"/*.pid; do
  [ -f "$pidfile" ] || continue
  if kill -0 "$(cat "$pidfile")" 2>/dev/null; then
    echo "A recorded process is still running: $pidfile" >&2
    exit 1
  fi
done
for i in 1 2 3; do
  bootstrap=()
  if [ ! -f "$LAVIK_ROOT/meta-$i/cluster_config.dat" ]; then
    bootstrap=(--initial-cluster-manifest "$LAVIK_ROOT/cluster.toml")
  fi
  nohup "$LAVIK_BIN_DIR/lavik-meta" --id "$i" \
    --addr "127.0.0.1:$((7100+i))" \
    --ctl-addr "127.0.0.1:$((7200+i))" \
    --data-control-addr "127.0.0.1:$((7300+i))" \
    --data-dir "$LAVIK_ROOT/meta-$i" "${bootstrap[@]}" \
    > "$LAVIK_ROOT/meta-$i.log" 2>&1 < /dev/null &
  echo "$!" > "$LAVIK_ROOT/meta-$i.pid"
done
# Wait for a leader before submitting the creation request.
deadline=$((SECONDS+30))
leader=0
while (( SECONDS < deadline )); do
  for directory in "$LAVIK_ROOT"/meta-*; do
    [ -d "$directory" ] || continue
    output=$("$LAVIK_BIN_DIR/lavik-ctl" --socket "$directory/meta-admin.sock" status 2>/dev/null) || continue
    if [[ "$output" == *"leader=1"* ]]; then leader=1; break; fi
  done
  (( leader == 1 )) && break
  sleep 1
done
(( leader == 1 )) || { echo "No Meta leader: inspect $LAVIK_ROOT/meta-*.log" >&2; exit 1; }

for i in 1 2; do
  node_id=$(printf '%040d' 0 | tr '0' "$i")
  nohup "$LAVIK_BIN_DIR/lavik" --bind 127.0.0.1 \
    --port "$((6370+i))" --metrics-port "$((9100+i))" \
    --network kernel --storage uring --threads 1 --no-pin-workers \
    --registered-buffer-mb-per-worker 64 --max-memory 512MiB \
    --shutdown-checkpoint --cluster-enabled --cluster-node-id "$node_id" \
    --cluster-announce-ip 127.0.0.1 \
    --cluster-meta-seed 127.0.0.1:7301 \
    --cluster-meta-seed 127.0.0.1:7302 \
    --cluster-meta-seed 127.0.0.1:7303 \
    --data-file "$LAVIK_ROOT/data-$i/lavik.data" \
    --log-dir "$LAVIK_ROOT/data-$i/logs" \
    > "$LAVIK_ROOT/data-$i.log" 2>&1 < /dev/null &
  echo "$!" > "$LAVIK_ROOT/data-$i.pid"
done

# Destructive, one-time initialization of the fresh Data files above.
# --yes accepts the displayed plan; omit it to confirm interactively.
"$LAVIK_BIN_DIR/lavik-ctl" cluster-create \
  --manifest "$LAVIK_ROOT/cluster.toml" \
  --socket "$LAVIK_ROOT/meta-1/meta-admin.sock" \
  --allow-plaintext-admin --yes

# cluster-create returning 0 means accepted, not ready to serve.
# Retry status reads only; never automatically retry cluster-create.
deadline=$((SECONDS+120))
ready=0
while (( SECONDS < deadline )); do
  if "$LAVIK_BIN_DIR/lavik-ctl" cluster-status \
    --socket "$LAVIK_ROOT/meta-1/meta-admin.sock" \
    --allow-plaintext-admin; then
    ready=1
    break
  else
    code=$?
    if (( code != 2 && code != 3 )); then exit "$code"; fi
  fi
  sleep 1
done
(( ready == 1 )) || { echo "Not ready: inspect status and logs; do not recreate." >&2; exit 1; }

"$LAVIK_BIN_DIR/lavik-ctl" cluster-status \
  --socket "$LAVIK_ROOT/meta-1/meta-admin.sock" \
  --allow-plaintext-admin --json

# Initial creation only: node 2 is the follower. After failover, discover roles again.
deadline=$((SECONDS+120))
synced=0
while (( SECONDS < deadline )); do
  if ! info=$(redis-cli -h 127.0.0.1 -p 6372 --raw INFO replication 2>/dev/null); then
    sleep 1
    continue
  fi
  if [[ "$info" == *"lavik_replication_failed_stopped:1"* ]]; then
    printf '%s\n' "$info" >&2
    echo "Follower replication stopped; inspect logs before a state-preserving follower restart." >&2
    break
  fi
  if [[ "$info" == *"master_link_status:up"* && "$info" == *"master_sync_in_progress:0"* ]]; then
    synced=1
    printf '%s\n' "$info"
    break
  fi
  sleep 1
done
if (( synced != 1 )); then
  printf '%s\n' "$info" >&2
  echo "Follower is not synchronized; inspect its log and Meta observations." >&2
fi
# Return a failed check without explicitly exiting a reader's interactive shell.
(( synced == 1 ))

# Only for node 2 in the initial two-Data lab, before any failover.
# Preserve its data file, node ID and Meta state. Requires python3.
(
if ! status=$("$LAVIK_BIN_DIR/lavik-ctl" cluster-status \
  --socket "$LAVIK_ROOT/meta-1/meta-admin.sock" --allow-plaintext-admin --json); then
  echo "Cluster status is not ready; stop and inspect." >&2
  exit 1
fi
printf '%s' "$status" | python3 -c '
import json,sys
s=json.load(sys.stdin)
assert s["result"] == "ready" and s["cluster_state"] == "created"
assert all(s[k] is True for k in ("meta_available", "meta_membership_stable", "topology_converged", "serving_ready", "cluster_ready"))
assert s["blockers"] == [] and len(s["groups"]) == 1
g = s["groups"][0]
assert g["group_id"] == "group-1" and g["term"] == "1"
assert g["owner_node_id"] == "1"*40 and g["serving_ready"] is True
assert g["topology_converged"] is True and g["automatic_failover_state"] == "healthy"
assert g["current_reason"] is None and g["blocked_reason"] is None
assert len(s["data_nodes"]) == 2
for node_id, role in (("1"*40, "primary"), ("2"*40, "replica")):
    n = next(n for n in s["data_nodes"] if n["node_id"] == node_id)
    assert n["role"] == role and n["group_id"] == "group-1" and n["retired"] is False
    assert all(n[k] is True for k in ("current_session", "projection_current", "health_fresh", "population_current"))
' || { echo "Initial serving roles, term or transition checks failed; stop and inspect." >&2; exit 1; }
pid=$(cat "$LAVIK_ROOT/data-2.pid")
test "$(readlink "/proc/$pid/exe")" = "$LAVIK_BIN_DIR/lavik" || exit 1
tr '\0' '\n' < "/proc/$pid/cmdline" | grep -F -- "$LAVIK_ROOT/data-2/lavik.data" > /dev/null || exit 1
kill -TERM "$pid"
deadline=$((SECONDS+60))
while [ "$(readlink "/proc/$pid/exe" 2>/dev/null)" = "$LAVIK_BIN_DIR/lavik" ]; do
  (( SECONDS < deadline )) || { echo "Follower did not stop; inspect logs." >&2; exit 1; }
  sleep 1
done
nohup "$LAVIK_BIN_DIR/lavik" --bind 127.0.0.1 --port 6372 --metrics-port 9102 \
  --network kernel --storage uring --threads 1 --no-pin-workers \
  --registered-buffer-mb-per-worker 64 --max-memory 512MiB --shutdown-checkpoint \
  --cluster-enabled --cluster-node-id 2222222222222222222222222222222222222222 \
  --cluster-announce-ip 127.0.0.1 \
  --cluster-meta-seed 127.0.0.1:7301 --cluster-meta-seed 127.0.0.1:7302 \
  --cluster-meta-seed 127.0.0.1:7303 \
  --data-file "$LAVIK_ROOT/data-2/lavik.data" --log-dir "$LAVIK_ROOT/data-2/logs" \
  >> "$LAVIK_ROOT/data-2.log" 2>&1 < /dev/null &
echo "$!" > "$LAVIK_ROOT/data-2.pid"
)

# SET and WAIT must share a connection. WAIT is not a failover/durability guarantee.
redis-cli -c -h 127.0.0.1 -p 6371 --raw <<'COMMANDS'
SET greeting "hello from Lavik"
WAIT 1 5000
GET greeting
COMMANDS
# The replica redirects this read to the primary; -c follows that redirect.
redis-cli -c -h 127.0.0.1 -p 6372 GET greeting
redis-cli -h 127.0.0.1 -p 6371 CLUSTER SLOTS

# These are local committed reads. Select the current Meta leader first.
# Check status on each member: leader=1 identifies the leader at that moment.
for directory in "$LAVIK_ROOT"/meta-*; do
  [ -d "$directory" ] || continue
  "$LAVIK_BIN_DIR/lavik-ctl" --socket "$directory/meta-admin.sock" status
done

export LAVIK_LEADER_SOCKET=''
for directory in "$LAVIK_ROOT"/meta-*; do
  [ -d "$directory" ] || continue
  output=$("$LAVIK_BIN_DIR/lavik-ctl" --socket "$directory/meta-admin.sock" status) || continue
  if [[ "$output" == *"leader=1"* ]]; then
    export LAVIK_LEADER_SOCKET="$directory/meta-admin.sock"
    break
  fi
done
: "${LAVIK_LEADER_SOCKET:?No current Meta leader; inspect quorum and logs}"
"$LAVIK_BIN_DIR/lavik-ctl" --socket "$LAVIK_LEADER_SOCKET" observations group-1
"$LAVIK_BIN_DIR/lavik-ctl" --socket "$LAVIK_LEADER_SOCKET" getpolicy lavik.automatic-uncontrolled-failover-v1
"$LAVIK_BIN_DIR/lavik-ctl" --socket "$LAVIK_LEADER_SOCKET" getpolicy lavik.authority-lease-v1
"$LAVIK_BIN_DIR/lavik-ctl" --socket "$LAVIK_LEADER_SOCKET" getpolicy lavik.candidate-recovery-v1

"$LAVIK_BIN_DIR/lavik-ctl" failover group-1 \
  --socket "$LAVIK_ROOT/meta-1/meta-admin.sock" \
  --allow-plaintext-admin --failover-timeout-ms 120000

# First export LAVIK_OPERATION_ID to the actual ID printed by the CLI.
# Refresh LAVIK_LEADER_SOCKET using the leader-discovery block above.
: "${LAVIK_OPERATION_ID:?Set the operation ID printed by cluster-create or failover}"
: "${LAVIK_LEADER_SOCKET:?Discover the current Meta leader first}"
"$LAVIK_BIN_DIR/lavik-ctl" --socket "$LAVIK_LEADER_SOCKET" getop "$LAVIK_OPERATION_ID"

"$LAVIK_BIN_DIR/lavik-ctl" cluster-status \
  --socket "$LAVIK_ROOT/meta-1/meta-admin.sock" \
  --allow-plaintext-admin --json

# Linux lab process management; this does not erase Meta or Data state.
: "${LAVIK_ROOT:?Set the existing state directory}"
: "${LAVIK_BIN_DIR:?Set the absolute package directory}"
owns_pid() {
  local pid="$1" executable
  [[ "$pid" =~ ^[0-9]+$ ]] || return 1
  executable=$(readlink "/proc/$pid/exe") || return 1
  [[ "$executable" == "$LAVIK_BIN_DIR/lavik" || "$executable" == "$LAVIK_BIN_DIR/lavik-meta" ]] || return 1
  tr '\0' '\n' < "/proc/$pid/cmdline" | grep -Fq -- "$LAVIK_ROOT/"
}
# Stop Data first, then Meta. Send SIGTERM and wait instead of using kill -9.
for role in data meta; do
  for pidfile in "$LAVIK_ROOT"/"$role"-*.pid; do
    [ -f "$pidfile" ] || continue
    pid=$(cat "$pidfile")
    if owns_pid "$pid"; then kill -TERM "$pid"; fi
  done
  deadline=$((SECONDS+60))
  while :; do
    running=0
    for pidfile in "$LAVIK_ROOT"/"$role"-*.pid; do
      [ -f "$pidfile" ] || continue
      if owns_pid "$(cat "$pidfile")"; then running=1; fi
    done
    (( running == 0 )) && break
    (( SECONDS < deadline )) || { echo "Shutdown timed out; inspect logs." >&2; exit 1; }
    sleep 1
  done
  for pidfile in "$LAVIK_ROOT"/"$role"-*.pid; do
    [ ! -f "$pidfile" ] || rm "$pidfile"
  done
done

curl -fsS http://127.0.0.1:9101/metrics
curl -fsS http://127.0.0.1:9102/metrics

# Use a new directory on the Linux host that will run monitoring.
mkdir lavik-monitoring-beta1
cd lavik-monitoring-beta1
curl -fL https://github.com/eloqdata/lavik/archive/3955b98d43b312324aa8d52775df52cfb111c0d0.tar.gz -o source.tar.gz
(
# These upstream configuration files contain no credentials.
# Containers run as other users and need readable files/traversable directories.
umask 022
tar --no-same-permissions -xzf source.tar.gz --strip-components=3 \
  lavik-3955b98d43b312324aa8d52775df52cfb111c0d0/deploy/monitoring
)

# Run inside lavik-monitoring-beta1. Keep this file private.
# Do not overwrite an existing monitoring installation's credentials.
test ! -e .env || { echo ".env already exists; edit it instead." >&2; exit 1; }
umask 077
password=$(openssl rand -hex 24)
printf '%s\n' \
  'LAVIK_TARGETS=127.0.0.1:9101,127.0.0.1:9102' \
  'GRAFANA_ADMIN_USER=admin' \
  "GRAFANA_ADMIN_PASSWORD=$password" \
  'GRAFANA_BIND_ADDRESS=127.0.0.1' \
  'PROMETHEUS_BIND_ADDRESS=127.0.0.1' > .env
unset password

# Linux Docker Engine, Compose v2.24.4+.
# Share the Linux host network to scrape this guide's loopback-only Lavik nodes.
cat > compose.local.yaml <<'YAML'
services:
  prometheus:
    network_mode: host
    ports: !reset []
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --storage.tsdb.retention.time=30d
      - --web.listen-address=127.0.0.1:9090
  grafana:
    network_mode: host
    ports: !reset []
    environment:
      GF_SERVER_HTTP_ADDR: 127.0.0.1
    extra_hosts:
      - prometheus:127.0.0.1
YAML
# --quiet validates without printing the Grafana password.
docker compose --env-file .env -f compose.yaml -f compose.local.yaml config --quiet
docker compose --env-file .env -f compose.yaml -f compose.local.yaml up -d
docker compose --env-file .env -f compose.yaml -f compose.local.yaml ps

# Edit LAVIK_TARGETS in this monitoring directory's .env first.
docker compose --env-file .env -f compose.yaml -f compose.local.yaml \
  up --force-recreate --exit-code-from target-config target-config
# After exit 0, allow up to 30 seconds for discovery, then check Prometheus Targets.

# Keep named volumes, dashboards and metric history.
docker compose --env-file .env -f compose.yaml -f compose.local.yaml down