108 lines
3.1 KiB
Elixir
108 lines
3.1 KiB
Elixir
defmodule Backend.NodeClusterIntegrationTests do
|
|
use ExUnit.Case, async: false
|
|
alias Backend.GameRunner
|
|
|
|
describe "GameRunner singleton across a 4-node cluster" do
|
|
test "GameRunner runs on exactly one of the four nodes" do
|
|
peers = start_cluster(4)
|
|
peer_nodes = Enum.map(peers, fn {_p, n} -> n end)
|
|
|
|
game_runner_pids =
|
|
Enum.map(peers, fn {peer_pid, peer_node_name} ->
|
|
runner_pid = :peer.call(peer_pid, :global, :whereis_name, [Backend.GameRunner])
|
|
|
|
assert is_pid(runner_pid),
|
|
"#{peer_node_name} could not find GameRunner in :global registry"
|
|
|
|
runner_pid
|
|
end)
|
|
|
|
assert length(Enum.uniq(game_runner_pids)) == 1,
|
|
"Peers disagree on GameRunner PID: #{inspect(Enum.uniq(game_runner_pids))}"
|
|
|
|
runner_pid = hd(game_runner_pids)
|
|
|
|
owner_node = node(runner_pid)
|
|
|
|
assert owner_node in peer_nodes,
|
|
"Owner node #{owner_node} is not one of the cluster peers #{inspect(peer_nodes)}"
|
|
|
|
assert length(peer_nodes -- [owner_node]) == 3,
|
|
"Expected 3 non-owner peer nodes, got #{inspect(peer_nodes -- [owner_node])}"
|
|
end
|
|
|
|
test "crashing GameRunner gets picked up on other node" do
|
|
peers = start_cluster(2)
|
|
|
|
game_runner_pid =
|
|
:peer.call(hd(peers) |> elem(0), :global, :whereis_name, [Backend.GameRunner])
|
|
|
|
assert is_pid(game_runner_pid), "Could not find GameRunner in :global registry"
|
|
|
|
first_node = node(game_runner_pid)
|
|
|
|
GameRunner.crash_game()
|
|
|
|
:timer.sleep(100)
|
|
|
|
new_pid = GameRunner.get_pid()
|
|
assert is_pid(new_pid), "GameRunner did not restart after crash "
|
|
|
|
restarted_node = node(new_pid)
|
|
|
|
assert restarted_node != first_node,
|
|
"GameRunner restarted on the same node after 10 crash attempts"
|
|
end
|
|
end
|
|
|
|
defp start_cluster(count) do
|
|
peers =
|
|
for index <- 1..count do
|
|
peer_name = :"cluster_peer_#{System.unique_integer([:positive])}_#{index}"
|
|
|
|
{:ok, peer, peer_node} =
|
|
:peer.start_link(%{
|
|
name: peer_name,
|
|
connection: :standard_io,
|
|
wait_boot: 30_000
|
|
})
|
|
|
|
_ = :peer.call(peer, :application, :ensure_all_started, [:elixir])
|
|
|
|
Enum.each(:code.get_path(), fn path ->
|
|
_ = :peer.call(peer, :code, :add_patha, [path])
|
|
end)
|
|
|
|
# needed for authentication
|
|
:peer.call(peer, :erlang, :set_cookie, [Node.get_cookie()])
|
|
|
|
:peer.call(peer, :application, :set_env, [:opentelemetry, :traces_exporter, :none])
|
|
:peer.call(peer, :application, :set_env, [:opentelemetry, :processors, []])
|
|
|
|
on_exit(fn ->
|
|
try do
|
|
:peer.stop(peer)
|
|
catch
|
|
:exit, _ -> :ok
|
|
end
|
|
end)
|
|
|
|
{peer, peer_node}
|
|
end
|
|
|
|
peer_nodes = Enum.map(peers, fn {_p, n} -> n end)
|
|
|
|
for {peer, _} <- peers do
|
|
Enum.each(peer_nodes, fn target ->
|
|
:peer.call(peer, :net_kernel, :connect_node, [target])
|
|
end)
|
|
end
|
|
|
|
for {peer, _} <- peers do
|
|
{:ok, _} = :peer.call(peer, :application, :ensure_all_started, [:backend])
|
|
end
|
|
|
|
peers
|
|
end
|
|
end
|