diff --git a/DISTRIBUTED_SETUP.md b/DISTRIBUTED_SETUP.md new file mode 100644 index 0000000..19ae8ad --- /dev/null +++ b/DISTRIBUTED_SETUP.md @@ -0,0 +1,236 @@ +# Distributed Phoenix WebSocket Game + +This project demonstrates a distributed Phoenix application with automatic failover using native Erlang clustering. + +## Architecture + +- **3 Phoenix Nodes**: Running in Docker containers, forming a distributed Erlang cluster +- **Global Process Registry**: Uses `:global` to ensure only one `GameState` GenServer runs across the cluster +- **Automatic Failover**: If a node goes down, another node automatically takes over the GameState +- **Nginx Load Balancer**: Routes WebSocket connections to healthy nodes +- **Client Failover**: Frontend automatically switches to another server if connection is lost + +## How It Works + +### Distributed Erlang Clustering + +- Each Phoenix container starts with a unique node name (e.g., `backend@phoenix1`) +- All nodes share the same Erlang cookie for authentication +- `Backend.Cluster` module automatically connects nodes on startup +- Nodes use EPMD (Erlang Port Mapper Daemon) for discovery + +### Singleton Game State + +- `Backend.GameState` is registered globally using `{:global, __MODULE__}` +- Only ONE instance runs across all nodes at any time +- If the node running GameState crashes, Erlang automatically starts it on another node +- All nodes can communicate with the GameState regardless of where it's running + +### Client Failover + +- Frontend maintains a list of all backend servers +- Automatically reconnects to the next server if connection fails +- Uses exponential backoff and retry logic +- Displays current connection status + +## Setup + +### Prerequisites + +- Docker and Docker Compose +- Or: Elixir 1.15+, Erlang 26+, Node.js 18+ + +### Running with Docker + +1. Build and start all services: + +```bash +docker-compose up --build +``` + +This starts: +- `phoenix1` on port 4001 +- `phoenix2` on port 4002 +- `phoenix3` on port 4003 +- `nginx` load balancer on port 4000 + +2. Open the client (in a separate terminal): + +```bash +cd client +pnpm install +pnpm dev +``` + +3. Open http://localhost:5173 in your browser + +### Running Locally (Development) + +Terminal 1 - Backend Node 1: +```bash +cd backend +mix deps.get +export RELEASE_NODE=backend@127.0.0.1 +export RELEASE_COOKIE=mycookie +export PORT=4001 +export CLUSTER_NODES="backend@127.0.0.1" +iex --name backend@127.0.0.1 --cookie mycookie -S mix phx.server +``` + +Terminal 2 - Backend Node 2: +```bash +cd backend +export RELEASE_NODE=backend@127.0.0.2 +export RELEASE_COOKIE=mycookie +export PORT=4002 +export CLUSTER_NODES="backend@127.0.0.1,backend@127.0.0.2" +iex --name backend@127.0.0.2 --cookie mycookie -S mix phx.server +``` + +Terminal 3 - Frontend: +```bash +cd client +pnpm install +pnpm dev +``` + +## Testing Failover + +### Test 1: Stop a node + +```bash +# Stop one container +docker-compose stop phoenix1 + +# The game continues running on phoenix2 or phoenix3 +# Clients automatically reconnect to available nodes +``` + +### Test 2: Kill the node running GameState + +1. Find which node is running GameState: +```bash +docker-compose exec phoenix1 /app/bin/backend remote +# In the IEx shell: +:global.whereis_name(Backend.GameState) +# This shows {pid, node_name} +``` + +2. Stop that specific node: +```bash +docker-compose stop phoenix2 # or whichever node is running it +``` + +3. The GameState automatically starts on another node +4. All players remain in the game + +### Test 3: Network partition + +```bash +# Disconnect a node from the network +docker network disconnect websocket-testing_app_net phoenix3 + +# Reconnect it +docker network connect websocket-testing_app_net phoenix3 +``` + +## Monitoring the Cluster + +### Check connected nodes + +```bash +docker-compose exec phoenix1 /app/bin/backend remote +``` + +In the IEx shell: +```elixir +# List all connected nodes +Node.list() + +# Check which node is running GameState +:global.whereis_name(Backend.GameState) + +# Get current game state +Backend.GameState.get_state() + +# Check registered global processes +:global.registered_names() +``` + +### View logs + +```bash +# All containers +docker-compose logs -f + +# Specific container +docker-compose logs -f phoenix1 +``` + +## Configuration + +### Environment Variables + +- `RELEASE_NODE`: Node name (e.g., `backend@phoenix1`) +- `RELEASE_COOKIE`: Erlang cookie for cluster authentication +- `CLUSTER_NODES`: Comma-separated list of nodes to connect to +- `PORT`: HTTP port for Phoenix endpoint +- `SECRET_KEY_BASE`: Phoenix secret key + +### Scaling + +To add more nodes, edit `docker-compose.yml`: + +```yaml +phoenix4: + # Same config as phoenix1-3, with unique: + # - container_name: phoenix4 + # - hostname: phoenix4 + # - RELEASE_NODE: backend@phoenix4 + # - ports: "4004:4000" + # - ipv4_address: 172.25.0.14 +``` + +Update `CLUSTER_NODES` in all services to include `backend@phoenix4`. + +## How to Play + +- Use **WASD** keys to move your player +- Your player is shown in red, others in blue +- The game state is shared across all nodes +- Try killing nodes to see failover in action! + +## Troubleshooting + +### Nodes not connecting + +1. Check all nodes have the same `RELEASE_COOKIE` +2. Verify EPMD is running: `docker-compose exec phoenix1 epmd -names` +3. Check firewall allows ports 4369 (EPMD) and 9000-9100 (distributed Erlang) + +### GameState not starting + +1. Check logs: `docker-compose logs -f` +2. Verify only one instance exists globally: `:global.registered_names()` +3. Restart all nodes: `docker-compose restart` + +### Frontend not connecting + +1. Check nginx is running: `docker-compose ps nginx` +2. Verify at least one Phoenix node is healthy +3. Check browser console for connection errors +4. Try connecting directly to a node: http://localhost:4001 + +## Production Considerations + +- **Change the Erlang cookie**: Use a strong secret +- **Use proper SSL/TLS**: Configure HTTPS for WebSocket connections +- **Add health checks**: Monitor node health and GameState availability +- **Persistent storage**: Add database for game state persistence +- **Rate limiting**: Protect against abuse +- **Monitoring**: Add Prometheus/Grafana for metrics +- **Logging**: Centralize logs with ELK or similar + +## License + +MIT diff --git a/client/.dockerignore b/client/.dockerignore new file mode 100644 index 0000000..2ec4eb6 --- /dev/null +++ b/client/.dockerignore @@ -0,0 +1,27 @@ +# Dependencies +node_modules + +# Build outputs +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? + +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* diff --git a/client/Dockerfile b/client/Dockerfile new file mode 100644 index 0000000..1c74f25 --- /dev/null +++ b/client/Dockerfile @@ -0,0 +1,21 @@ +# Build stage +FROM node:20-alpine AS build + +WORKDIR /app + +COPY package.json pnpm-lock.yaml ./ + +RUN npm install -g pnpm && pnpm install --frozen-lockfile + +COPY . . + +RUN pnpm build + +FROM nginx:alpine + +COPY --from=build /app/dist /usr/share/nginx/html + +COPY nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 +CMD ["nginx", "-g", "daemon off;"] diff --git a/client/nginx.conf b/client/nginx.conf new file mode 100644 index 0000000..30c1743 --- /dev/null +++ b/client/nginx.conf @@ -0,0 +1,22 @@ +server { + listen 80; + server_name localhost; + root /usr/share/nginx/html; + index index.html; + + # Enable gzip compression + gzip on; + gzip_vary on; + gzip_min_length 1024; + gzip_types text/plain text/css text/xml text/javascript application/javascript application/xml+rss application/json; + + location / { + try_files $uri $uri/ /index.html; + } + + # Cache static assets + location ~* \.(js|css|png|jpg|jpeg|gif|ico|svg|woff|woff2|ttf|eot)$ { + expires 1y; + add_header Cache-Control "public, immutable"; + } +} diff --git a/client/package.json b/client/package.json index d716634..78cac76 100644 --- a/client/package.json +++ b/client/package.json @@ -10,12 +10,14 @@ "preview": "vite preview" }, "dependencies": { + "phoenix": "^1.8.4", "react": "^19.2.0", "react-dom": "^19.2.0" }, "devDependencies": { "@eslint/js": "^9.39.1", "@types/node": "^24.10.1", + "@types/phoenix": "^1.6.7", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.1", diff --git a/client/pnpm-lock.yaml b/client/pnpm-lock.yaml index f1fc1fe..1f77db0 100644 --- a/client/pnpm-lock.yaml +++ b/client/pnpm-lock.yaml @@ -8,6 +8,9 @@ importers: .: dependencies: + phoenix: + specifier: ^1.8.4 + version: 1.8.4 react: specifier: ^19.2.0 version: 19.2.4 @@ -21,6 +24,9 @@ importers: '@types/node': specifier: ^24.10.1 version: 24.10.13 + '@types/phoenix': + specifier: ^1.6.7 + version: 1.6.7 '@types/react': specifier: ^19.2.7 version: 19.2.14 @@ -515,6 +521,9 @@ packages: '@types/node@24.10.13': resolution: {integrity: sha512-oH72nZRfDv9lADUBSo104Aq7gPHpQZc4BTx38r9xf9pg5LfP6EzSyH2n7qFmmxRQXh7YlUXODcYsg6PuTDSxGg==} + '@types/phoenix@1.6.7': + resolution: {integrity: sha512-oN9ive//QSBkf19rfDv45M7eZPi0eEXylht2OLEXicu5b4KoQ1OzXIw+xDSGWxSxe1JmepRR/ZH283vsu518/Q==} + '@types/react-dom@19.2.3': resolution: {integrity: sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==} peerDependencies: @@ -930,6 +939,9 @@ packages: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} + phoenix@1.8.4: + resolution: {integrity: sha512-5wdO9mqU4l0AmcexN8mA0m1BsC4ROv2l55MN31gbxmJPzghc4SxVxnNTh9qaummYa2pbwkoBvG8FezO7cjdr8A==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -1480,6 +1492,8 @@ snapshots: dependencies: undici-types: 7.16.0 + '@types/phoenix@1.6.7': {} + '@types/react-dom@19.2.3(@types/react@19.2.14)': dependencies: '@types/react': 19.2.14 @@ -1930,6 +1944,8 @@ snapshots: path-key@3.1.1: {} + phoenix@1.8.4: {} + picocolors@1.1.1: {} picomatch@4.0.3: {} diff --git a/client/src/App.css b/client/src/App.css index b9d355d..6cd2806 100644 --- a/client/src/App.css +++ b/client/src/App.css @@ -1,8 +1,6 @@ #root { - max-width: 1280px; - margin: 0 auto; - padding: 2rem; - text-align: center; + margin: 0; + padding: 0; } .logo { diff --git a/client/src/App.tsx b/client/src/App.tsx index 3d7ded3..3fb64c7 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -1,35 +1,232 @@ -import { useState } from 'react' -import reactLogo from './assets/react.svg' -import viteLogo from '/vite.svg' -import './App.css' +import { useEffect, useRef, useState } from "react"; +import { Socket, Channel } from "phoenix"; +import "./App.css"; -function App() { - const [count, setCount] = useState(0) - - return ( - <> -
- - Vite logo - - - React logo - -
-

Vite + React

-
- -

- Edit src/App.tsx and save to test HMR -

-
-

- Click on the Vite and React logos to learn more -

- - ) +interface Player { + x: number; + y: number; } -export default App +interface GameState { + [playerId: string]: Player; +} + +// List of WebSocket servers - we'll connect to all of them +const WS_SERVERS = [ + "ws://localhost:4001/socket", + "ws://localhost:4002/socket", + "ws://localhost:4003/socket", +]; + +function App() { + const [players, setPlayers] = useState({}); + const [myPlayerId, setMyPlayerId] = useState(null); + const [connectionStatus, setConnectionStatus] = + useState("connecting"); + const socketsRef = useRef([]); + const channelsRef = useRef([]); + const keysPressed = useRef>(new Set()); + + useEffect(() => { + // Connect to all servers concurrently + const sockets = WS_SERVERS.map((serverUrl) => { + console.log(`Connecting to ${serverUrl}`); + + const socket = new Socket(serverUrl, { + timeout: 3000, + reconnectAfterMs: () => 2000, // Keep trying to reconnect + }); + + // Handle connection events + socket.onOpen(() => { + console.log(`✓ Connected to ${serverUrl}`); + updateConnectionStatus(); + }); + + socket.onError((error) => { + console.error(`✗ Error on ${serverUrl}:`, error); + updateConnectionStatus(); + }); + + socket.onClose(() => { + console.log(`✗ Disconnected from ${serverUrl}`); + updateConnectionStatus(); + }); + + socket.connect(); + return socket; + }); + + socketsRef.current = sockets; + + // Join game channel on all connected sockets + const channels = sockets.map((socket, index) => { + const channel = socket.channel("game:lobby", {}); + + channel + .join() + .receive("ok", () => { + console.log(`✓ Joined channel on ${WS_SERVERS[index]}`); + updateConnectionStatus(); + }) + .receive("error", (resp) => { + console.log(`✗ Failed to join on ${WS_SERVERS[index]}:`, resp); + }) + .receive("timeout", () => { + console.log(`✗ Timeout joining on ${WS_SERVERS[index]}`); + }); + + // Listen for game state updates from any server + channel.on("game_state", (payload: { players: GameState }) => { + setPlayers(payload.players); + + // Set our player ID from the first state update if not set + if (!myPlayerId && Object.keys(payload.players).length > 0) { + const playerIds = Object.keys(payload.players); + if (playerIds.length > 0) { + setMyPlayerId(playerIds[playerIds.length - 1]); + } + } + }); + + return channel; + }); + + channelsRef.current = channels; + + const updateConnectionStatus = () => { + const joined = channels.filter((c) => c.state === "joined").length; + setConnectionStatus(`${joined}/${WS_SERVERS.length} servers active`); + }; + + // Periodic status update + const statusInterval = setInterval(updateConnectionStatus, 1000); + + // Cleanup on unmount + return () => { + clearInterval(statusInterval); + channels.forEach((channel) => channel.leave()); + sockets.forEach((socket) => socket.disconnect()); + }; + }, [myPlayerId]); + + // Handle keyboard input - send to first available channel + useEffect(() => { + const handleKeyDown = (e: KeyboardEvent) => { + const key = e.key.toLowerCase(); + if (["w", "a", "s", "d"].includes(key)) { + e.preventDefault(); + + // Only send if not already pressed (prevent repeat) + if (!keysPressed.current.has(key)) { + keysPressed.current.add(key); + + // Send to first joined channel (they all share same game state) + const activeChannel = channelsRef.current.find( + (c) => c.state === "joined", + ); + if (activeChannel) { + activeChannel.push("move", { direction: key }); + } + } + } + }; + + const handleKeyUp = (e: KeyboardEvent) => { + const key = e.key.toLowerCase(); + if (["w", "a", "s", "d"].includes(key)) { + keysPressed.current.delete(key); + } + }; + + window.addEventListener("keydown", handleKeyDown); + window.addEventListener("keyup", handleKeyUp); + + return () => { + window.removeEventListener("keydown", handleKeyDown); + window.removeEventListener("keyup", handleKeyUp); + }; + }, []); + + return ( +
+ {/* Connection status */} +
+
Status: {connectionStatus}
+
Players: {Object.keys(players).length}
+
+ + {/* Game canvas */} +
+ {Object.entries(players).map(([id, player]) => ( +
+ {id === myPlayerId && ( +
+ You +
+ )} +
+ ))} +
+
+ ); +} + +export default App; diff --git a/client/src/index.css b/client/src/index.css index 08a3ac9..fb92a9f 100644 --- a/client/src/index.css +++ b/client/src/index.css @@ -24,8 +24,6 @@ a:hover { body { margin: 0; - display: flex; - place-items: center; min-width: 320px; min-height: 100vh; } diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a773e52 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,82 @@ +version: "3.8" + +services: + phoenix1: + build: + context: ./backend + dockerfile: Dockerfile + container_name: phoenix1 + hostname: phoenix1 + environment: + - RELEASE_NODE=backend@phoenix1 + - RELEASE_DISTRIBUTION=sname + - RELEASE_COOKIE=super_secret_cookie_change_in_production + - PHX_HOST=localhost + - PHX_SERVER=true + - PORT=4000 + - DATABASE_URL=ecto://postgres:postgres@db/backend_dev + - SECRET_KEY_BASE=W8nGKNhNR8vKj6A4VnwN5h5h7RZvkKmZPqxqzLzYxXGQqC6HnKp2Wm8MNqKpQxZv + - CLUSTER_NODES=backend@phoenix1,backend@phoenix2,backend@phoenix3 + ports: + - "4001:4000" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:4000/api/health"] + interval: 10s + timeout: 5s + retries: 5 + + phoenix2: + build: + context: ./backend + dockerfile: Dockerfile + container_name: phoenix2 + hostname: phoenix2 + environment: + - RELEASE_NODE=backend@phoenix2 + - RELEASE_DISTRIBUTION=sname + - RELEASE_COOKIE=super_secret_cookie_change_in_production + - PHX_HOST=localhost + - PHX_SERVER=true + - PORT=4000 + - DATABASE_URL=ecto://postgres:postgres@db/backend_dev + - SECRET_KEY_BASE=W8nGKNhNR8vKj6A4VnwN5h5h7RZvkKmZPqxqzLzYxXGQqC6HnKp2Wm8MNqKpQxZv + - CLUSTER_NODES=backend@phoenix1,backend@phoenix2,backend@phoenix3 + ports: + - "4002:4000" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:4000/api/health"] + interval: 10s + timeout: 5s + retries: 5 + + phoenix3: + build: + context: ./backend + dockerfile: Dockerfile + container_name: phoenix3 + hostname: phoenix3 + environment: + - RELEASE_NODE=backend@phoenix3 + - RELEASE_DISTRIBUTION=sname + - RELEASE_COOKIE=super_secret_cookie_change_in_production + - PHX_HOST=localhost + - PHX_SERVER=true + - PORT=4000 + - DATABASE_URL=ecto://postgres:postgres@db/backend_dev + - SECRET_KEY_BASE=W8nGKNhNR8vKj6A4VnwN5h5h7RZvkKmZPqxqzLzYxXGQqC6HnKp2Wm8MNqKpQxZv + - CLUSTER_NODES=backend@phoenix1,backend@phoenix2,backend@phoenix3 + ports: + - "4003:4000" + healthcheck: + test: ["CMD", "wget", "-qO-", "http://localhost:4000/api/health"] + interval: 10s + timeout: 5s + retries: 5 + + client: + build: + context: ./client + dockerfile: Dockerfile + container_name: client + ports: + - "5173:80" diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..84fb65c --- /dev/null +++ b/nginx.conf @@ -0,0 +1,48 @@ +events { + worker_connections 1024; +} + +http { + upstream phoenix_backend { + least_conn; + server phoenix1:4000 max_fails=3 fail_timeout=30s; + server phoenix2:4000 max_fails=3 fail_timeout=30s; + server phoenix3:4000 max_fails=3 fail_timeout=30s; + } + + # Map to track which backend server a WebSocket connection should use + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 4000; + server_name localhost; + + # WebSocket configuration + location /socket { + proxy_pass http://phoenix_backend; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # WebSocket timeout settings + proxy_read_timeout 86400; + proxy_send_timeout 86400; + } + + # HTTP fallback for health checks + location / { + proxy_pass http://phoenix_backend; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + } +}