alex/elixirAI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

identified performance issues with streamed markdown

Browse Source
This commit is contained in:
Alex Mickelson
2026-03-06 13:39:32 -07:00
parent aee7aa7b16
commit c747f1d4ce
8 changed files with 257 additions and 96 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
lib/elixir_ai/ai_utils/chat_utils.ex
View File

@@ -68,7 +68,8 @@ defmodule ElixirAi.ChatUtils do
{:cont, acc}
end
) do
{:ok, _} ->
{:ok, _response} ->
# Logger.info("AI request completed with response #{inspect(response)}")
:ok
{:error, reason} ->

26
lib/elixir_ai/ai_utils/stream_line_utils.ex
View File

@@ -34,10 +34,15 @@ defmodule ElixirAi.AiUtils.StreamLineUtils do
end
# last streamed response
def handle_stream_line(server, %{
"choices" => [%{"finish_reason" => "stop"}],
"id" => id
}) do
def handle_stream_line(
server,
%{
"choices" => [%{"finish_reason" => "stop"}],
"id" => id
} = msg
) do
Logger.info("Received end of AI response stream for id #{id} with message: #{inspect(msg)}")
send(
server,
{:ai_text_stream_finish, id}
@@ -113,11 +118,14 @@ defmodule ElixirAi.AiUtils.StreamLineUtils do
end
# end tool call
def handle_stream_line(server, %{
"choices" => [%{"finish_reason" => "tool_calls"}],
"id" => id
}) do
# Logger.info("Received tool call end")
def handle_stream_line(
server,
%{
"choices" => [%{"finish_reason" => "tool_calls"}],
"id" => id
} = message
) do
Logger.info("Received tool_calls_finished with message: #{inspect(message)}")
send(server, {:ai_tool_call_end, id})
end

6
lib/elixir_ai/application.ex
View File

@@ -8,9 +8,11 @@ defmodule ElixirAi.Application do
ElixirAiWeb.Telemetry,
{DNSCluster, query: Application.get_env(:elixir_ai, :dns_cluster_query) || :ignore},
{Phoenix.PubSub, name: ElixirAi.PubSub},
ElixirAi.ChatRunner,
ElixirAi.ToolTesting,
ElixirAiWeb.Endpoint
ElixirAiWeb.Endpoint,
{Registry, keys: :unique, name: ElixirAi.ChatRegistry},
{DynamicSupervisor, name: ElixirAi.ChatRunnerSupervisor, strategy: :one_for_one},
ElixirAi.ConversationManager
]
opts = [strategy: :one_for_one, name: ElixirAi.Supervisor]

122
lib/elixir_ai/chat_runner.ex
View File

@@ -3,49 +3,51 @@ defmodule ElixirAi.ChatRunner do
use GenServer
import ElixirAi.ChatUtils
@topic "ai_chat"
defp via(name), do: {:via, Registry, {ElixirAi.ChatRegistry, name}}
defp topic(name), do: "ai_chat:#{name}"
def new_user_message(text_content) do
GenServer.cast(__MODULE__, {:user_message, text_content})
def new_user_message(name, text_content) do
GenServer.cast(via(name), {:user_message, text_content})
end
@spec get_conversation() :: any()
def get_conversation do
GenServer.call(__MODULE__, :get_conversation)
@spec get_conversation(String.t()) :: any()
def get_conversation(name) do
GenServer.call(via(name), :get_conversation)
end
def start_link(_opts) do
GenServer.start_link(
__MODULE__,
%{
messages: [],
streaming_response: nil,
pending_tool_calls: [],
tools: tools()
},
name: __MODULE__
)
def get_streaming_response(name) do
GenServer.call(via(name), :get_streaming_response)
end
def init(state) do
{:ok, state}
def start_link(name: name) do
GenServer.start_link(__MODULE__, name, name: via(name))
end
def tools do
def init(name) do
{:ok, %{
name: name,
messages: [],
streaming_response: nil,
pending_tool_calls: [],
tools: tools(self())
}}
end
def tools(server) do
[
ai_tool(
name: "store_thing",
description: "store a key value pair in memory",
function: &ElixirAi.ToolTesting.hold_thing/1,
parameters: ElixirAi.ToolTesting.hold_thing_params(),
server: __MODULE__
server: server
),
ai_tool(
name: "read_thing",
description: "read a key value pair that was previously stored with store_thing",
function: &ElixirAi.ToolTesting.get_thing/1,
parameters: ElixirAi.ToolTesting.get_thing_params(),
server: __MODULE__
server: server
),
ai_tool(
name: "set_background_color",
@@ -53,14 +55,14 @@ defmodule ElixirAi.ChatRunner do
"set the background color of the chat interface, accepts specified tailwind colors",
function: &ElixirAi.ToolTesting.set_background_color/1,
parameters: ElixirAi.ToolTesting.set_background_color_params(),
server: __MODULE__
server: server
)
]
end
def handle_cast({:user_message, text_content}, state) do
new_message = %{role: :user, content: text_content}
broadcast({:user_chat_message, new_message})
broadcast(state.name, {:user_chat_message, new_message})
new_state = %{state | messages: state.messages ++ [new_message]}
request_ai_response(self(), new_state.messages, state.tools)
@@ -69,7 +71,7 @@ defmodule ElixirAi.ChatRunner do
def handle_info({:start_new_ai_response, id}, state) do
starting_response = %{id: id, reasoning_content: "", content: "", tool_calls: []}
broadcast({:start_ai_response_stream, starting_response})
broadcast(state.name, {:start_ai_response_stream, starting_response})
{:noreply, %{state | streaming_response: starting_response}}
end
@@ -87,7 +89,7 @@ defmodule ElixirAi.ChatRunner do
end
def handle_info({:ai_reasoning_chunk, _id, reasoning_content}, state) do
broadcast({:reasoning_chunk_content, reasoning_content})
broadcast(state.name, {:reasoning_chunk_content, reasoning_content})
{:noreply,
%{
@@ -100,7 +102,7 @@ defmodule ElixirAi.ChatRunner do
end
def handle_info({:ai_text_chunk, _id, text_content}, state) do
broadcast({:text_chunk_content, text_content})
broadcast(state.name, {:text_chunk_content, text_content})
{:noreply,
%{
@@ -124,7 +126,7 @@ defmodule ElixirAi.ChatRunner do
tool_calls: state.streaming_response.tool_calls
}
broadcast({:end_ai_response, final_message})
broadcast(state.name, {:end_ai_response, final_message})
{:noreply,
%{
@@ -182,17 +184,6 @@ defmodule ElixirAi.ChatRunner do
def handle_info({:ai_tool_call_end, id}, state) do
Logger.info("ending tool call with tools: #{inspect(state.streaming_response.tool_calls)}")
parsed_tool_calls =
Enum.map(state.streaming_response.tool_calls, fn tool_call ->
case Jason.decode(tool_call.arguments) do
{:ok, decoded_args} ->
{:ok, tool_call, decoded_args}
{:error, e} ->
{:error, tool_call, "Failed to decode tool arguments: #{inspect(e)}"}
end
end)
tool_request_message = %{
role: :assistant,
content: state.streaming_response.content,
@@ -200,38 +191,27 @@ defmodule ElixirAi.ChatRunner do
tool_calls: state.streaming_response.tool_calls
}
broadcast({:tool_request_message, tool_request_message})
broadcast(state.name, {:tool_request_message, tool_request_message})
failed_call_messages =
parsed_tool_calls
|> Enum.filter(fn
{:error, _tool_call, _error_msg} -> true
_ -> false
end)
|> Enum.map(fn {:error, tool_call, error_msg} ->
Logger.error("Tool call #{tool_call.name} failed with error: #{error_msg}")
%{role: :tool, content: error_msg, tool_call_id: tool_call.id}
end)
pending_call_ids =
parsed_tool_calls
|> Enum.filter(fn
{:ok, _tool_call, _decoded_args} -> true
_ -> false
end)
|> Enum.map(fn {:ok, tool_call, decoded_args} ->
case Enum.find(state.tools, fn t -> t.name == tool_call.name end) do
{failed_call_messages, pending_call_ids} =
Enum.reduce(state.streaming_response.tool_calls, {[], []}, fn tool_call, {failed, pending} ->
with {:ok, decoded_args} <- Jason.decode(tool_call.arguments),
tool when not is_nil(tool) <- Enum.find(state.tools, fn t -> t.name == tool_call.name end) do
tool.run_function.(id, tool_call.id, decoded_args)
{failed, [tool_call.id | pending]}
else
{:error, e} ->
error_msg = "Failed to decode tool arguments: #{inspect(e)}"
Logger.error("Tool call #{tool_call.name} failed: #{error_msg}")
{[%{role: :tool, content: error_msg, tool_call_id: tool_call.id} | failed], pending}
nil ->
Logger.error("No tool definition found for #{tool_call.name}")
nil
tool ->
tool.run_function.(id, tool_call.id, decoded_args)
tool_call.id
error_msg = "No tool definition found for #{tool_call.name}"
Logger.error(error_msg)
{[%{role: :tool, content: error_msg, tool_call_id: tool_call.id} | failed], pending}
end
end)
|> Enum.filter(& &1)
{:noreply,
%{
@@ -244,7 +224,7 @@ defmodule ElixirAi.ChatRunner do
def handle_info({:tool_response, _id, tool_call_id, result}, state) do
new_message = %{role: :tool, content: inspect(result), tool_call_id: tool_call_id}
broadcast({:one_tool_finished, new_message})
broadcast(state.name, {:one_tool_finished, new_message})
new_pending_tool_calls =
Enum.filter(state.pending_tool_calls, fn id -> id != tool_call_id end)
@@ -259,7 +239,7 @@ defmodule ElixirAi.ChatRunner do
end
if new_pending_tool_calls == [] do
broadcast(:tool_calls_finished)
broadcast(state.name, :tool_calls_finished)
request_ai_response(self(), state.messages ++ [new_message], state.tools)
end
@@ -276,5 +256,9 @@ defmodule ElixirAi.ChatRunner do
{:reply, state, state}
end
defp broadcast(msg), do: Phoenix.PubSub.broadcast(ElixirAi.PubSub, @topic, msg)
def handle_call(:get_streaming_response, _from, state) do
{:reply, state.streaming_response, state}
end
defp broadcast(name, msg), do: Phoenix.PubSub.broadcast(ElixirAi.PubSub, topic(name), msg)
end

49
lib/elixir_ai/conversation_manager.ex Normal file
View File

@@ -0,0 +1,49 @@
defmodule ElixirAi.ConversationManager do
use GenServer
def start_link(_opts), do: GenServer.start_link(__MODULE__, [], name: __MODULE__)
def init(names), do: {:ok, names}
def create_conversation(name) do
GenServer.call(__MODULE__, {:create, name})
end
def open_conversation(name) do
GenServer.call(__MODULE__, {:open, name})
end
def list_conversations do
GenServer.call(__MODULE__, :list)
end
def handle_call({:create, name}, _from, names) do
if name in names do
{:reply, {:error, :already_exists}, names}
else
{:reply, start_runner(name), [name | names]}
end
end
def handle_call({:open, name}, _from, names) do
if name in names do
{:reply, start_runner(name), names}
else
{:reply, {:error, :not_found}, names}
end
end
def handle_call(:list, _from, names) do
{:reply, names, names}
end
defp start_runner(name) do
case DynamicSupervisor.start_child(
ElixirAi.ChatRunnerSupervisor,
{ElixirAi.ChatRunner, name: name}
) do
{:ok, pid} -> {:ok, pid}
{:error, {:already_started, pid}} -> {:ok, pid}
error -> error
end
end
end

64
lib/elixir_ai_web/live/chat_live.ex
View File

@@ -4,8 +4,7 @@ defmodule ElixirAiWeb.ChatLive do
import ElixirAiWeb.Spinner
import ElixirAiWeb.ChatMessage
alias ElixirAi.ChatRunner
@topic "ai_chat"
alias ElixirAi.ConversationManager
def valid_background_colors do
[
@@ -19,23 +18,35 @@ defmodule ElixirAiWeb.ChatLive do
]
end
def mount(_params, _session, socket) do
if connected?(socket), do: Phoenix.PubSub.subscribe(ElixirAi.PubSub, @topic)
conversation = ChatRunner.get_conversation()
def mount(%{"name" => name}, _session, socket) do
case ConversationManager.open_conversation(name) do
{:ok, _pid} ->
if connected?(socket),
do: Phoenix.PubSub.subscribe(ElixirAi.PubSub, "ai_chat:#{name}")
{:ok,
socket
|> assign(user_input: "")
|> assign(messages: conversation.messages)
|> assign(streaming_response: nil)
|> assign(background_color: "bg-cyan-950/30")}
conversation = ChatRunner.get_conversation(name)
{:ok,
socket
|> assign(conversation_name: name)
|> assign(user_input: "")
|> assign(messages: conversation.messages)
|> assign(streaming_response: conversation.streaming_response)
|> assign(background_color: "bg-cyan-950/30")}
{:error, :not_found} ->
{:ok, push_navigate(socket, to: "/")}
end
end
def render(assigns) do
~H"""
<div class="flex flex-col h-full rounded-lg">
<div class="px-4 py-3 font-semibold ">
Live Chat
<div class="px-4 py-3 font-semibold flex items-center gap-3">
<.link navigate={~p"/"} class="text-cyan-700 hover:text-cyan-400 transition-colors">
←
</.link>
{@conversation_name}
</div>
<div
id="chat-messages"
@@ -88,7 +99,7 @@ defmodule ElixirAiWeb.ChatLive do
end
def handle_event("submit", %{"user_input" => user_input}, socket) when user_input != "" do
ChatRunner.new_user_message(user_input)
ChatRunner.new_user_message(socket.assigns.conversation_name, user_input)
{:noreply, assign(socket, user_input: "")}
end
@@ -104,6 +115,15 @@ defmodule ElixirAiWeb.ChatLive do
{:noreply, assign(socket, streaming_response: starting_response)}
end
# chunk arrived before :start_ai_response_stream — fetch snapshot from runner and apply
def handle_info(
{:reasoning_chunk_content, reasoning_content},
%{assigns: %{streaming_response: nil}} = socket
) do
base = get_snapshot(socket) |> Map.update!(:reasoning_content, &(&1 <> reasoning_content))
{:noreply, assign(socket, streaming_response: base)}
end
def handle_info({:reasoning_chunk_content, reasoning_content}, socket) do
updated_response = %{
socket.assigns.streaming_response
@@ -114,6 +134,14 @@ defmodule ElixirAiWeb.ChatLive do
{:noreply, assign(socket, streaming_response: updated_response)}
end
def handle_info(
{:text_chunk_content, text_content},
%{assigns: %{streaming_response: nil}} = socket
) do
base = get_snapshot(socket) |> Map.update!(:content, &(&1 <> text_content))
{:noreply, assign(socket, streaming_response: base)}
end
def handle_info({:text_chunk_content, text_content}, socket) do
updated_response = %{
socket.assigns.streaming_response
@@ -158,4 +186,12 @@ defmodule ElixirAiWeb.ChatLive do
Logger.info("setting background color to #{color}")
{:noreply, assign(socket, background_color: color)}
end
defp get_snapshot(socket) do
ChatRunner.get_streaming_response(socket.assigns.conversation_name)
|> case do
nil -> %{id: nil, content: "", reasoning_content: "", tool_calls: []}
snapshot -> snapshot
end
end
end

80
lib/elixir_ai_web/live/home_live.ex Normal file
View File

@@ -0,0 +1,80 @@
defmodule ElixirAiWeb.HomeLive do
use ElixirAiWeb, :live_view
alias ElixirAi.ConversationManager
def mount(_params, _session, socket) do
{:ok,
socket
|> assign(conversations: ConversationManager.list_conversations())
|> assign(new_name: "")
|> assign(error: nil)}
end
def render(assigns) do
~H"""
<div class="max-w-lg mx-auto mt-16 px-4">
<h1 class="text-lg font-semibold text-cyan-300 mb-8">Conversations</h1>
<ul class="mb-8 space-y-2">
<%= if @conversations == [] do %>
<li class="text-sm text-cyan-700">No conversations yet.</li>
<% end %>
<%= for name <- @conversations do %>
<li>
<.link
navigate={~p"/chat/#{name}"}
class="block px-4 py-2 rounded-lg border border-cyan-900/40 bg-cyan-950/20 text-cyan-300 hover:border-cyan-700 hover:bg-cyan-950/40 transition-colors text-sm"
>
{name}
</.link>
</li>
<% end %>
</ul>
<form phx-submit="create" class="flex gap-2">
<input
type="text"
name="name"
value={@new_name}
placeholder="New conversation name"
class="flex-1 rounded px-3 py-2 text-sm bg-cyan-950/20 border border-cyan-900/40 text-cyan-100 placeholder-cyan-800 focus:outline-none focus:ring-1 focus:ring-cyan-700"
autocomplete="off"
/>
<button
type="submit"
class="px-4 py-2 rounded text-sm border border-cyan-900/40 bg-cyan-950/20 text-cyan-300 hover:border-cyan-700 hover:bg-cyan-950/40 transition-colors"
>
Create
</button>
</form>
<%= if @error do %>
<p class="mt-2 text-sm text-red-400">{@error}</p>
<% end %>
</div>
"""
end
@spec handle_event(<<_::48>>, map(), any()) :: {:noreply, any()}
def handle_event("create", %{"name" => name}, socket) do
name = String.trim(name)
if name == "" do
{:noreply, assign(socket, error: "Name can't be blank")}
else
case ConversationManager.create_conversation(name) do
{:ok, _pid} ->
{:noreply,
socket
|> push_navigate(to: ~p"/chat/#{name}")
|> assign(error: nil)}
{:error, :already_exists} ->
{:noreply, assign(socket, error: "A conversation with that name already exists")}
_ ->
{:noreply, assign(socket, error: "Failed to create conversation")}
end
end
end
end

3
lib/elixir_ai_web/router.ex
View File

@@ -17,7 +17,8 @@ defmodule ElixirAiWeb.Router do
scope "/", ElixirAiWeb do
pipe_through :browser
get "/", PageController, :home
live "/", HomeLive
live "/chat/:name", ChatLive
end
# Other scopes may use custom stacks.