general solution to voice control

lib/elixir_ai/ai_controllable.ex

@@ -0,0 +1,68 @@
+defmodule ElixirAi.AiControllable do
+  @moduledoc """
+  Behaviour + macro for LiveViews that expose AI-controllable tools.
+
+  Any LiveView that `use`s this module must implement:
+
+    - `ai_tools/0`  — returns a list of tool spec maps
+    - `handle_ai_tool_call(tool_name, args, socket)` — handles a dispatched tool call,
+      returns `{result_string, socket}`.
+
+  The macro injects:
+
+    - A `handle_info` clause that dispatches `{:page_tool_call, tool_name, args, reply_to}`
+      messages to the callback and sends the result back to the caller.
+    - An `on_mount` hook registration that joins the `:pg` group keyed by
+      `voice_session_id` so VoiceLive can discover sibling page LiveViews.
+
+  ## Usage
+
+      defmodule MyAppWeb.SomeLive do
+        use MyAppWeb, :live_view
+        use ElixirAi.AiControllable
+
+        @impl ElixirAi.AiControllable
+        def ai_tools do
+          [
+            %{
+              name: "do_something",
+              description: "Does something useful",
+              parameters: %{
+                "type" => "object",
+                "properties" => %{"value" => %{"type" => "string"}},
+                "required" => ["value"]
+              }
+            }
+          ]
+        end
+
+        @impl ElixirAi.AiControllable
+        def handle_ai_tool_call("do_something", %{"value" => val}, socket) do
+          {"done: \#{val}", assign(socket, value: val)}
+        end
+      end
+  """
+
+  @callback ai_tools() :: [map()]
+  @callback handle_ai_tool_call(tool_name :: String.t(), args :: map(), socket :: term()) ::
+              {String.t(), term()}
+
+  defmacro __using__(_opts) do
+    quote do
+      @behaviour ElixirAi.AiControllable
+
+      on_mount ElixirAi.AiControllable.Hook
+
+      def handle_info({:page_tool_call, tool_name, args, reply_to}, socket) do
+        {result, socket} = handle_ai_tool_call(tool_name, args, socket)
+        send(reply_to, {:page_tool_result, tool_name, result})
+        {:noreply, socket}
+      end
+
+      def handle_info({:get_ai_tools, reply_to}, socket) do
+        send(reply_to, {:ai_tools_response, self(), ai_tools()})
+        {:noreply, socket}
+      end
+    end
+  end
+end

lib/elixir_ai/ai_controllable/hook.ex

@@ -0,0 +1,35 @@
+defmodule ElixirAi.AiControllable.Hook do
+  @moduledoc """
+  LiveView on_mount hook that registers a page LiveView in the
+  `:ai_page_tools` pg group so VoiceLive can discover it.
+
+  The group key is `{:page, voice_session_id}` where `voice_session_id`
+  comes from the Plug session, tying the page LiveView to the same browser
+  tab as VoiceLive.
+
+  Only joins when the LiveView module implements `ai_tools/0`
+  (i.e. uses `ElixirAi.AiControllable`).
+  """
+
+  import Phoenix.LiveView
+  import Phoenix.Component, only: [assign: 3]
+
+  def on_mount(:default, _params, session, socket) do
+    voice_session_id = session["voice_session_id"]
+    module = socket.view
+
+    if voice_session_id && function_exported?(module, :ai_tools, 0) do
+      if connected?(socket) do
+        try do
+          :pg.join(ElixirAi.PageToolsPG, {:page, voice_session_id}, self())
+        catch
+          :exit, _ -> :ok
+        end
+      end
+
+      {:cont, assign(socket, :voice_session_id, voice_session_id)}
+    else
+      {:cont, socket}
+    end
+  end
+end

lib/elixir_ai/ai_tools.ex

@@ -126,6 +126,42 @@ defmodule ElixirAi.AiTools do
     )
   end
 
+  # ---------------------------------------------------------------------------
+  # Page tools (dynamic, from AiControllable LiveViews)
+  # ---------------------------------------------------------------------------
+
+  @doc """
+  Builds tool structs for page tools discovered from AiControllable LiveViews.
+
+  Each entry in `pids_and_specs` is `{page_pid, [tool_spec, ...]}` where
+  `tool_spec` is a map with `:name`, `:description`, and `:parameters`.
+
+  The generated function sends `{:page_tool_call, name, args, self()}` to
+  the page LiveView pid and blocks (inside a Task) waiting for the reply.
+  """
+  def build_page_tools(server, pids_and_specs) do
+    Enum.flat_map(pids_and_specs, fn {page_pid, tool_specs} ->
+      Enum.map(tool_specs, fn spec ->
+        ai_tool(
+          name: spec.name,
+          description: spec.description,
+          function: fn args ->
+            send(page_pid, {:page_tool_call, spec.name, args, self()})
+
+            receive do
+              {:page_tool_result, tool_name, result} when tool_name == spec.name ->
+                {:ok, result}
+            after
+              5_000 -> {:ok, "page tool #{spec.name} timed out"}
+            end
+          end,
+          parameters: spec.parameters,
+          server: server
+        )
+      end)
+    end)
+  end
+
   # ---------------------------------------------------------------------------
   # Private
   # ---------------------------------------------------------------------------

lib/elixir_ai/application.ex

@@ -2,7 +2,6 @@ defmodule ElixirAi.Application do
   @moduledoc false
   use Application
 
-  @impl true
   def start(_type, _args) do
     children = [
       ElixirAiWeb.Telemetry,
@@ -13,6 +12,7 @@ defmodule ElixirAi.Application do
        [Application.get_env(:libcluster, :topologies, []), [name: ElixirAi.ClusterSupervisor]]},
       {Phoenix.PubSub, name: ElixirAi.PubSub},
       {ElixirAi.LiveViewPG, []},
+      {ElixirAi.PageToolsPG, []},
       {ElixirAi.AudioProcessingPG, []},
       {DynamicSupervisor, name: ElixirAi.AudioWorkerSupervisor, strategy: :one_for_one},
       ElixirAi.ToolTesting,
@@ -39,7 +39,6 @@ defmodule ElixirAi.Application do
     Supervisor.start_link(children, opts)
   end
 
-  @impl true
   def config_change(changed, _new, removed) do
     ElixirAiWeb.Endpoint.config_change(changed, removed)
     :ok

lib/elixir_ai/chat_runner/chat_runner.ex

@@ -50,6 +50,10 @@ defmodule ElixirAi.ChatRunner do
     GenServer.call(via(name), {:session, {:deregister_liveview_pid, liveview_pid}})
   end
 
+  def register_page_tools(name, page_tools) when is_list(page_tools) do
+    GenServer.call(via(name), {:session, {:register_page_tools, page_tools}})
+  end
+
   def get_conversation(name) do
     GenServer.call(via(name), {:conversation, :get_conversation})
   end
@@ -130,6 +134,7 @@ defmodule ElixirAi.ChatRunner do
        tool_choice: tool_choice,
        server_tools: server_tools,
        liveview_tools: liveview_tools,
+       page_tools: [],
        provider: provider,
        liveview_pids: %{}
      }}

lib/elixir_ai/chat_runner/conversation_calls.ex

@@ -11,7 +11,7 @@ defmodule ElixirAi.ChatRunner.ConversationCalls do
     ElixirAi.ChatUtils.request_ai_response(
       self(),
       messages_with_system_prompt(new_state.messages, state.system_prompt),
-      state.server_tools ++ state.liveview_tools,
+      state.server_tools ++ state.liveview_tools ++ state.page_tools,
       state.provider,
       effective_tool_choice
     )

lib/elixir_ai/chat_runner/liveview_session.ex

@@ -10,6 +10,10 @@ defmodule ElixirAi.ChatRunner.LiveviewSession do
     {:reply, :ok, %{state | liveview_pids: Map.put(state.liveview_pids, liveview_pid, ref)}}
   end
 
+  def handle_call({:register_page_tools, page_tools}, _from, state) do
+    {:reply, :ok, %{state | page_tools: page_tools}}
+  end
+
   def handle_call({:deregister_liveview_pid, liveview_pid}, _from, state) do
     case Map.pop(state.liveview_pids, liveview_pid) do
       {nil, _} ->

lib/elixir_ai/chat_runner/stream_handler.ex

@@ -111,7 +111,7 @@ defmodule ElixirAi.ChatRunner.StreamHandler do
                                                                     {failed, pending} ->
         with {:ok, decoded_args} <- Jason.decode(tool_call.arguments),
              tool when not is_nil(tool) <-
-               Enum.find(state.server_tools ++ state.liveview_tools, fn t ->
+               Enum.find(state.server_tools ++ state.liveview_tools ++ state.page_tools, fn t ->
                  t.name == tool_call.name
                end) do
           tool.run_function.(id, tool_call.id, decoded_args)
@@ -160,7 +160,7 @@ defmodule ElixirAi.ChatRunner.StreamHandler do
       ElixirAi.ChatUtils.request_ai_response(
         self(),
         messages_with_system_prompt(state.messages ++ [new_message], state.system_prompt),
-        state.server_tools ++ state.liveview_tools,
+        state.server_tools ++ state.liveview_tools ++ state.page_tools,
         state.provider,
         state.tool_choice
       )

lib/elixir_ai/page_tools_pg.ex

@@ -0,0 +1,15 @@
+defmodule ElixirAi.PageToolsPG do
+  @moduledoc """
+  Named :pg scope for tracking LiveViews that implement AiControllable.
+  Group key is `{:page, voice_session_id}` — one group per browser session.
+  """
+
+  def child_spec(_opts) do
+    %{
+      id: __MODULE__,
+      start: {:pg, :start_link, [__MODULE__]},
+      type: :worker,
+      restart: :permanent
+    }
+  end
+end