From 37602fc78a0405b8e72c32419120169b580e9a0b Mon Sep 17 00:00:00 2001
From: Alex Mickelson <alexmickelson96@gmail.com>
Date: Mon, 16 Mar 2026 20:44:47 -0600
Subject: [PATCH] scraper

---
 lib/cobblemon_ui/tier_list_scraper.ex         | 43 +++++++++++++++----
 lib/cobblemon_ui_web/live/dashboard_live.ex   | 25 +++--------
 .../live/pokemon_components.ex                | 18 ++++++++
 3 files changed, 58 insertions(+), 28 deletions(-)

diff --git a/lib/cobblemon_ui/tier_list_scraper.ex b/lib/cobblemon_ui/tier_list_scraper.ex
index 305b0c7..4ea718d 100644
--- a/lib/cobblemon_ui/tier_list_scraper.ex
+++ b/lib/cobblemon_ui/tier_list_scraper.ex
@@ -1,4 +1,6 @@
 defmodule CobblemonUi.TierListScraper do
+  require Logger
+
   @url "https://rankedboost.com/pokemon/tier-list/"
   @filename "pokemon_tier_list.json"
 
@@ -8,26 +10,49 @@ defmodule CobblemonUi.TierListScraper do
   end
 
   def run do
+    Logger.info("[TierListScraper] Starting scrape from #{@url}")
+
     with {:ok, html} <- fetch_page(),
-         pokemon <- parse(html),
+         {:ok, pokemon} <- parse(html),
          :ok <- write_json(pokemon) do
+      Logger.info("[TierListScraper] Successfully scraped and saved #{length(pokemon)} pokemon")
       {:ok, pokemon}
+    else
+      {:error, reason} = err ->
+        Logger.error("[TierListScraper] Scrape failed: #{inspect(reason)}")
+        err
     end
   end
 
   def fetch_page do
-    case Req.get(@url) do
-      {:ok, %Req.Response{status: 200, body: body}} -> {:ok, body}
-      {:ok, %Req.Response{status: status}} -> {:error, {:http_error, status}}
-      {:error, err} -> {:error, err}
+    Logger.debug("[TierListScraper] Fetching #{@url}")
+    case Req.get(@url, headers: [{"user-agent", "Mozilla/5.0 (compatible; CobblemonUI/1.0)"}]) do
+      {:ok, %Req.Response{status: 200, body: body}} ->
+        Logger.debug("[TierListScraper] Fetch OK, body size: #{byte_size(body)} bytes")
+        {:ok, body}
+      {:ok, %Req.Response{status: status}} ->
+        Logger.warning("[TierListScraper] Unexpected HTTP status: #{status}")
+        {:error, {:http_error, status}}
+      {:error, err} ->
+        Logger.error("[TierListScraper] HTTP request failed: #{inspect(err)}")
+        {:error, err}
     end
   end
 
   def parse(html) do
-    html
-    |> Floki.parse_document!()
-    |> Floki.find(".pokemon-tier")
-    |> Enum.map(&extract_pokemon/1)
+    nodes = html |> Floki.parse_document!() |> Floki.find(".pokemon-tier")
+    Logger.debug("[TierListScraper] Found #{length(nodes)} .pokemon-tier nodes")
+
+    case nodes do
+      [] ->
+        Logger.warning("[TierListScraper] No .pokemon-tier elements found — page structure may have changed or content is JS-rendered")
+        {:error, :no_pokemon_found}
+      _ ->
+        pokemon = Enum.map(nodes, &extract_pokemon/1)
+        valid = Enum.filter(pokemon, fn %{name: n} -> n != "" end)
+        Logger.info("[TierListScraper] Parsed #{length(valid)} valid pokemon (#{length(pokemon) - length(valid)} skipped with empty names)")
+        {:ok, valid}
+    end
   end
 
   defp extract_pokemon(node) do
diff --git a/lib/cobblemon_ui_web/live/dashboard_live.ex b/lib/cobblemon_ui_web/live/dashboard_live.ex
index c5fe209..4d466ff 100644
--- a/lib/cobblemon_ui_web/live/dashboard_live.ex
+++ b/lib/cobblemon_ui_web/live/dashboard_live.ex
@@ -1,6 +1,8 @@
 defmodule CobblemonUiWeb.DashboardLive do
   use CobblemonUiWeb, :live_view
 
+  require Logger
+
   import CobblemonUiWeb.PokemonComponents
   import CobblemonUiWeb.BattleComponents
 
@@ -94,9 +96,6 @@ defmodule CobblemonUiWeb.DashboardLive do
     {:noreply, assign(socket, view_mode: String.to_existing_atom(mode), selected_pokemon: nil)}
   end
 
-  def handle_event("refresh", _params, socket) do
-    {:noreply, do_refresh(socket)}
-  end
 
   @impl true
   def handle_info(:tick, socket) do
@@ -106,8 +105,10 @@ defmodule CobblemonUiWeb.DashboardLive do
   def handle_info(:scrape_tier_list, socket) do
     lv = self()
     Task.start(fn ->
-      CobblemonUi.TierListScraper.run()
-      send(lv, :reload_tier_list)
+      case CobblemonUi.TierListScraper.run() do
+        {:ok, _} -> send(lv, :reload_tier_list)
+        {:error, reason} -> Logger.error("[DashboardLive] Tier list scrape failed: #{inspect(reason)}")
+      end
     end)
     {:noreply, socket}
   end
@@ -156,13 +157,6 @@ defmodule CobblemonUiWeb.DashboardLive do
                   <p class="text-sm text-base-content/50">Who are you?</p>
                 </div>
               </div>
-              <button
-                id="refresh-btn"
-                phx-click="refresh"
-                class="btn btn-ghost btn-sm gap-2 hover:bg-base-300/50 transition-colors"
-              >
-                <.icon name="hero-arrow-path" class="size-4" /> Refresh
-              </button>
             </div>
 
             <div
@@ -211,13 +205,6 @@ defmodule CobblemonUiWeb.DashboardLive do
                   <p class="text-sm text-base-content/50">Player Data Explorer</p>
                 </div>
               </div>
-              <button
-                id="refresh-btn"
-                phx-click="refresh"
-                class="btn btn-ghost btn-sm gap-2 hover:bg-base-300/50 transition-colors"
-              >
-                <.icon name="hero-arrow-path" class="size-4" /> Refresh
-              </button>
             </div>
 
             <%!-- Scrollable content --%>
diff --git a/lib/cobblemon_ui_web/live/pokemon_components.ex b/lib/cobblemon_ui_web/live/pokemon_components.ex
index fee7c06..571d701 100644
--- a/lib/cobblemon_ui_web/live/pokemon_components.ex
+++ b/lib/cobblemon_ui_web/live/pokemon_components.ex
@@ -46,6 +46,17 @@ defmodule CobblemonUiWeb.PokemonComponents do
         </div>
         <div class="flex items-center gap-1 shrink-0">
           <.tier_badge :if={@tier} tier={@tier} species={@pokemon.species} compact={@compact} />
+          <div
+            :if={!@tier}
+            title="Tier data not downloaded"
+            class={[
+              "inline-flex items-center justify-center font-black rounded leading-none shrink-0",
+              "bg-base-300/20 text-base-content/25 ring-1 ring-base-300/30",
+              if(@compact, do: "text-[9px] w-4 h-4", else: "text-[10px] w-5 h-5")
+            ]}
+          >
+            ?
+          </div>
           <div :if={@pokemon.shiny} title="Shiny">
             <.icon
               name="hero-sparkles"
@@ -126,6 +137,13 @@ defmodule CobblemonUiWeb.PokemonComponents do
             <h3 class="font-bold text-base-content capitalize text-lg flex items-center gap-2">
               {@pokemon.species || "Unknown"}
               <.tier_badge :if={@tier} tier={@tier} species={@pokemon.species} />
+              <span
+                :if={!@tier}
+                class="inline-flex items-center gap-1 text-xs text-base-content/30 font-normal italic"
+                title="Tier data not downloaded"
+              >
+                <.icon name="hero-arrow-down-tray" class="size-3" /> tier unavailable
+              </span>
               <span :if={@pokemon.shiny} class="text-warning text-sm" title="Shiny">★</span>
             </h3>
             <p class="text-xs text-base-content/40">