defmodule CobblemonUi.TierListScraper do require Logger @url "https://rankedboost.com/pokemon/tier-list/" @filename "pokemon_tier_list.json" def output_file do dir = System.get_env("CACHE_DIR", ".") Path.join(dir, @filename) end def run do Logger.info("[TierListScraper] Starting scrape from #{@url}") with {:ok, html} <- fetch_page(), {:ok, pokemon} <- parse(html), :ok <- write_json(pokemon) do Logger.info("[TierListScraper] Successfully scraped and saved #{length(pokemon)} pokemon") {:ok, pokemon} else {:error, reason} = err -> Logger.error("[TierListScraper] Scrape failed: #{inspect(reason)}") err end end def fetch_page do Logger.debug("[TierListScraper] Fetching #{@url}") case Req.get(@url, headers: [{"user-agent", "Mozilla/5.0 (compatible; CobblemonUI/1.0)"}]) do {:ok, %Req.Response{status: 200, body: body}} -> Logger.debug("[TierListScraper] Fetch OK, body size: #{byte_size(body)} bytes") {:ok, body} {:ok, %Req.Response{status: status}} -> Logger.warning("[TierListScraper] Unexpected HTTP status: #{status}") {:error, {:http_error, status}} {:error, err} -> Logger.error("[TierListScraper] HTTP request failed: #{inspect(err)}") {:error, err} end end def parse(html) do nodes = html |> Floki.parse_document!() |> Floki.find(".pokemon-tier") Logger.debug("[TierListScraper] Found #{length(nodes)} .pokemon-tier nodes") case nodes do [] -> Logger.warning("[TierListScraper] No .pokemon-tier elements found — page structure may have changed or content is JS-rendered") {:error, :no_pokemon_found} _ -> pokemon = Enum.map(nodes, &extract_pokemon/1) valid = Enum.filter(pokemon, fn %{name: n} -> n != "" end) Logger.info("[TierListScraper] Parsed #{length(valid)} valid pokemon (#{length(pokemon) - length(valid)} skipped with empty names)") {:ok, valid} end end defp extract_pokemon(node) do name = node |> Floki.find(".name") |> Floki.text() |> String.trim() tier = node |> Floki.find(".tier") |> Floki.text() |> String.trim() %{name: name, tier: tier} end def write_json(data) do json = Jason.encode!(data, pretty: true) File.write(output_file(), json) end # Returns a map of %{lowercase_name => tier} for fast lookup, or an empty map if unavailable. def load_tier_list do with {:ok, contents} <- File.read(output_file()), {:ok, entries} <- Jason.decode(contents) do Map.new(entries, fn %{"name" => name, "tier" => tier} -> {String.downcase(name), tier} end) else _ -> %{} end end end