Alex Mickelson
37602fc78a
All checks were successful
Build and Deploy / Build & Push Image (push) Successful in 37s
91 lines
2.7 KiB
Elixir
91 lines
2.7 KiB
Elixir
defmodule CobblemonUi.TierListScraper do
|
|
require Logger
|
|
|
|
@url "https://rankedboost.com/pokemon/tier-list/"
|
|
@filename "pokemon_tier_list.json"
|
|
|
|
def output_file do
|
|
dir = System.get_env("CACHE_DIR", ".")
|
|
Path.join(dir, @filename)
|
|
end
|
|
|
|
def run do
|
|
Logger.info("[TierListScraper] Starting scrape from #{@url}")
|
|
|
|
with {:ok, html} <- fetch_page(),
|
|
{:ok, pokemon} <- parse(html),
|
|
:ok <- write_json(pokemon) do
|
|
Logger.info("[TierListScraper] Successfully scraped and saved #{length(pokemon)} pokemon")
|
|
{:ok, pokemon}
|
|
else
|
|
{:error, reason} = err ->
|
|
Logger.error("[TierListScraper] Scrape failed: #{inspect(reason)}")
|
|
err
|
|
end
|
|
end
|
|
|
|
def fetch_page do
|
|
Logger.debug("[TierListScraper] Fetching #{@url}")
|
|
case Req.get(@url, headers: [{"user-agent", "Mozilla/5.0 (compatible; CobblemonUI/1.0)"}]) do
|
|
{:ok, %Req.Response{status: 200, body: body}} ->
|
|
Logger.debug("[TierListScraper] Fetch OK, body size: #{byte_size(body)} bytes")
|
|
{:ok, body}
|
|
{:ok, %Req.Response{status: status}} ->
|
|
Logger.warning("[TierListScraper] Unexpected HTTP status: #{status}")
|
|
{:error, {:http_error, status}}
|
|
{:error, err} ->
|
|
Logger.error("[TierListScraper] HTTP request failed: #{inspect(err)}")
|
|
{:error, err}
|
|
end
|
|
end
|
|
|
|
def parse(html) do
|
|
nodes = html |> Floki.parse_document!() |> Floki.find(".pokemon-tier")
|
|
Logger.debug("[TierListScraper] Found #{length(nodes)} .pokemon-tier nodes")
|
|
|
|
case nodes do
|
|
[] ->
|
|
Logger.warning("[TierListScraper] No .pokemon-tier elements found — page structure may have changed or content is JS-rendered")
|
|
{:error, :no_pokemon_found}
|
|
_ ->
|
|
pokemon = Enum.map(nodes, &extract_pokemon/1)
|
|
valid = Enum.filter(pokemon, fn %{name: n} -> n != "" end)
|
|
Logger.info("[TierListScraper] Parsed #{length(valid)} valid pokemon (#{length(pokemon) - length(valid)} skipped with empty names)")
|
|
{:ok, valid}
|
|
end
|
|
end
|
|
|
|
defp extract_pokemon(node) do
|
|
name =
|
|
node
|
|
|> Floki.find(".name")
|
|
|> Floki.text()
|
|
|> String.trim()
|
|
|
|
tier =
|
|
node
|
|
|> Floki.find(".tier")
|
|
|> Floki.text()
|
|
|> String.trim()
|
|
|
|
%{name: name, tier: tier}
|
|
end
|
|
|
|
def write_json(data) do
|
|
json = Jason.encode!(data, pretty: true)
|
|
File.write(output_file(), json)
|
|
end
|
|
|
|
# Returns a map of %{lowercase_name => tier} for fast lookup, or an empty map if unavailable.
|
|
def load_tier_list do
|
|
with {:ok, contents} <- File.read(output_file()),
|
|
{:ok, entries} <- Jason.decode(contents) do
|
|
Map.new(entries, fn %{"name" => name, "tier" => tier} ->
|
|
{String.downcase(name), tier}
|
|
end)
|
|
else
|
|
_ -> %{}
|
|
end
|
|
end
|
|
end
|