defmodule Frenzy.Pipeline.GeminiScrapeStage do require Logger alias Frenzy.Network alias Frenzy.Pipeline.Stage @behaviour Stage @impl Stage def apply(opts, %{url: url} = item_params) do case get_content(url, opts) do {:error, reason} -> Logger.warn("Unable to get Gemini content for #{url}: #{reason}") {:ok, item_params} content -> {:ok, %{item_params | content: content}} end end @impl Stage def validate_opts(opts) do {:ok, opts} end @impl Stage def default_opts(), do: %{} @spec get_content(String.t(), map()) :: {:ok, String.t()} | {:error, term()} def get_content(url, _opts) do case Network.gemini_request(url) do {:error, reason} -> {:error, reason} {:ok, %Gemini.Response{body: body}} -> body end end end