diff --git a/lib/frenzy/pipeline/gemini_scrape_stage.ex b/lib/frenzy/pipeline/gemini_scrape_stage.ex new file mode 100644 index 0000000..35dfc1f --- /dev/null +++ b/lib/frenzy/pipeline/gemini_scrape_stage.ex @@ -0,0 +1,37 @@ +defmodule Frenzy.Pipeline.GeminiScrapeStage do + require Logger + alias Frenzy.Network + alias Frenzy.Pipeline.Stage + @behaviour Stage + + @impl Stage + def apply(opts, %{url: url} = item_params) do + case get_content(url, opts) do + {:error, reason} -> + Logger.warn("Unable to get Gemini content for #{url}: #{reason}") + {:ok, item_params} + + content -> + {:ok, %{item_params | content: content}} + end + end + + @impl Stage + def validate_opts(opts) do + {:ok, opts} + end + + @impl Stage + def default_opts(), do: %{} + + @spec get_content(String.t(), map()) :: {:ok, String.t()} | {:error, term()} + def get_content(url, _opts) do + case Network.gemini_request(url) do + {:error, reason} -> + {:error, reason} + + {:ok, %Gemini.Response{body: body}} -> + body + end + end +end diff --git a/lib/frenzy/update_feeds.ex b/lib/frenzy/update_feeds.ex index 3b6c172..2e14aeb 100644 --- a/lib/frenzy/update_feeds.ex +++ b/lib/frenzy/update_feeds.ex @@ -30,6 +30,9 @@ defmodule Frenzy.UpdateFeeds do {:noreply, state} end + # workaround for unhanled {:ssl_closed, {:sslsocket, {:gen_tcp, ...}}} message when Gemini module + def handle_info({:ssl_closed, _}, state), do: {:noreply, state} + defp schedule_update() do # 30 minutes Process.send_after(self(), :update_feeds, 30 * 60 * 1000) diff --git a/lib/frenzy_web/live/edit_pipeline_live.ex b/lib/frenzy_web/live/edit_pipeline_live.ex index e7ddcf1..1a0a782 100644 --- a/lib/frenzy_web/live/edit_pipeline_live.ex +++ b/lib/frenzy_web/live/edit_pipeline_live.ex @@ -6,7 +6,8 @@ defmodule FrenzyWeb.EditPipelineLive do @stages [ {"Filter Stage", "Frenzy.Pipeline.FilterStage"}, {"Scrape Stage", "Frenzy.Pipeline.ScrapeStage"}, - {"Conditional Stage", "Frenzy.Pipeline.ConditionalStage"} + {"Conditional Stage", "Frenzy.Pipeline.ConditionalStage"}, + {"Gemini Scrape Stage", "Frenzy.Pipeline.GeminiScrapeStage"} ] def stages, do: @stages