45 lines
1.0 KiB
Elixir
45 lines
1.0 KiB
Elixir
defmodule Frenzy.Pipeline.GeminiScrapeStage do
|
|
require Logger
|
|
alias Frenzy.Network
|
|
alias Frenzy.Pipeline.Stage
|
|
@behaviour Stage
|
|
|
|
@impl Stage
|
|
def apply(opts, %{url: url} = item_params) do
|
|
case get_content(url, opts) do
|
|
{:error, reason} ->
|
|
Logger.warn("Unable to get Gemini content for #{url}: #{reason}")
|
|
{:ok, item_params}
|
|
|
|
{content, content_type} ->
|
|
{:ok, %{item_params | content: content, content_type: content_type}}
|
|
end
|
|
end
|
|
|
|
@impl Stage
|
|
def validate_opts(opts) do
|
|
{:ok, opts}
|
|
end
|
|
|
|
@impl Stage
|
|
def default_opts(), do: %{}
|
|
|
|
@spec get_content(String.t(), map()) :: {String.t(), String.t()} | {:error, term()}
|
|
def get_content(url, _opts) do
|
|
case Network.gemini_request(url) do
|
|
{:error, reason} ->
|
|
{:error, reason}
|
|
|
|
{:ok, %Gemini.Response{body: body, meta: meta}} ->
|
|
{body, parse_content_type(meta)}
|
|
end
|
|
end
|
|
|
|
defp parse_content_type(meta) do
|
|
meta
|
|
|> String.split(";")
|
|
|> hd()
|
|
|> String.trim()
|
|
end
|
|
end
|