Improve The Verge extractor

This commit is contained in:
Shadowfacts 2023-07-12 21:49:53 -07:00
parent b0089083db
commit fec640a37a
1 changed files with 56 additions and 1 deletions

View File

@ -16,8 +16,10 @@ defmodule Frenzy.Pipeline.Extractor.TheVerge do
html_tree
|> Floki.find("article#content > div:not(.duet--article--lede)")
|> Floki.filter_out(
".duet--layout--rail, .duet--article--article-pullquote, .duet--article--comments-join-the-conversation, .duet--recirculation--related-list, .duet--article--comments-button"
".hidden, .duet--layout--rail, .duet--article--article-pullquote, .duet--article--comments-join-the-conversation, .duet--recirculation--related-list, .duet--article--comments-button, .duet--article--share-buttons"
)
|> Readability.Helper.remove_attrs("style")
|> Floki.traverse_and_update(&rewrite/1)
{:ok, image ++ content}
end
@ -54,4 +56,57 @@ defmodule Frenzy.Pipeline.Extractor.TheVerge do
[]
end
end
defp rewrite({tag, attrs, children} = el) do
cond do
is_empty_gif(el) ->
nil
is_gallery(el) ->
images =
Floki.find(children, ":not(noscript) > img")
|> Enum.map(fn el ->
[src] = Floki.attribute(el, "src")
src
end)
{
"div",
[
{"style",
"display: flex; flex-direction: row; overflow-x: auto; scroll-snap-type: x mandatory;;"}
],
Enum.map(images, fn src ->
{
"img",
[
{"src", src},
{"loading", "lazy"},
{"style", "max-width: 85%; max-height: 30vh; scroll-snap-align: start;"}
],
[]
}
end)
}
true ->
el
end
end
defp rewrite(other), do: other
defp is_gallery(el) do
case Floki.attribute(el, "class") do
[classes] -> String.contains?(classes, "duet--article--gallery")
_ -> false
end
end
defp is_empty_gif(el) do
case Floki.attribute(el, "src") do
[src] -> String.starts_with?(src, "data:image/gif;")
_ -> false
end
end
end