Improve The Verge extractor
This commit is contained in:
parent
b0089083db
commit
fec640a37a
|
@ -16,8 +16,10 @@ defmodule Frenzy.Pipeline.Extractor.TheVerge do
|
|||
html_tree
|
||||
|> Floki.find("article#content > div:not(.duet--article--lede)")
|
||||
|> Floki.filter_out(
|
||||
".duet--layout--rail, .duet--article--article-pullquote, .duet--article--comments-join-the-conversation, .duet--recirculation--related-list, .duet--article--comments-button"
|
||||
".hidden, .duet--layout--rail, .duet--article--article-pullquote, .duet--article--comments-join-the-conversation, .duet--recirculation--related-list, .duet--article--comments-button, .duet--article--share-buttons"
|
||||
)
|
||||
|> Readability.Helper.remove_attrs("style")
|
||||
|> Floki.traverse_and_update(&rewrite/1)
|
||||
|
||||
{:ok, image ++ content}
|
||||
end
|
||||
|
@ -54,4 +56,57 @@ defmodule Frenzy.Pipeline.Extractor.TheVerge do
|
|||
[]
|
||||
end
|
||||
end
|
||||
|
||||
defp rewrite({tag, attrs, children} = el) do
|
||||
cond do
|
||||
is_empty_gif(el) ->
|
||||
nil
|
||||
|
||||
is_gallery(el) ->
|
||||
images =
|
||||
Floki.find(children, ":not(noscript) > img")
|
||||
|> Enum.map(fn el ->
|
||||
[src] = Floki.attribute(el, "src")
|
||||
src
|
||||
end)
|
||||
|
||||
{
|
||||
"div",
|
||||
[
|
||||
{"style",
|
||||
"display: flex; flex-direction: row; overflow-x: auto; scroll-snap-type: x mandatory;;"}
|
||||
],
|
||||
Enum.map(images, fn src ->
|
||||
{
|
||||
"img",
|
||||
[
|
||||
{"src", src},
|
||||
{"loading", "lazy"},
|
||||
{"style", "max-width: 85%; max-height: 30vh; scroll-snap-align: start;"}
|
||||
],
|
||||
[]
|
||||
}
|
||||
end)
|
||||
}
|
||||
|
||||
true ->
|
||||
el
|
||||
end
|
||||
end
|
||||
|
||||
defp rewrite(other), do: other
|
||||
|
||||
defp is_gallery(el) do
|
||||
case Floki.attribute(el, "class") do
|
||||
[classes] -> String.contains?(classes, "duet--article--gallery")
|
||||
_ -> false
|
||||
end
|
||||
end
|
||||
|
||||
defp is_empty_gif(el) do
|
||||
case Floki.attribute(el, "src") do
|
||||
[src] -> String.starts_with?(src, "data:image/gif;")
|
||||
_ -> false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue