58 lines
1.3 KiB
Elixir
58 lines
1.3 KiB
Elixir
defmodule Frenzy.Pipeline.Extractor.TheVerge do
|
|
@moduledoc """
|
|
Extractor for https://theverge.com
|
|
Handles their bizarro new layout that's a pile of unsemantic classes
|
|
"""
|
|
|
|
require Logger
|
|
alias Frenzy.Pipeline.Extractor
|
|
@behaviour Extractor
|
|
|
|
@impl Extractor
|
|
def extract(html_tree) do
|
|
image = extract_header_image(html_tree)
|
|
|
|
content =
|
|
html_tree
|
|
|> Floki.find("article#content > div:not(.duet--article--lede)")
|
|
|> Floki.filter_out(
|
|
".duet--layout--rail, .duet--article--article-pullquote, .duet--article--comments-join-the-conversation"
|
|
)
|
|
|
|
{:ok, image ++ content}
|
|
end
|
|
|
|
@spec extract_header_image(Floki.html_tree()) :: Floki.html_tree()
|
|
defp extract_header_image(html_tree) do
|
|
case Floki.find(html_tree, "article#content > .duet--article--lede figure") do
|
|
[figure | _] ->
|
|
img =
|
|
case Floki.find(figure, "img") do
|
|
[img | _] ->
|
|
[img]
|
|
|
|
_ ->
|
|
nil
|
|
end
|
|
|
|
caption =
|
|
case Floki.find(figure, ".duet--media--caption") do
|
|
[{_tag, _attrs, children} | _] ->
|
|
[{"figcaption", [], [Floki.text(children)]}]
|
|
|
|
_ ->
|
|
[]
|
|
end
|
|
|
|
if img do
|
|
[{"figure", [], img ++ caption}]
|
|
else
|
|
[]
|
|
end
|
|
|
|
_ ->
|
|
[]
|
|
end
|
|
end
|
|
end
|