Fix whatever.scalzi.com extractor
This commit is contained in:
parent
0ded09a65d
commit
26b832b622
|
@ -20,15 +20,55 @@ defmodule Frenzy.Pipeline.Extractor.WhateverScalzi do
|
|||
defp get_article_content(html_tree) do
|
||||
# there's no element that contains only the post content
|
||||
# .postarea contains the headline, post content, social media buttons, and comments
|
||||
with [{_tag, _attrs, postarea_children} | _] <- Floki.find(html_tree, ".postarea"),
|
||||
{_before_headline, [_headline | rest]} <-
|
||||
Enum.split_while(postarea_children, fn {tag, _attrs, _children} -> tag != "h1" end),
|
||||
{article_content, _rest} <-
|
||||
Enum.split_while(rest, fn {tag, attrs, _children} ->
|
||||
tag != "div" || !({"id", "jp-post-flair"} in attrs)
|
||||
end) do
|
||||
article_content
|
||||
else
|
||||
case Floki.find(html_tree, ".postarea") do
|
||||
[{_tag, _attrs, postarea_children}] ->
|
||||
Enum.split_while(postarea_children, fn
|
||||
{"h1", _, _} -> true
|
||||
_ -> false
|
||||
end)
|
||||
|> case do
|
||||
{_before_headline, [_headline | rest]} ->
|
||||
{article_content, _rest} =
|
||||
Enum.split_while(rest, fn
|
||||
{"div", attrs, _} = el ->
|
||||
class = Floki.attribute(el, "class") |> List.first()
|
||||
|
||||
if {"id", "comments"} in attrs do
|
||||
false
|
||||
else
|
||||
is_nil(class) || !String.contains?(class, "sharedaddy")
|
||||
end
|
||||
|
||||
_ ->
|
||||
true
|
||||
end)
|
||||
|
||||
article_content
|
||||
|> Floki.map(fn
|
||||
{"img", attrs} = el ->
|
||||
class = Enum.find(attrs, fn {k, _} -> k == "class" end)
|
||||
class = if is_nil(class), do: nil, else: elem(class, 1)
|
||||
|
||||
if !is_nil(class) && String.contains?(class, "jetpack-lazy-image") do
|
||||
{
|
||||
"img",
|
||||
Enum.filter(attrs, fn
|
||||
{"srcset", _} -> false
|
||||
_ -> true
|
||||
end)
|
||||
}
|
||||
else
|
||||
el
|
||||
end
|
||||
|
||||
el ->
|
||||
el
|
||||
end)
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
||||
_ ->
|
||||
nil
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue