Fix whatever.scalzi.com extractor

This commit is contained in:
Shadowfacts 2021-03-31 15:30:05 -04:00
parent 0ded09a65d
commit 26b832b622
Signed by: shadowfacts
GPG Key ID: 94A5AB95422746E5
1 changed files with 49 additions and 9 deletions

View File

@ -20,15 +20,55 @@ defmodule Frenzy.Pipeline.Extractor.WhateverScalzi do
defp get_article_content(html_tree) do
# there's no element that contains only the post content
# .postarea contains the headline, post content, social media buttons, and comments
with [{_tag, _attrs, postarea_children} | _] <- Floki.find(html_tree, ".postarea"),
{_before_headline, [_headline | rest]} <-
Enum.split_while(postarea_children, fn {tag, _attrs, _children} -> tag != "h1" end),
{article_content, _rest} <-
Enum.split_while(rest, fn {tag, attrs, _children} ->
tag != "div" || !({"id", "jp-post-flair"} in attrs)
end) do
article_content
else
case Floki.find(html_tree, ".postarea") do
[{_tag, _attrs, postarea_children}] ->
Enum.split_while(postarea_children, fn
{"h1", _, _} -> true
_ -> false
end)
|> case do
{_before_headline, [_headline | rest]} ->
{article_content, _rest} =
Enum.split_while(rest, fn
{"div", attrs, _} = el ->
class = Floki.attribute(el, "class") |> List.first()
if {"id", "comments"} in attrs do
false
else
is_nil(class) || !String.contains?(class, "sharedaddy")
end
_ ->
true
end)
article_content
|> Floki.map(fn
{"img", attrs} = el ->
class = Enum.find(attrs, fn {k, _} -> k == "class" end)
class = if is_nil(class), do: nil, else: elem(class, 1)
if !is_nil(class) && String.contains?(class, "jetpack-lazy-image") do
{
"img",
Enum.filter(attrs, fn
{"srcset", _} -> false
_ -> true
end)
}
else
el
end
el ->
el
end)
_ ->
nil
end
_ ->
nil
end