shadowfacts
/
frenzy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix whatever.scalzi.com extractor

This commit is contained in:
Shadowfacts 2020-08-14 21:55:38 -04:00
parent 5f3be52132
commit bd42073e24
Signed by: shadowfacts
GPG Key ID: 94A5AB95422746E5
1 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
lib/frenzy/pipeline/extractor/whatever_scalzi.ex
View File

@ -18,11 +18,17 @@ defmodule Frenzy.Pipeline.Extractor.WhateverScalzi do
end
defp get_article_content(html_tree) do
case Floki.find(html_tree, "article.post > div.entry-content") do
[content_elem | _] ->
# remove social media buttons that are included in the .entry-content element
Floki.filter_out(content_elem, "div#jp-post-flair")
# there's no element that contains only the post content
# .postarea contains the headline, post content, social media buttons, and comments
with [{_tag, _attrs, postarea_children} | _] <- Floki.find(html_tree, ".postarea"),
{_before_headline, [_headline | rest]} <-
Enum.split_while(postarea_children, fn {tag, _attrs, _children} -> tag != "h1" end),
{article_content, _rest} <-
Enum.split_while(rest, fn {tag, attrs, _children} ->
tag != "div" || !({"id", "jp-post-flair"} in attrs)
end) do
article_content
else
_ ->
nil
end