Add Slate extractor

This commit is contained in:
Shadowfacts 2021-09-03 16:24:35 -04:00
parent fd75482779
commit 5990d0e4c2
Signed by: shadowfacts
GPG Key ID: 94A5AB95422746E5
3 changed files with 41 additions and 7 deletions

View File

@ -0,0 +1,39 @@
defmodule Frenzy.Pipeline.Extractor.Slate do
@moduledoc """
Extractor for https://slate.com
"""
alias Frenzy.Pipeline.Extractor
@behaviour Extractor
@impl Extractor
def extract(html_tree) do
case get_article_content(html_tree) do
nil ->
{:error, "no matching elements"}
elem ->
{:ok, elem}
end
end
defp get_article_content(html_tree) do
case Floki.find(html_tree, ".article__content") do
[el] ->
article_content = Floki.filter_out(el, ".slate-ad, .in-article-recirc, .social-share")
image = Floki.find(html_tree, ".article__top-image img")
case image do
[] ->
article_content
[image | _] ->
[image, article_content]
end
_ ->
nil
end
end
end

View File

@ -113,14 +113,8 @@ defmodule Frenzy.Pipeline.ScrapeStage do
html = Floki.map(html, rewrite_image_urls(convert_to_data_uris, URI.parse(url))) html = Floki.map(html, rewrite_image_urls(convert_to_data_uris, URI.parse(url)))
case opts["extractor"] do
"builtin" ->
{:ok, Readability.readable_html(html)} {:ok, Readability.readable_html(html)}
_ ->
{:ok, Floki.raw_html(html)}
end
res -> res ->
res res
end end

View File

@ -9,6 +9,7 @@ defmodule FrenzyWeb.ConfigureStage.ScrapeStageLive do
{"finertech.com", Frenzy.Pipeline.Extractor.FinerTech}, {"finertech.com", Frenzy.Pipeline.Extractor.FinerTech},
{"macstories.net", Frenzy.Pipeline.Extractor.MacStories}, {"macstories.net", Frenzy.Pipeline.Extractor.MacStories},
{"om.co", Frenzy.Pipeline.Extractor.OmMalik}, {"om.co", Frenzy.Pipeline.Extractor.OmMalik},
{"slate.com", Frenzy.Pipeline.Extractor.Slate},
{"whatever.scalzi.com", Frenzy.Pipeline.Extractor.WhateverScalzi} {"whatever.scalzi.com", Frenzy.Pipeline.Extractor.WhateverScalzi}
] ]
|> Enum.map(fn {pretty_name, module} -> |> Enum.map(fn {pretty_name, module} ->