Add Slate extractor

lib/frenzy/pipeline/extractor/slate.ex

@@ -0,0 +1,39 @@
+defmodule Frenzy.Pipeline.Extractor.Slate do
+  @moduledoc """
+  Extractor for https://slate.com
+  """
+
+  alias Frenzy.Pipeline.Extractor
+  @behaviour Extractor
+
+  @impl Extractor
+  def extract(html_tree) do
+    case get_article_content(html_tree) do
+      nil ->
+        {:error, "no matching elements"}
+
+      elem ->
+        {:ok, elem}
+    end
+  end
+
+  defp get_article_content(html_tree) do
+    case Floki.find(html_tree, ".article__content") do
+      [el] ->
+        article_content = Floki.filter_out(el, ".slate-ad, .in-article-recirc, .social-share")
+
+        image = Floki.find(html_tree, ".article__top-image img")
+
+        case image do
+          [] ->
+            article_content
+
+          [image | _] ->
+            [image, article_content]
+        end
+
+      _ ->
+        nil
+    end
+  end
+end

lib/frenzy/pipeline/scrape_stage.ex

@@ -113,13 +113,7 @@ defmodule Frenzy.Pipeline.ScrapeStage do
 
         html = Floki.map(html, rewrite_image_urls(convert_to_data_uris, URI.parse(url)))
 
-        case opts["extractor"] do
-          "builtin" ->
-            {:ok, Readability.readable_html(html)}
-
-          _ ->
-            {:ok, Floki.raw_html(html)}
-        end
+        {:ok, Readability.readable_html(html)}
 
       res ->
         res

lib/frenzy_web/live/configure_stage/scrape_stage_live.ex

@@ -9,6 +9,7 @@ defmodule FrenzyWeb.ConfigureStage.ScrapeStageLive do
                 {"finertech.com", Frenzy.Pipeline.Extractor.FinerTech},
                 {"macstories.net", Frenzy.Pipeline.Extractor.MacStories},
                 {"om.co", Frenzy.Pipeline.Extractor.OmMalik},
+                {"slate.com", Frenzy.Pipeline.Extractor.Slate},
                 {"whatever.scalzi.com", Frenzy.Pipeline.Extractor.WhateverScalzi}
               ]
               |> Enum.map(fn {pretty_name, module} ->