lib/frenzy/pipeline/extractor.ex

@@ -1,3 +0,0 @@
-defmodule Frenzy.Pipeline.Extractor do
-  @callback extract(Floki.html_tree()) :: {:ok, Floki.html_tree()} | {:error, String.t()}
-end

lib/frenzy/pipeline/extractor/daring_fireball.ex

@@ -1,40 +0,0 @@
-defmodule Frenzy.Pipeline.Extractor.DaringFireball do
-  @moduledoc """
-  Extractor for https://daringfireball.net
-  """
-
-  alias Frenzy.Pipeline.Extractor
-  @behaviour Extractor
-
-  @impl Extractor
-  def extract(html_tree) do
-    case get_article_element(html_tree) || get_link_element(html_tree) do
-      nil ->
-        {:error, "no matching elements"}
-
-      elem ->
-        {:ok, elem}
-    end
-  end
-
-  defp get_article_element(html_tree) do
-    case Floki.find(html_tree, "div.article") do
-      [article_elem | _] ->
-        # articles include extra information in the div.article element
-        Floki.filter_out(article_elem, "h1, .dateline, #PreviousNext")
-
-      _ ->
-        nil
-    end
-  end
-
-  defp get_link_element(html_tree) do
-    case Floki.find(html_tree, "dl.linkedlist dd") do
-      [dd_elem | _] ->
-        dd_elem
-
-      _ ->
-        nil
-    end
-  end
-end

lib/frenzy/pipeline/extractor/erica_sadun.ex

@@ -1,23 +0,0 @@
-defmodule Frenzy.Pipeline.Extractor.EricaSadun do
-  @moduledoc """
-  Extractor for https://ericasadun.com
-  """
-
-  alias Frenzy.Pipeline.Extractor
-  @behaviour Extractor
-
-  @impl Extractor
-  def extract(html_tree) do
-    case Floki.find(html_tree, ".post-content") do
-      [content_elem | _] ->
-        # content element includes social media buttons and related posts
-        {
-          :ok,
-          Floki.filter_out(content_elem, "div.sharedaddy, div#jp-relatedposts")
-        }
-
-      _ ->
-        {:error, "no matching elements"}
-    end
-  end
-end

lib/frenzy/pipeline/extractor/whatever_scalzi.ex

@@ -1,30 +0,0 @@
-defmodule Frenzy.Pipeline.Extractor.WhateverScale do
-  @moduledoc """
-  Extractor for https://whatever.scalzi.com
-  """
-
-  alias Frenzy.Pipeline.Extractor
-  @behaviour Extractor
-
-  @impl Extractor
-  def extract(html_tree) do
-    case get_article_content(html_tree) do
-      nil ->
-        {:error, "no matching elements"}
-
-      elem ->
-        {:ok, elem}
-    end
-  end
-
-  defp get_article_content(html_tree) do
-    case Floki.find(html_tree, "article.post > div.entry-content") do
-      [content_elem | _] ->
-        # remove social media buttons that are included in the .entry-content element
-        Floki.filter_out(content_elem, "div#jp-post-flair")
-
-      _ ->
-        nil
-    end
-  end
-end

lib/frenzy/pipeline/scrape_stage.ex

@@ -4,8 +4,8 @@ defmodule Frenzy.Pipeline.ScrapeStage do
   @behaviour Stage
 
   @impl Stage
-  def apply(opts, %{url: url} = item_params) do
+  def apply(_opts, %{url: url} = item_params) do
-    case get_article_content(url, opts["extractor"]) do
+    case get_article_content(url) do
       {:ok, content} ->
         {:ok, %{item_params | content: content}}
 
@@ -16,41 +16,17 @@ defmodule Frenzy.Pipeline.ScrapeStage do
   end
 
   @impl Stage
-  def validate_opts(opts) when is_map(opts) do
+  def validate_opts(opts), do: {:ok, opts}
-    # todo: figure out why this errors when an empty map is provided
-    case opts["extractor"] do
-      nil ->
-        {:ok, %{opts | extractor: "builtin"}}
 
-      extractor when not is_binary(extractor) ->
+  @spec get_article_content(String.t()) :: {:ok, String.t()} | {:error, String.t()}
-        {:error, "extractor must be a string"}
+  defp get_article_content(url) when is_binary(url) and url != "" do
-
-      "builtin" ->
-        {:ok, opts}
-
-      extractor ->
-        try do
-          String.to_existing_atom("Elixir." <> extractor)
-          {:ok, opts}
-        rescue
-          ArgumentError ->
-            {:error, "extractor must be \"builtin\" or a module that exists"}
-        end
-    end
-  end
-
-  @impl Stage
-  def validate_opts(_), do: {:error, "options must be a map"}
-
-  @spec get_article_content(String.t(), String.t()) :: {:ok, String.t()} | {:error, String.t()}
-  defp get_article_content(url, extractor) when is_binary(url) and url != "" do
     Logger.debug("Getting article from #{url}")
 
     url
     |> HTTPoison.get()
     |> case do
       {:ok, response} ->
-        handle_response(url, response, extractor)
+        handle_response(url, response)
 
       {:error, %HTTPoison.Error{reason: reason}} ->
         {:error, "HTTPoison error: #{reason}"}
@@ -59,82 +35,38 @@ defmodule Frenzy.Pipeline.ScrapeStage do
 
   defp get_article_content(_url), do: {:error, "URL must be a non-empty string"}
 
-  @spec handle_response(String.t(), HTTPoison.Response.t(), String.t()) ::
+  @spec handle_response(String.t(), HTTPoison.Response.t()) ::
           {:ok, String.t()} | {:error, String.t()}
-  defp handle_response(url, %HTTPoison.Response{status_code: 200, body: body}, extractor) do
+  defp handle_response(_url, %HTTPoison.Response{status_code: 200, body: body}) do
-    case extractor do
+    article = Readability.article(body)
-      "builtin" ->
+    {:ok, Readability.readable_html(article)}
-        {:ok, Readability.article(body)}
-
-      module_name ->
-        html_tree = Floki.parse(body)
-        apply(String.to_existing_atom("Elixir." <> module_name), :extract, [html_tree])
-    end
-    |> case do
-      {:ok, html} ->
-        html = Floki.map(html, rewrite_image_urls(URI.parse(url)))
-
-        case extractor do
-          "builtin" ->
-            {:ok, Readability.readable_html(html)}
-
-          _ ->
-            {:ok, Floki.raw_html(html)}
   end
 
-      res ->
+  defp handle_response(_url, %HTTPoison.Response{status_code: 404}) do
-        res
-    end
-  end
-
-  defp handle_response(_url, %HTTPoison.Response{status_code: 404}, _extractor) do
     {:error, "404 not found"}
   end
 
-  defp handle_response(
+  defp handle_response(url, %HTTPoison.Response{status_code: status_code, headers: headers})
-         url,
-         %HTTPoison.Response{status_code: status_code, headers: headers},
-         extractor
-       )
        when status_code in [301, 302] do
+    {"Location", new_url} = Enum.find(headers, fn {name, _value} -> name == "Location" end)
+
     headers
     |> Enum.find(fn {name, _value} -> name == "Location" end)
     |> case do
       {"Location", new_url} ->
         Logger.debug("Got 301 redirect from #{url} to #{new_url}")
-        get_article_content(new_url, extractor)
+        get_article_content(new_url)
 
       _ ->
         {:error, "Missing Location header for redirect"}
     end
   end
 
-  defp handle_response(_url, %HTTPoison.Response{status_code: 403}, _extractor) do
+  defp handle_response(_url, %HTTPoison.Response{status_code: 403}) do
     {:error, "403 Forbidden"}
   end
 
-  defp handle_response(_url, %HTTPoison.Response{} = response, _extractor) do
+  defp handle_response(_url, %HTTPoison.Response{status_code: status_code} = response) do
     {:error, "No handler for response #{inspect(response)}"}
   end
-
-  # Generates a helper function for the article with the given URI that takes an HTML element and,
-  # if it's an <img> element whose src attribute does not have a hostname, adds the hostname and
-  # scheme to the element.
-  defp rewrite_image_urls(%URI{host: host, scheme: scheme}) do
-    fn
-      {"img", [{"src", src} | attrs]} = elem ->
-        case URI.parse(src) do
-          %URI{host: nil, path: path} ->
-            new_src = URI.to_string(%URI{path: path, host: host, scheme: scheme})
-
-            {"img", [{"src", new_src} | attrs]}
-
-          _ ->
-            elem
-        end
-
-      elem ->
-        elem
-    end
-  end
 end

lib/frenzy/pipeline/site/daring_fireball_scrape_stage.ex

@@ -0,0 +1,105 @@
+defmodule Frenzy.Pipeline.Site.DaringFireballScrapeStage do
+  require Logger
+  alias Frenzy.Pipeline.Stage
+  @behaviour Stage
+
+  @impl Stage
+  def apply(_opts, %{url: url} = item_params) do
+    case get_article_content(url) do
+      {:ok, content} ->
+        {:ok, %{item_params | content: content}}
+
+      {:error, reason} ->
+        Logger.warn("Unable to get Daring Fireball article content for #{url}: #{reason}")
+        {:ok, item_params}
+    end
+  end
+
+  @impl Stage
+  def validate_opts(opts), do: {:ok, opts}
+
+  @spec get_article_content(String.t()) :: {:ok, String.t()} | {:error, String.t()}
+  defp get_article_content(url) when is_binary(url) and url != "" do
+    Logger.debug("Get Daring Fireball article from #{url}")
+
+    url
+    |> HTTPoison.get()
+    |> case do
+      {:ok, response} ->
+        handle_response(url, response)
+
+      {:error, %HTTPoison.Error{reason: reason}} ->
+        {:error, "HTTPoison error: #{reason}"}
+    end
+  end
+
+  defp get_article_content(_url), do: {:error, "URL must be a non-empty string"}
+
+  @spec handle_response(String.t(), HTTPoison.Response.t()) ::
+          {:ok, String.t()} | {:error, String.t()}
+  defp handle_response(url, %HTTPoison.Response{status_code: 200, body: body}) do
+    html_tree = Floki.parse(body)
+
+    case get_article_element(html_tree) || get_link_element(html_tree) do
+      nil ->
+        {:error, "no matching element"}
+
+      elem ->
+        readable_html =
+          elem
+          |> Floki.filter_out(:comment)
+          |> Readability.readable_html()
+
+        {:ok, readable_html}
+    end
+  end
+
+  defp handle_response(_url, %HTTPoison.Response{status_code: 404}) do
+    {:error, "404 not found"}
+  end
+
+  defp handle_response(url, %HTTPoison.Response{status_code: status_code, headers: headers})
+       when status_code in [301, 302] do
+    {"Location", new_url} = Enum.find(headers, fn {name, _value} -> name == "Location" end)
+
+    headers
+    |> Enum.find(fn {name, _value} -> name == "Location" end)
+    |> case do
+      {"Location", new_url} ->
+        Logger.debug("Got 301 redirect from #{url} to #{new_url}")
+        get_article_content(new_url)
+
+      _ ->
+        {:error, "Missing Location header for redirect"}
+    end
+  end
+
+  defp handle_response(_url, %HTTPoison.Response{status_code: 403}) do
+    {:error, "403 Forbidden"}
+  end
+
+  defp handle_response(_url, %HTTPoison.Response{status_code: status_code} = response) do
+    {:error, "No handler for response #{inspect(response)}"}
+  end
+
+  defp get_article_element(html_tree) do
+    case Floki.find(html_tree, "div.article") do
+      [article_elem | _] ->
+        # articles include extra information in the div.article element
+        Floki.filter_out(article_elem, "h1, .dateline, #PreviousNext")
+
+      _ ->
+        nil
+    end
+  end
+
+  defp get_link_element(html_tree) do
+    case Floki.find(html_tree, "dl.linkedlist dd") do
+      [dd_elem | _] ->
+        dd_elem
+
+      _ ->
+        nil
+    end
+  end
+end

lib/frenzy_web/controllers/account_controller.ex

@@ -1,6 +1,6 @@
 defmodule FrenzyWeb.AccountController do
   use FrenzyWeb, :controller
-  alias Frenzy.{Repo, User, FervorClient}
+  alias Frenzy.{Repo, User, FervorClient, Filter}
   alias FrenzyWeb.Router.Helpers, as: Routes
   alias FrenzyWeb.Endpoint
 
@@ -125,7 +125,11 @@ defmodule FrenzyWeb.AccountController do
         Enum.each(feeds, fn feed_url ->
           feed_changeset =
             Ecto.build_assoc(group, :feeds, %{
-              feed_url: feed_url
+              feed_url: feed_url,
+              filter: %Filter{
+                mode: "reject",
+                score: 0
+              }
             })
 
           {:ok, _feed} = Repo.insert(feed_changeset)

lib/frenzy_web/controllers/feed_controller.ex

@@ -67,7 +67,7 @@ defmodule FrenzyWeb.FeedController do
   end
 
   def edit(conn, _params) do
-    feed = conn.assigns[:feed] |> Repo.preload(:pipeline_stages)
+    feed = conn.assigns[:feed]
     stages = Enum.sort_by(feed.pipeline_stages, fn stage -> stage.index end)
 
     render(conn, "edit.html", %{

lib/frenzy_web/controllers/fervor/feeds_controller.ex

@@ -1,6 +1,6 @@
 defmodule FrenzyWeb.Fervor.FeedsController do
   use FrenzyWeb, :controller
-  alias Frenzy.{Repo, Feed, Item}
+  alias Frenzy.{Repo, Feed, Filter, Item}
   import Ecto.Query
   alias FrenzyWeb.Fervor.Paginator
 
@@ -85,7 +85,11 @@ defmodule FrenzyWeb.Fervor.FeedsController do
           group ->
             changeset =
               Ecto.build_assoc(group, :feeds, %{
-                feed_url: feed_url
+                feed_url: feed_url,
+                filter: %Filter{
+                  mode: "reject",
+                  score: 0
+                }
               })
 
             {:ok, feed} = Repo.insert(changeset)

lib/frenzy_web/templates/feed/show.html.eex

@@ -17,7 +17,7 @@
     <%= for item <- @items do %>
       <tr <%= if item.read do %>class="item-read"<% end %>>
         <td>
-          <a href="<%= Routes.item_path(@conn, :show, item.id) %>"><%= item.title || "(Untitled)" %></a>
+          <a href="<%= Routes.item_path(@conn, :show, item.id) %>"><%= item.title %></a>
         </td>
         <td>
           <% {:ok, date} = Timex.format(item.date, "{YYYY}-{M}-{D} {h12}:{m} {AM}") %>

priv/static/css/app.css

@@ -3,8 +3,3 @@
 .item-read a {
   color: #606c76;
 }
-
-blockquote {
-	padding-left: 2rem;
-	border-left: 4px solid lightgray;
-}