Add configurable refresh frequency to feeds

lib/frenzy/feed.ex

@@ -36,6 +36,8 @@ defmodule Frenzy.Feed do
     field :title, :string
     field :favicon, :string
     field :favicon_url, :string
+    field :last_refreshed_at, :utc_datetime
+    field :refresh_frequency, :integer, default: 30
 
     belongs_to :group, Frenzy.Group
     belongs_to :pipeline, Frenzy.Pipeline
@@ -54,6 +56,8 @@ defmodule Frenzy.Feed do
           title: String.t() | nil,
           favicon: String.t() | nil,
           favicon_url: String.t() | nil,
+          last_refreshed_at: DateTime.t() | nil,
+          refresh_frequency: integer(),
           group: Frenzy.Group.t() | Ecto.Association.NotLoaded.t(),
           pipeline: Frenzy.Pipeline.t() | nil | Ecto.Association.NotLoaded.t(),
           items: [Frenzy.Item.t()] | Ecto.Association.NotLoaded.t(),
@@ -71,8 +75,11 @@ defmodule Frenzy.Feed do
       :last_updated,
       :pipeline_id,
       :favicon,
-      :favicon_url
+      :favicon_url,
+      :last_refreshed_at,
+      :refresh_frequency
     ])
     |> validate_required([:feed_url])
+    |> validate_inclusion(:refresh_frequency, [30 * 60, 60 * 60, 6 * 60 * 60, 24 * 60 * 60])
   end
 end

lib/frenzy/update_feeds.ex

@@ -40,8 +40,17 @@ defmodule Frenzy.UpdateFeeds do
   defp update_feeds() do
     Logger.info("Updating all feeds")
 
-    Repo.all(from(Feed))
+    {_count, feeds} =
-    |> Enum.map(&update_feed/1)
+      Feed
+      |> where(
+        [f],
+        is_nil(f.last_refreshed_at) or
+          f.last_refreshed_at <= from_now(-1 * f.refresh_frequency, "second")
+      )
+      |> select([f], f)
+      |> Repo.update_all(set: [last_refreshed_at: DateTime.utc_now()])
+
+    Enum.map(feeds, &update_feed/1)
 
     prune_old_items()
   end

lib/frenzy_web/controllers/feed_controller.ex

@@ -77,7 +77,13 @@ defmodule FrenzyWeb.FeedController do
     render(conn, "edit.html", %{
       feed: feed,
       changeset: edit_changeset,
-      pipelines: [{"No Pipeline", nil} | pipelines]
+      pipelines: [{"No Pipeline", nil} | pipelines],
+      refresh_frequencies: [
+        {"Every 30 minutes", 30 * 60},
+        {"Every hour", 60 * 60},
+        {"Every 6 hours", 6 * 60 * 60},
+        {"Every 24 hours", 24 * 60 * 60}
+      ]
     })
   end
 

lib/frenzy_web/templates/feed/edit.html.eex

@@ -1,6 +1,12 @@
 <h1>Edit Feed</h1>
 
 <%= form_for @changeset, Routes.feed_path(@conn, :update, @feed.id), fn f -> %>
+  <div class="form-group row">
+    <label class="col-sm-2 col-form-label" for="refresh_frequency">Refresh Frequency</label>
+    <div class="col-sm-10">
+      <%= select f, :refresh_frequency, @refresh_frequencies, class: "custom-select" %>
+    </div>
+  </div>
   <div class="form-group row">
     <label class="col-sm-2 col-form-label" for="pipeline_id">Pipeline ID</label>
     <div class="col-sm-10">

priv/repo/migrations/20200531201714_feeds_add_refresh_frequency.exs

@@ -0,0 +1,10 @@
+defmodule Frenzy.Repo.Migrations.FeedsAddRefreshFrequency do
+  use Ecto.Migration
+
+  def change do
+    alter table(:feeds) do
+      add :last_refreshed_at, :naive_datetime, default: nil
+      add :refresh_frequency, :integer, default: 30 * 60
+    end
+  end
+end