Remove old code from before filters and scraping were combined into pipelines

This commit is contained in:
Shadowfacts 2019-10-31 14:18:41 -04:00
parent a856c80d4a
commit 07a62fec25
Signed by: shadowfacts
GPG Key ID: 94A5AB95422746E5
5 changed files with 11 additions and 89 deletions

View File

@ -34,13 +34,10 @@ defmodule Frenzy.Feed do
field :last_updated, :utc_datetime field :last_updated, :utc_datetime
field :site_url, :string field :site_url, :string
field :title, :string field :title, :string
field :filter_enabled, :boolean
field :scrape_remote_content, :boolean
belongs_to :group, Frenzy.Group belongs_to :group, Frenzy.Group
has_many :items, Frenzy.Item, on_delete: :delete_all has_many :items, Frenzy.Item, on_delete: :delete_all
has_one :filter, Frenzy.Filter, on_delete: :delete_all
has_many :pipeline_stages, Frenzy.PipelineStage, on_delete: :delete_all has_many :pipeline_stages, Frenzy.PipelineStage, on_delete: :delete_all
timestamps() timestamps()
@ -53,11 +50,8 @@ defmodule Frenzy.Feed do
last_updated: DateTime.t() | nil, last_updated: DateTime.t() | nil,
site_url: String.t() | nil, site_url: String.t() | nil,
title: String.t() | nil, title: String.t() | nil,
filter_enabled: boolean(),
scrape_remote_content: boolean(),
group: Frenzy.Group.t() | Ecto.Association.NotLoaded.t(), group: Frenzy.Group.t() | Ecto.Association.NotLoaded.t(),
items: [Frenzy.Item.t()] | Ecto.Association.NotLoaded.t(), items: [Frenzy.Item.t()] | Ecto.Association.NotLoaded.t(),
filter: Frenzy.Filter.t() | Ecto.Association.NotLoaded.t(),
pipeline_stages: [Frenzy.PipelineStage.t()] | Ecto.Association.NotLoaded.t(), pipeline_stages: [Frenzy.PipelineStage.t()] | Ecto.Association.NotLoaded.t(),
inserted_at: NaiveDateTime.t(), inserted_at: NaiveDateTime.t(),
updated_at: NaiveDateTime.t() updated_at: NaiveDateTime.t()
@ -70,11 +64,8 @@ defmodule Frenzy.Feed do
:title, :title,
:feed_url, :feed_url,
:site_url, :site_url,
:last_updated, :last_updated
:filter_enabled,
:scrape_remote_content
]) ])
|> cast_assoc(:filter, required: true) |> validate_required([:feed_url])
|> validate_required([:feed_url, :filter])
end end
end end

View File

@ -1,33 +0,0 @@
defmodule Frenzy.Filter do
use Ecto.Schema
import Ecto.Changeset
schema "filters" do
field :mode, :string
field :score, :integer
belongs_to :feed, Frenzy.Feed
has_many :rules, Frenzy.Rule, on_delete: :delete_all
timestamps()
end
@type t() :: %__MODULE__{
__meta__: Ecto.Schema.Metadata.t(),
id: integer() | nil,
mode: String.t(),
score: integer(),
feed: Frenzy.Feed.t() | Ecto.Association.NotLoaded.t(),
rules: [Frenzy.Rule.t()] | Ecto.Association.NotLoaded.t(),
inserted_at: NaiveDateTime.t(),
updated_at: NaiveDateTime.t()
}
@doc false
def changeset(filter, attrs) do
filter
|> cast(attrs, [:mode, :score])
|> cast_assoc(:rules)
|> validate_required([:mode, :score])
end
end

View File

@ -1,34 +0,0 @@
defmodule Frenzy.Rule do
use Ecto.Schema
import Ecto.Changeset
schema "rules" do
field :mode, :string
field :property, :string
field :param, :string
field :weight, :integer
belongs_to :filter, Frenzy.Filter
timestamps()
end
@type t() :: %__MODULE__{
__meta__: Ecto.Schema.Metadata.t(),
id: integer() | nil,
mode: String.t(),
property: String.t(),
param: String.t(),
weight: integer(),
filter: Frenzy.Filter.t() | Ecto.Association.NotLoaded.t(),
inserted_at: NaiveDateTime.t(),
updated_at: NaiveDateTime.t()
}
@doc false
def changeset(rule, attrs) do
rule
|> cast(attrs, [:mode, :property, :param, :weight])
|> validate_required([:mode, :property, :param, :weight])
end
end

View File

@ -40,7 +40,7 @@ defmodule Frenzy.UpdateFeeds do
defp update_feeds() do defp update_feeds() do
Logger.info("Updating all feeds") Logger.info("Updating all feeds")
Repo.all(from Feed, preload: [:filter]) Repo.all(from(Feed))
|> Enum.map(&update_feed/1) |> Enum.map(&update_feed/1)
prune_old_items() prune_old_items()
@ -102,7 +102,9 @@ defmodule Frenzy.UpdateFeeds do
update_feed(feed) update_feed(feed)
{:ok, %HTTPoison.Response{} = response} -> {:ok, %HTTPoison.Response{} = response} ->
Logger.error("Couldn't load RSS feed #{feed.feed_url}, got unexpected response: #{inspect(response)}") Logger.error(
"Couldn't load RSS feed #{feed.feed_url}, got unexpected response: #{inspect(response)}"
)
{:error, %HTTPoison.Error{reason: reason}} -> {:error, %HTTPoison.Error{reason: reason}} ->
Logger.error("Couldn't load RSS feed #{feed.feed_url}: #{inspect(reason)}") Logger.error("Couldn't load RSS feed #{feed.feed_url}: #{inspect(reason)}")
@ -119,7 +121,7 @@ defmodule Frenzy.UpdateFeeds do
Repo.update(changeset) Repo.update(changeset)
feed = Repo.preload(feed, items: [], filter: [:rules]) feed = Repo.preload(feed, [:items])
Enum.map(rss.items, fn entry -> Enum.map(rss.items, fn entry ->
# todo: use Repo.exists for this # todo: use Repo.exists for this

View File

@ -1,6 +1,6 @@
defmodule FrenzyWeb.FeedController do defmodule FrenzyWeb.FeedController do
use FrenzyWeb, :controller use FrenzyWeb, :controller
alias Frenzy.{Repo, Group, Feed, Filter, Item} alias Frenzy.{Repo, Group, Feed, Item}
alias FrenzyWeb.Router.Helpers, as: Routes alias FrenzyWeb.Router.Helpers, as: Routes
alias FrenzyWeb.Endpoint alias FrenzyWeb.Endpoint
import Ecto.Query import Ecto.Query
@ -26,7 +26,7 @@ defmodule FrenzyWeb.FeedController do
defp user_owns_feed(conn, _opts), do: conn defp user_owns_feed(conn, _opts), do: conn
def show(conn, %{"id" => id}) do def show(conn, %{"id" => id}) do
feed = conn.assigns[:feed] |> Repo.preload(:filter) feed = conn.assigns[:feed]
items = Repo.all(from Item, where: [feed_id: ^id, tombstone: false], order_by: [desc: :date]) items = Repo.all(from Item, where: [feed_id: ^id, tombstone: false], order_by: [desc: :date])
render(conn, "show.html", %{ render(conn, "show.html", %{
@ -45,11 +45,7 @@ defmodule FrenzyWeb.FeedController do
if String.length(feed_url) > 0 do if String.length(feed_url) > 0 do
changeset = changeset =
Ecto.build_assoc(group, :feeds, %{ Ecto.build_assoc(group, :feeds, %{
feed_url: feed_url, feed_url: feed_url
filter: %Filter{
mode: "reject",
score: 0
}
}) })
{:ok, feed} = Repo.insert(changeset) {:ok, feed} = Repo.insert(changeset)
@ -71,7 +67,7 @@ defmodule FrenzyWeb.FeedController do
end end
def edit(conn, _params) do def edit(conn, _params) do
feed = conn.assigns[:feed] |> Repo.preload([:filter, :pipeline_stages]) feed = conn.assigns[:feed] |> Repo.preload([:pipeline_stages])
stages = Enum.sort_by(feed.pipeline_stages, fn stage -> stage.index end) stages = Enum.sort_by(feed.pipeline_stages, fn stage -> stage.index end)
render(conn, "edit.html", %{ render(conn, "edit.html", %{