diff --git a/lib/frenzy/pipeline/conditional_stage.ex b/lib/frenzy/pipeline/conditional_stage.ex index 8b6e4d3..b60c4d9 100644 --- a/lib/frenzy/pipeline/conditional_stage.ex +++ b/lib/frenzy/pipeline/conditional_stage.ex @@ -37,8 +37,8 @@ defmodule Frenzy.Pipeline.ConditionalStage do @impl Stage def validate_opts(opts) do cond do - not (Map.has_key?(opts, "stage") and is_binary(opts["stage"]) and - module_exists(opts["stage"])) -> + not (Map.has_key?(opts, "stage") and + ((is_binary(opts["stage"]) and module_exists(opts["stage"])) or opts["stage"] == "")) -> {:error, "stage must be a string containg a module that exists"} not (Map.has_key?(opts, "opts") and is_map(opts["opts"])) -> @@ -47,6 +47,9 @@ defmodule Frenzy.Pipeline.ConditionalStage do not (Map.has_key?(opts, "condition") and is_map(opts["condition"])) -> {:error, "condition must be a map"} + opts["stage"] == "" -> + {:ok, opts} + true -> with {:ok, stage_opts} <- apply(String.to_existing_atom("Elixir." <> opts["stage"]), :validate_opts, [ @@ -66,6 +69,9 @@ defmodule Frenzy.Pipeline.ConditionalStage do end end + @impl Stage + def default_opts(), do: %{"stage" => "", "opts" => %{}, "condition" => %{}} + defp module_exists(module_name) do try do String.to_existing_atom("Elixir." <> module_name) diff --git a/lib/frenzy/pipeline/filter_stage.ex b/lib/frenzy/pipeline/filter_stage.ex index e038170..86ba83d 100644 --- a/lib/frenzy/pipeline/filter_stage.ex +++ b/lib/frenzy/pipeline/filter_stage.ex @@ -17,4 +17,7 @@ defmodule Frenzy.Pipeline.FilterStage do def validate_opts(opts) do FilterEngine.validate_filter(opts) end + + @impl Stage + def default_opts(), do: %{"mode" => "reject", "score" => 1, "rules" => []} end diff --git a/lib/frenzy/pipeline/scrape_stage.ex b/lib/frenzy/pipeline/scrape_stage.ex index 0d67371..e072438 100644 --- a/lib/frenzy/pipeline/scrape_stage.ex +++ b/lib/frenzy/pipeline/scrape_stage.ex @@ -60,6 +60,9 @@ defmodule Frenzy.Pipeline.ScrapeStage do @impl Stage def validate_opts(_), do: {:error, "options must be a map"} + @impl Stage + def default_opts(), do: %{} + @spec get_article_content(String.t(), map()) :: {:ok, String.t()} | {:error, String.t()} defp get_article_content(url, opts) when is_binary(url) and url != "" do Logger.debug("Getting article from #{url}") diff --git a/lib/frenzy/pipeline/stage.ex b/lib/frenzy/pipeline/stage.ex index 9eb6067..223e272 100644 --- a/lib/frenzy/pipeline/stage.ex +++ b/lib/frenzy/pipeline/stage.ex @@ -2,4 +2,6 @@ defmodule Frenzy.Pipeline.Stage do @callback apply(Map.t(), Map.t()) :: {:ok, Map.t()} | :tombstone | {:error, String.t()} @callback validate_opts(Map.t()) :: {:ok, Map.t()} | {:error, String.t()} + + @callback default_opts() :: Map.t() end diff --git a/lib/frenzy_web/live/configure_stage/scrape_stage_live.ex b/lib/frenzy_web/live/configure_stage/scrape_stage_live.ex new file mode 100644 index 0000000..a4d4ba6 --- /dev/null +++ b/lib/frenzy_web/live/configure_stage/scrape_stage_live.ex @@ -0,0 +1,51 @@ +defmodule FrenzyWeb.ConfigureStage.ScrapeStageLive do + use FrenzyWeb, :live_component + alias Frenzy.JSONSchema + + @extractors [ + {"Builtin", "builtin"}, + {"beckyhansmeyer.com", Frenzy.Pipeline.Extractor.BeckyHansmeyer}, + {"daringfireball.net", Frenzy.Pipeline.Extractor.DaringFireball}, + {"ericasadun.com", Frenzy.Pipeline.Extractor.EricaSadun}, + {"finertech.com", Frenzy.Pipeline.Extractor.FinerTech}, + {"macstories.net", Frenzy.Pipeline.Extractor.MacStories}, + {"om.co", Frenzy.Pipeline.Extractor.OmMalik}, + {"whatever.scalzi.com", Frenzy.Pipeline.Extractor.WhateverScalzi} + ] + |> Enum.map(fn {pretty_name, module} -> + { + pretty_name, + case module do + "builtin" -> "builtin" + module -> module |> to_string() |> String.slice(7..-1) + end + } + end) + + @schema %{ + "convert_to_data_uris" => :boolean, + "extractor" => :string + } + + @impl true + def mount(socket) do + {:ok, assign(socket, extractors: @extractors)} + end + + @impl true + def handle_event( + "update_stage", + %{"opts" => %{"convert_to_data_uris" => convert_to_data_uris, "extractor" => extractor}}, + socket + ) do + convert_to_data_uris = String.downcase(convert_to_data_uris) == "true" + + new_opts = + socket.assigns.opts + |> Map.put("convert_to_data_uris", convert_to_data_uris) + |> Map.put("extractor", extractor) + + send(self(), {:update_stage_opts, socket.assigns.index, new_opts}) + {:noreply, socket} + end +end diff --git a/lib/frenzy_web/live/configure_stage/scrape_stage_live.html.leex b/lib/frenzy_web/live/configure_stage/scrape_stage_live.html.leex new file mode 100644 index 0000000..72c4eff --- /dev/null +++ b/lib/frenzy_web/live/configure_stage/scrape_stage_live.html.leex @@ -0,0 +1,15 @@ +
+
<%= Jason.encode!(@opts, pretty: true) %>
+ <%= f = form_for @opts, "#", [as: :opts, phx_change: :update_stage, phx_target: @myself] %> +
+ <%= checkbox f, :convert_to_data_uris, id: "#{@id}-convert_to_data_uris", class: "form-check-input" %> + +
+
+ +
+ <%= select f, :extractor, @extractors, id: "#{@id}-extractor", class: "custom-select" %> +
+
+ +
diff --git a/lib/frenzy_web/live/edit_pipeline_live.ex b/lib/frenzy_web/live/edit_pipeline_live.ex new file mode 100644 index 0000000..ae12186 --- /dev/null +++ b/lib/frenzy_web/live/edit_pipeline_live.ex @@ -0,0 +1,80 @@ +defmodule FrenzyWeb.EditPipelineLive do + use FrenzyWeb, :live_view + use Phoenix.HTML + alias Frenzy.{Repo, Pipeline} + + @impl true + def mount(%{"id" => pipeline_id}, _session, socket) do + pipeline = Repo.get(Pipeline, pipeline_id) + + {:ok, + assign(socket, + pipeline: pipeline, + stages: [ + {"Filter Stage", "Frenzy.Pipeline.FilterStage"}, + {"Scrape Stage", "Frenzy.Pipeline.ScrapeStage"}, + {"Conditional Stage", "Frenzy.Pipeline.ConditionalStage"} + ] + )} + end + + @impl true + def handle_event("add_stage", %{"stage" => %{"module_name" => module}}, socket) do + pipeline = socket.assigns[:pipeline] + default_options = apply(String.to_existing_atom("Elixir." <> module), :default_opts, []) + + changeset = + Pipeline.changeset(pipeline, %{ + stages: + pipeline.stages ++ + [%{"module_name" => module, "options" => default_options}] + }) + + {:ok, pipeline} = Repo.update(changeset) + {:noreply, assign(socket, pipeline: pipeline)} + end + + def handle_event("delete_stage", %{"index" => index}, socket) do + pipeline = socket.assigns.pipeline + index = String.to_integer(index) + + changeset = + Pipeline.changeset(pipeline, %{ + stages: List.delete_at(pipeline.stages, index) + }) + + {:ok, pipeline} = Repo.update(changeset) + {:noreply, assign(socket, pipeline: pipeline)} + end + + @impl true + def handle_info({:update_stage_opts, index, new_opts}, socket) do + pipeline = socket.assigns.pipeline + stages = pipeline.stages + stage = Enum.at(stages, index) + new_stage = Map.put(stage, "options", new_opts) + new_stages = List.replace_at(stages, index, new_stage) + changeset = Pipeline.changeset(pipeline, %{stages: new_stages}) + {:ok, pipeline} = Repo.update(changeset) + {:noreply, assign(socket, pipeline: pipeline)} + end + + def component_for(socket, %{"module_name" => module, "options" => opts}, index) do + component = + case module do + "Frenzy.Pipeline.ScrapeStage" -> + FrenzyWeb.ConfigureStage.ScrapeStageLive + + _ -> + nil + end + + case component do + nil -> + nil + + component -> + live_component(socket, component, index: index, id: "stage-#{index}", opts: opts) + end + end +end diff --git a/lib/frenzy_web/live/edit_pipeline_live.html.leex b/lib/frenzy_web/live/edit_pipeline_live.html.leex new file mode 100644 index 0000000..17f1382 --- /dev/null +++ b/lib/frenzy_web/live/edit_pipeline_live.html.leex @@ -0,0 +1,31 @@ +

Edit <%= @pipeline.name %>

+ +<%= for {stage, index} <- Enum.with_index(@pipeline.stages) do %> +
+
+
+
+

<%= stage["module_name"] %>

+
+
+ +
+
+
+
+ <%#
<%= Jason.encode!(stage["options"], pretty: true) %1>
%> + <%# <%= live_component(@socket, ) %1> %> + <%= component_for(@socket, stage, index) %> +
+
+<% end %> + +<%= f = form_for :stage, "#", [class: "mt-4", phx_submit: :add_stage] %> +
+ +
+ <%= select f, :module_name, @stages, class: "custom-select" %> +
+
+ <%= submit "Add Stage", class: "btn btn-primary" %> + diff --git a/lib/frenzy_web/map_form_data.ex b/lib/frenzy_web/map_form_data.ex new file mode 100644 index 0000000..b72edba --- /dev/null +++ b/lib/frenzy_web/map_form_data.ex @@ -0,0 +1,47 @@ +defimpl Phoenix.HTML.FormData, for: Map do + alias Phoenix.HTML.Form + + def to_form(map, opts) do + {name, opts} = Keyword.pop(opts, :as) + {errors, opts} = Keyword.pop(opts, :errors) + + %Form{ + source: map, + impl: __MODULE__, + id: name, + name: name, + params: map, + data: %{}, + errors: errors, + options: opts + } + end + + def to_form(_data, _form, _field, _opts) do + raise "Unimplemented" + end + + def input_value(_data, %Form{data: data, params: params}, field) do + key = + case field do + field when is_atom(field) -> Atom.to_string(field) + field -> field + end + + case Map.fetch(params, key) do + {:ok, value} -> + value + + :error -> + Map.get(data, key) + end + end + + def input_validations(_data, _form, _field) do + raise "Unimplemented" + end + + def input_type(_data, _form, _field) do + raise "Unimplemented" + end +end diff --git a/lib/frenzy_web/router.ex b/lib/frenzy_web/router.ex index 39c161e..43d3ed7 100644 --- a/lib/frenzy_web/router.ex +++ b/lib/frenzy_web/router.ex @@ -54,7 +54,8 @@ defmodule FrenzyWeb.Router do resources "/feeds", FeedController, except: [:index, :new] post "/feeds/:id/refresh", FeedController, :refresh - resources "/pipelines", PipelineController + resources "/pipelines", PipelineController, except: [:edit] + live "/pipelines/:id/edit", EditPipelineLive resources "/items", ItemController, only: [:show] post "/items/:id/read", ItemController, :read diff --git a/lib/frenzy_web/templates/pipeline/show.html.eex b/lib/frenzy_web/templates/pipeline/show.html.eex index 101dd2b..582a73b 100644 --- a/lib/frenzy_web/templates/pipeline/show.html.eex +++ b/lib/frenzy_web/templates/pipeline/show.html.eex @@ -1,7 +1,7 @@

<%= @pipeline.name %>

Stages: <%= @pipeline.stages |> Enum.count() %>

-Edit Pipeline +Edit Pipeline <%= form_tag Routes.pipeline_path(@conn, :delete, @pipeline.id), method: :delete do %> <%= submit "Delete Pipeline", class: "btn btn-danger mt-2" %> <% end %> diff --git a/mix.exs b/mix.exs index 58d3a73..02e3fa5 100644 --- a/mix.exs +++ b/mix.exs @@ -53,7 +53,8 @@ defmodule Frenzy.MixProject do {:dialyxir, "~> 1.0.0-rc.6", only: :dev, runtime: false}, {:xml_builder, "~> 2.1.1"}, {:floki, "~> 0.23"}, - {:phoenix_live_view, "~> 0.13.3"} + {:phoenix_live_view, + git: "https://github.com/phoenixframework/phoenix_live_view", branch: "master"} ] end diff --git a/mix.lock b/mix.lock index a2889ba..3b1d0e2 100644 --- a/mix.lock +++ b/mix.lock @@ -33,7 +33,7 @@ "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"}, "phoenix_html": {:hex, :phoenix_html, "2.14.2", "b8a3899a72050f3f48a36430da507dd99caf0ac2d06c77529b1646964f3d563e", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "58061c8dfd25da5df1ea0ca47c972f161beb6c875cd293917045b92ffe1bf617"}, "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.2.0", "3bb31a9fbd40ffe8652e60c8660dffd72dd231efcdf49b744fb75b9ef7db5dd2", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "b61c87a23fabcd85ff369fb9c041d9c01787d210322749026f56a69a914b7503"}, - "phoenix_live_view": {:hex, :phoenix_live_view, "0.13.3", "2186c55cc7c54ca45b97c6f28cfd267d1c61b5f205f3c83533704cd991bdfdec", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.4.17 or ~> 1.5.2", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm", "c6309a7da2e779cb9cdf2fb603d75f38f49ef324bedc7a81825998bd1744ff8a"}, + "phoenix_live_view": {:git, "https://github.com/phoenixframework/phoenix_live_view", "a622fed5e496561eb05a7fce5423d238c2597142", [branch: "master"]}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.0.0", "a1ae76717bb168cdeb10ec9d92d1480fec99e3080f011402c0a2d68d47395ffb", [:mix], [], "hexpm", "c52d948c4f261577b9c6fa804be91884b381a7f8f18450c5045975435350f771"}, "plug": {:hex, :plug, "1.10.2", "0079345cfdf9e17da3858b83eb46bc54beb91554c587b96438f55c1477af5a86", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7898d0eb4767efb3b925fd7f9d1870d15e66e9c33b89c58d8d2ad89aa75ab3c1"}, "plug_cowboy": {:hex, :plug_cowboy, "2.2.2", "7a09aa5d10e79b92d332a288f21cc49406b1b994cbda0fde76160e7f4cc890ea", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e82364b29311dbad3753d588febd7e5ef05062cd6697d8c231e0e007adab3727"},