Add basic LiveView pipeline editor, scrape stage config editing

lib/frenzy/pipeline/conditional_stage.ex

@@ -37,8 +37,8 @@ defmodule Frenzy.Pipeline.ConditionalStage do
   @impl Stage
   def validate_opts(opts) do
     cond do
-      not (Map.has_key?(opts, "stage") and is_binary(opts["stage"]) and
+      not (Map.has_key?(opts, "stage") and
-               module_exists(opts["stage"])) ->
+               ((is_binary(opts["stage"]) and module_exists(opts["stage"])) or opts["stage"] == "")) ->
         {:error, "stage must be a string containg a module that exists"}
 
       not (Map.has_key?(opts, "opts") and is_map(opts["opts"])) ->
@@ -47,6 +47,9 @@ defmodule Frenzy.Pipeline.ConditionalStage do
       not (Map.has_key?(opts, "condition") and is_map(opts["condition"])) ->
         {:error, "condition must be a map"}
 
+      opts["stage"] == "" ->
+        {:ok, opts}
+
       true ->
         with {:ok, stage_opts} <-
                apply(String.to_existing_atom("Elixir." <> opts["stage"]), :validate_opts, [
@@ -66,6 +69,9 @@ defmodule Frenzy.Pipeline.ConditionalStage do
     end
   end
 
+  @impl Stage
+  def default_opts(), do: %{"stage" => "", "opts" => %{}, "condition" => %{}}
+
   defp module_exists(module_name) do
     try do
       String.to_existing_atom("Elixir." <> module_name)

lib/frenzy/pipeline/filter_stage.ex

@@ -17,4 +17,7 @@ defmodule Frenzy.Pipeline.FilterStage do
   def validate_opts(opts) do
     FilterEngine.validate_filter(opts)
   end
+
+  @impl Stage
+  def default_opts(), do: %{"mode" => "reject", "score" => 1, "rules" => []}
 end

lib/frenzy/pipeline/scrape_stage.ex

@@ -60,6 +60,9 @@ defmodule Frenzy.Pipeline.ScrapeStage do
   @impl Stage
   def validate_opts(_), do: {:error, "options must be a map"}
 
+  @impl Stage
+  def default_opts(), do: %{}
+
   @spec get_article_content(String.t(), map()) :: {:ok, String.t()} | {:error, String.t()}
   defp get_article_content(url, opts) when is_binary(url) and url != "" do
     Logger.debug("Getting article from #{url}")

lib/frenzy/pipeline/stage.ex

@@ -2,4 +2,6 @@ defmodule Frenzy.Pipeline.Stage do
   @callback apply(Map.t(), Map.t()) :: {:ok, Map.t()} | :tombstone | {:error, String.t()}
 
   @callback validate_opts(Map.t()) :: {:ok, Map.t()} | {:error, String.t()}
+
+  @callback default_opts() :: Map.t()
 end

lib/frenzy_web/live/configure_stage/scrape_stage_live.ex

@@ -0,0 +1,51 @@
+defmodule FrenzyWeb.ConfigureStage.ScrapeStageLive do
+  use FrenzyWeb, :live_component
+  alias Frenzy.JSONSchema
+
+  @extractors [
+                {"Builtin", "builtin"},
+                {"beckyhansmeyer.com", Frenzy.Pipeline.Extractor.BeckyHansmeyer},
+                {"daringfireball.net", Frenzy.Pipeline.Extractor.DaringFireball},
+                {"ericasadun.com", Frenzy.Pipeline.Extractor.EricaSadun},
+                {"finertech.com", Frenzy.Pipeline.Extractor.FinerTech},
+                {"macstories.net", Frenzy.Pipeline.Extractor.MacStories},
+                {"om.co", Frenzy.Pipeline.Extractor.OmMalik},
+                {"whatever.scalzi.com", Frenzy.Pipeline.Extractor.WhateverScalzi}
+              ]
+              |> Enum.map(fn {pretty_name, module} ->
+                {
+                  pretty_name,
+                  case module do
+                    "builtin" -> "builtin"
+                    module -> module |> to_string() |> String.slice(7..-1)
+                  end
+                }
+              end)
+
+  @schema %{
+    "convert_to_data_uris" => :boolean,
+    "extractor" => :string
+  }
+
+  @impl true
+  def mount(socket) do
+    {:ok, assign(socket, extractors: @extractors)}
+  end
+
+  @impl true
+  def handle_event(
+        "update_stage",
+        %{"opts" => %{"convert_to_data_uris" => convert_to_data_uris, "extractor" => extractor}},
+        socket
+      ) do
+    convert_to_data_uris = String.downcase(convert_to_data_uris) == "true"
+
+    new_opts =
+      socket.assigns.opts
+      |> Map.put("convert_to_data_uris", convert_to_data_uris)
+      |> Map.put("extractor", extractor)
+
+    send(self(), {:update_stage_opts, socket.assigns.index, new_opts})
+    {:noreply, socket}
+  end
+end

lib/frenzy_web/live/configure_stage/scrape_stage_live.html.leex

@@ -0,0 +1,15 @@
+<div id="<%= @id %>">
+  <pre><%= Jason.encode!(@opts, pretty: true) %></pre>
+  <%= f = form_for @opts, "#", [as: :opts, phx_change: :update_stage, phx_target: @myself] %>
+    <div class="form-group form-check">
+      <%= checkbox f, :convert_to_data_uris, id: "#{@id}-convert_to_data_uris", class: "form-check-input" %>
+      <label class="form-check-label" for="<%= @id %>-convert_to_data_uris">Convert Images to Embedded Data URIs</label>
+    </div>
+    <div class="form-group row">
+      <label class="col-sm-2 col-form-label" for="<%= @id %>-extractor">Extractor</label>
+      <div class="col-sm-10">
+        <%= select f, :extractor, @extractors, id: "#{@id}-extractor", class: "custom-select" %>
+      </div>
+    </div>
+  </form>
+</div>

lib/frenzy_web/live/edit_pipeline_live.ex

@@ -0,0 +1,80 @@
+defmodule FrenzyWeb.EditPipelineLive do
+  use FrenzyWeb, :live_view
+  use Phoenix.HTML
+  alias Frenzy.{Repo, Pipeline}
+
+  @impl true
+  def mount(%{"id" => pipeline_id}, _session, socket) do
+    pipeline = Repo.get(Pipeline, pipeline_id)
+
+    {:ok,
+     assign(socket,
+       pipeline: pipeline,
+       stages: [
+         {"Filter Stage", "Frenzy.Pipeline.FilterStage"},
+         {"Scrape Stage", "Frenzy.Pipeline.ScrapeStage"},
+         {"Conditional Stage", "Frenzy.Pipeline.ConditionalStage"}
+       ]
+     )}
+  end
+
+  @impl true
+  def handle_event("add_stage", %{"stage" => %{"module_name" => module}}, socket) do
+    pipeline = socket.assigns[:pipeline]
+    default_options = apply(String.to_existing_atom("Elixir." <> module), :default_opts, [])
+
+    changeset =
+      Pipeline.changeset(pipeline, %{
+        stages:
+          pipeline.stages ++
+            [%{"module_name" => module, "options" => default_options}]
+      })
+
+    {:ok, pipeline} = Repo.update(changeset)
+    {:noreply, assign(socket, pipeline: pipeline)}
+  end
+
+  def handle_event("delete_stage", %{"index" => index}, socket) do
+    pipeline = socket.assigns.pipeline
+    index = String.to_integer(index)
+
+    changeset =
+      Pipeline.changeset(pipeline, %{
+        stages: List.delete_at(pipeline.stages, index)
+      })
+
+    {:ok, pipeline} = Repo.update(changeset)
+    {:noreply, assign(socket, pipeline: pipeline)}
+  end
+
+  @impl true
+  def handle_info({:update_stage_opts, index, new_opts}, socket) do
+    pipeline = socket.assigns.pipeline
+    stages = pipeline.stages
+    stage = Enum.at(stages, index)
+    new_stage = Map.put(stage, "options", new_opts)
+    new_stages = List.replace_at(stages, index, new_stage)
+    changeset = Pipeline.changeset(pipeline, %{stages: new_stages})
+    {:ok, pipeline} = Repo.update(changeset)
+    {:noreply, assign(socket, pipeline: pipeline)}
+  end
+
+  def component_for(socket, %{"module_name" => module, "options" => opts}, index) do
+    component =
+      case module do
+        "Frenzy.Pipeline.ScrapeStage" ->
+          FrenzyWeb.ConfigureStage.ScrapeStageLive
+
+        _ ->
+          nil
+      end
+
+    case component do
+      nil ->
+        nil
+
+      component ->
+        live_component(socket, component, index: index, id: "stage-#{index}", opts: opts)
+    end
+  end
+end

lib/frenzy_web/live/edit_pipeline_live.html.leex

@@ -0,0 +1,31 @@
+<h1>Edit <%= @pipeline.name %></h1>
+
+<%= for {stage, index} <- Enum.with_index(@pipeline.stages) do %>
+  <div class="card mt-4">
+    <div class="card-header container-fluid">
+      <div class="row">
+        <div class="col">
+          <h4 class="m-0"><%= stage["module_name"] %></h4>
+        </div>
+        <div class="col text-right">
+          <button phx-click="delete_stage" phx-value-index="<%= index %>">Delete</button>
+        </div>
+      </div>
+    </div>
+    <div class="card-body">
+      <%# <pre><%= Jason.encode!(stage["options"], pretty: true) %1></pre> %>
+      <%# <%= live_component(@socket, ) %1> %>
+      <%= component_for(@socket, stage, index) %>
+    </div>
+  </div>
+<% end %>
+
+<%= f = form_for :stage, "#", [class: "mt-4", phx_submit: :add_stage] %>
+  <div class="form-group row">
+    <label class="col-sm-2 col-form-label" for="module_name">Module</label>
+    <div class="col-sm-10">
+      <%= select f, :module_name, @stages, class: "custom-select" %>
+    </div>
+  </div>
+  <%= submit "Add Stage", class: "btn btn-primary" %>
+</form>

lib/frenzy_web/map_form_data.ex

@@ -0,0 +1,47 @@
+defimpl Phoenix.HTML.FormData, for: Map do
+  alias Phoenix.HTML.Form
+
+  def to_form(map, opts) do
+    {name, opts} = Keyword.pop(opts, :as)
+    {errors, opts} = Keyword.pop(opts, :errors)
+
+    %Form{
+      source: map,
+      impl: __MODULE__,
+      id: name,
+      name: name,
+      params: map,
+      data: %{},
+      errors: errors,
+      options: opts
+    }
+  end
+
+  def to_form(_data, _form, _field, _opts) do
+    raise "Unimplemented"
+  end
+
+  def input_value(_data, %Form{data: data, params: params}, field) do
+    key =
+      case field do
+        field when is_atom(field) -> Atom.to_string(field)
+        field -> field
+      end
+
+    case Map.fetch(params, key) do
+      {:ok, value} ->
+        value
+
+      :error ->
+        Map.get(data, key)
+    end
+  end
+
+  def input_validations(_data, _form, _field) do
+    raise "Unimplemented"
+  end
+
+  def input_type(_data, _form, _field) do
+    raise "Unimplemented"
+  end
+end

lib/frenzy_web/router.ex

@@ -54,7 +54,8 @@ defmodule FrenzyWeb.Router do
     resources "/feeds", FeedController, except: [:index, :new]
     post "/feeds/:id/refresh", FeedController, :refresh
 
-    resources "/pipelines", PipelineController
+    resources "/pipelines", PipelineController, except: [:edit]
+    live "/pipelines/:id/edit", EditPipelineLive
 
     resources "/items", ItemController, only: [:show]
     post "/items/:id/read", ItemController, :read

lib/frenzy_web/templates/pipeline/show.html.eex

@@ -1,7 +1,7 @@
 <h1><%= @pipeline.name %></h1>
 <h2>Stages: <%= @pipeline.stages |> Enum.count() %></h2>
 
-<a href="<%= Routes.pipeline_path(@conn, :edit, @pipeline.id) %>" class="btn btn-primary">Edit Pipeline</a>
+<a href="<%= Routes.live_path(@conn, FrenzyWeb.EditPipelineLive, @pipeline.id) %>" class="btn btn-primary">Edit Pipeline</a>
 <%= form_tag Routes.pipeline_path(@conn, :delete, @pipeline.id), method: :delete do %>
   <%= submit "Delete Pipeline", class: "btn btn-danger mt-2" %>
 <% end %>

mix.exs

@@ -53,7 +53,8 @@ defmodule Frenzy.MixProject do
       {:dialyxir, "~> 1.0.0-rc.6", only: :dev, runtime: false},
       {:xml_builder, "~> 2.1.1"},
       {:floki, "~> 0.23"},
-      {:phoenix_live_view, "~> 0.13.3"}
+      {:phoenix_live_view,
+       git: "https://github.com/phoenixframework/phoenix_live_view", branch: "master"}
     ]
   end
 

mix.lock

@@ -33,7 +33,7 @@
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"},
   "phoenix_html": {:hex, :phoenix_html, "2.14.2", "b8a3899a72050f3f48a36430da507dd99caf0ac2d06c77529b1646964f3d563e", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "58061c8dfd25da5df1ea0ca47c972f161beb6c875cd293917045b92ffe1bf617"},
   "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.2.0", "3bb31a9fbd40ffe8652e60c8660dffd72dd231efcdf49b744fb75b9ef7db5dd2", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "b61c87a23fabcd85ff369fb9c041d9c01787d210322749026f56a69a914b7503"},
-  "phoenix_live_view": {:hex, :phoenix_live_view, "0.13.3", "2186c55cc7c54ca45b97c6f28cfd267d1c61b5f205f3c83533704cd991bdfdec", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.4.17 or ~> 1.5.2", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm", "c6309a7da2e779cb9cdf2fb603d75f38f49ef324bedc7a81825998bd1744ff8a"},
+  "phoenix_live_view": {:git, "https://github.com/phoenixframework/phoenix_live_view", "a622fed5e496561eb05a7fce5423d238c2597142", [branch: "master"]},
   "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.0.0", "a1ae76717bb168cdeb10ec9d92d1480fec99e3080f011402c0a2d68d47395ffb", [:mix], [], "hexpm", "c52d948c4f261577b9c6fa804be91884b381a7f8f18450c5045975435350f771"},
   "plug": {:hex, :plug, "1.10.2", "0079345cfdf9e17da3858b83eb46bc54beb91554c587b96438f55c1477af5a86", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7898d0eb4767efb3b925fd7f9d1870d15e66e9c33b89c58d8d2ad89aa75ab3c1"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.2.2", "7a09aa5d10e79b92d332a288f21cc49406b1b994cbda0fde76160e7f4cc890ea", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e82364b29311dbad3753d588febd7e5ef05062cd6697d8c231e0e007adab3727"},