shadowfacts
/
frenzy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add basic LiveView pipeline editor, scrape stage config editing

This commit is contained in:
Shadowfacts 2020-06-08 22:49:45 -04:00
parent a66990782e
commit fc2b8f6036
Signed by: shadowfacts
GPG Key ID: 94A5AB95422746E5
13 changed files with 246 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lib/frenzy/pipeline/conditional_stage.ex
View File

@ -37,8 +37,8 @@ defmodule Frenzy.Pipeline.ConditionalStage do
@impl Stage
def validate_opts(opts) do
cond do
not (Map.has_key?(opts, "stage") and is_binary(opts["stage"]) and
module_exists(opts["stage"])) ->
not (Map.has_key?(opts, "stage") and
((is_binary(opts["stage"]) and module_exists(opts["stage"])) or opts["stage"] == "")) ->
{:error, "stage must be a string containg a module that exists"}
not (Map.has_key?(opts, "opts") and is_map(opts["opts"])) ->
@ -47,6 +47,9 @@ defmodule Frenzy.Pipeline.ConditionalStage do
not (Map.has_key?(opts, "condition") and is_map(opts["condition"])) ->
{:error, "condition must be a map"}
opts["stage"] == "" ->
{:ok, opts}
true ->
with {:ok, stage_opts} <-
apply(String.to_existing_atom("Elixir." <> opts["stage"]), :validate_opts, [
@ -66,6 +69,9 @@ defmodule Frenzy.Pipeline.ConditionalStage do
end
end
@impl Stage
def default_opts(), do: %{"stage" => "", "opts" => %{}, "condition" => %{}}
defp module_exists(module_name) do
try do
String.to_existing_atom("Elixir." <> module_name)

3
lib/frenzy/pipeline/filter_stage.ex
View File

@ -17,4 +17,7 @@ defmodule Frenzy.Pipeline.FilterStage do
def validate_opts(opts) do
FilterEngine.validate_filter(opts)
end
@impl Stage
def default_opts(), do: %{"mode" => "reject", "score" => 1, "rules" => []}
end

3
lib/frenzy/pipeline/scrape_stage.ex
View File

@ -60,6 +60,9 @@ defmodule Frenzy.Pipeline.ScrapeStage do
@impl Stage
def validate_opts(_), do: {:error, "options must be a map"}
@impl Stage
def default_opts(), do: %{}
@spec get_article_content(String.t(), map()) :: {:ok, String.t()} | {:error, String.t()}
defp get_article_content(url, opts) when is_binary(url) and url != "" do
Logger.debug("Getting article from #{url}")

2
lib/frenzy/pipeline/stage.ex
View File

@ -2,4 +2,6 @@ defmodule Frenzy.Pipeline.Stage do
@callback apply(Map.t(), Map.t()) :: {:ok, Map.t()} | :tombstone | {:error, String.t()}
@callback validate_opts(Map.t()) :: {:ok, Map.t()} | {:error, String.t()}
@callback default_opts() :: Map.t()
end

51
lib/frenzy_web/live/configure_stage/scrape_stage_live.ex Normal file
View File

@ -0,0 +1,51 @@
defmodule FrenzyWeb.ConfigureStage.ScrapeStageLive do
use FrenzyWeb, :live_component
alias Frenzy.JSONSchema
@extractors [
{"Builtin", "builtin"},
{"beckyhansmeyer.com", Frenzy.Pipeline.Extractor.BeckyHansmeyer},
{"daringfireball.net", Frenzy.Pipeline.Extractor.DaringFireball},
{"ericasadun.com", Frenzy.Pipeline.Extractor.EricaSadun},
{"finertech.com", Frenzy.Pipeline.Extractor.FinerTech},
{"macstories.net", Frenzy.Pipeline.Extractor.MacStories},
{"om.co", Frenzy.Pipeline.Extractor.OmMalik},
{"whatever.scalzi.com", Frenzy.Pipeline.Extractor.WhateverScalzi}
]
|> Enum.map(fn {pretty_name, module} ->
{
pretty_name,
case module do
"builtin" -> "builtin"
module -> module |> to_string() |> String.slice(7..-1)
end
}
end)
@schema %{
"convert_to_data_uris" => :boolean,
"extractor" => :string
}
@impl true
def mount(socket) do
{:ok, assign(socket, extractors: @extractors)}
end
@impl true
def handle_event(
"update_stage",
%{"opts" => %{"convert_to_data_uris" => convert_to_data_uris, "extractor" => extractor}},
socket
) do
convert_to_data_uris = String.downcase(convert_to_data_uris) == "true"
new_opts =
socket.assigns.opts
|> Map.put("convert_to_data_uris", convert_to_data_uris)
|> Map.put("extractor", extractor)
send(self(), {:update_stage_opts, socket.assigns.index, new_opts})
{:noreply, socket}
end
end

15
lib/frenzy_web/live/configure_stage/scrape_stage_live.html.leex Normal file
View File

@ -0,0 +1,15 @@
<div id="<%= @id %>">
<pre><%= Jason.encode!(@opts, pretty: true) %></pre>
<%= f = form_for @opts, "#", [as: :opts, phx_change: :update_stage, phx_target: @myself] %>
<div class="form-group form-check">
<%= checkbox f, :convert_to_data_uris, id: "#{@id}-convert_to_data_uris", class: "form-check-input" %>
<label class="form-check-label" for="<%= @id %>-convert_to_data_uris">Convert Images to Embedded Data URIs</label>
</div>
<div class="form-group row">
<label class="col-sm-2 col-form-label" for="<%= @id %>-extractor">Extractor</label>
<div class="col-sm-10">
<%= select f, :extractor, @extractors, id: "#{@id}-extractor", class: "custom-select" %>
</div>
</div>
</form>
</div>

80
lib/frenzy_web/live/edit_pipeline_live.ex Normal file
View File

@ -0,0 +1,80 @@
defmodule FrenzyWeb.EditPipelineLive do
use FrenzyWeb, :live_view
use Phoenix.HTML
alias Frenzy.{Repo, Pipeline}
@impl true
def mount(%{"id" => pipeline_id}, _session, socket) do
pipeline = Repo.get(Pipeline, pipeline_id)
{:ok,
assign(socket,
pipeline: pipeline,
stages: [
{"Filter Stage", "Frenzy.Pipeline.FilterStage"},
{"Scrape Stage", "Frenzy.Pipeline.ScrapeStage"},
{"Conditional Stage", "Frenzy.Pipeline.ConditionalStage"}
]
)}
end
@impl true
def handle_event("add_stage", %{"stage" => %{"module_name" => module}}, socket) do
pipeline = socket.assigns[:pipeline]
default_options = apply(String.to_existing_atom("Elixir." <> module), :default_opts, [])
changeset =
Pipeline.changeset(pipeline, %{
stages:
pipeline.stages ++
[%{"module_name" => module, "options" => default_options}]
})
{:ok, pipeline} = Repo.update(changeset)
{:noreply, assign(socket, pipeline: pipeline)}
end
def handle_event("delete_stage", %{"index" => index}, socket) do
pipeline = socket.assigns.pipeline
index = String.to_integer(index)
changeset =
Pipeline.changeset(pipeline, %{
stages: List.delete_at(pipeline.stages, index)
})
{:ok, pipeline} = Repo.update(changeset)
{:noreply, assign(socket, pipeline: pipeline)}
end
@impl true
def handle_info({:update_stage_opts, index, new_opts}, socket) do
pipeline = socket.assigns.pipeline
stages = pipeline.stages
stage = Enum.at(stages, index)
new_stage = Map.put(stage, "options", new_opts)
new_stages = List.replace_at(stages, index, new_stage)
changeset = Pipeline.changeset(pipeline, %{stages: new_stages})
{:ok, pipeline} = Repo.update(changeset)
{:noreply, assign(socket, pipeline: pipeline)}
end
def component_for(socket, %{"module_name" => module, "options" => opts}, index) do
component =
case module do
"Frenzy.Pipeline.ScrapeStage" ->
FrenzyWeb.ConfigureStage.ScrapeStageLive
_ ->
nil
end
case component do
nil ->
nil
component ->
live_component(socket, component, index: index, id: "stage-#{index}", opts: opts)
end
end
end

31
lib/frenzy_web/live/edit_pipeline_live.html.leex Normal file
View File

@ -0,0 +1,31 @@
<h1>Edit <%= @pipeline.name %></h1>
<%= for {stage, index} <- Enum.with_index(@pipeline.stages) do %>
<div class="card mt-4">
<div class="card-header container-fluid">
<div class="row">
<div class="col">
<h4 class="m-0"><%= stage["module_name"] %></h4>
</div>
<div class="col text-right">
<button phx-click="delete_stage" phx-value-index="<%= index %>">Delete</button>
</div>
</div>
</div>
<div class="card-body">
<%# <pre><%= Jason.encode!(stage["options"], pretty: true) %1></pre> %>
<%# <%= live_component(@socket, ) %1> %>
<%= component_for(@socket, stage, index) %>
</div>
</div>
<% end %>
<%= f = form_for :stage, "#", [class: "mt-4", phx_submit: :add_stage] %>
<div class="form-group row">
<label class="col-sm-2 col-form-label" for="module_name">Module</label>
<div class="col-sm-10">
<%= select f, :module_name, @stages, class: "custom-select" %>
</div>
</div>
<%= submit "Add Stage", class: "btn btn-primary" %>
</form>

47
lib/frenzy_web/map_form_data.ex Normal file
View File

@ -0,0 +1,47 @@
defimpl Phoenix.HTML.FormData, for: Map do
alias Phoenix.HTML.Form
def to_form(map, opts) do
{name, opts} = Keyword.pop(opts, :as)
{errors, opts} = Keyword.pop(opts, :errors)
%Form{
source: map,
impl: __MODULE__,
id: name,
name: name,
params: map,
data: %{},
errors: errors,
options: opts
}
end
def to_form(_data, _form, _field, _opts) do
raise "Unimplemented"
end
def input_value(_data, %Form{data: data, params: params}, field) do
key =
case field do
field when is_atom(field) -> Atom.to_string(field)
field -> field
end
case Map.fetch(params, key) do
{:ok, value} ->
value
:error ->
Map.get(data, key)
end
end
def input_validations(_data, _form, _field) do
raise "Unimplemented"
end
def input_type(_data, _form, _field) do
raise "Unimplemented"
end
end

3
lib/frenzy_web/router.ex
View File

@ -54,7 +54,8 @@ defmodule FrenzyWeb.Router do
resources "/feeds", FeedController, except: [:index, :new]
post "/feeds/:id/refresh", FeedController, :refresh
resources "/pipelines", PipelineController
resources "/pipelines", PipelineController, except: [:edit]
live "/pipelines/:id/edit", EditPipelineLive
resources "/items", ItemController, only: [:show]
post "/items/:id/read", ItemController, :read

2
lib/frenzy_web/templates/pipeline/show.html.eex
View File

@ -1,7 +1,7 @@
<h1><%= @pipeline.name %></h1>
<h2>Stages: <%= @pipeline.stages |> Enum.count() %></h2>
<a href="<%= Routes.pipeline_path(@conn, :edit, @pipeline.id) %>" class="btn btn-primary">Edit Pipeline</a>
<a href="<%= Routes.live_path(@conn, FrenzyWeb.EditPipelineLive, @pipeline.id) %>" class="btn btn-primary">Edit Pipeline</a>
<%= form_tag Routes.pipeline_path(@conn, :delete, @pipeline.id), method: :delete do %>
<%= submit "Delete Pipeline", class: "btn btn-danger mt-2" %>
<% end %>

3
mix.exs
View File

@ -53,7 +53,8 @@ defmodule Frenzy.MixProject do
{:dialyxir, "~> 1.0.0-rc.6", only: :dev, runtime: false},
{:xml_builder, "~> 2.1.1"},
{:floki, "~> 0.23"},
{:phoenix_live_view, "~> 0.13.3"}
{:phoenix_live_view,
git: "https://github.com/phoenixframework/phoenix_live_view", branch: "master"}
]
end

2
mix.lock
View File

@ -33,7 +33,7 @@
"phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"},
"phoenix_html": {:hex, :phoenix_html, "2.14.2", "b8a3899a72050f3f48a36430da507dd99caf0ac2d06c77529b1646964f3d563e", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "58061c8dfd25da5df1ea0ca47c972f161beb6c875cd293917045b92ffe1bf617"},
"phoenix_live_reload": {:hex, :phoenix_live_reload, "1.2.0", "3bb31a9fbd40ffe8652e60c8660dffd72dd231efcdf49b744fb75b9ef7db5dd2", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "b61c87a23fabcd85ff369fb9c041d9c01787d210322749026f56a69a914b7503"},
"phoenix_live_view": {:hex, :phoenix_live_view, "0.13.3", "2186c55cc7c54ca45b97c6f28cfd267d1c61b5f205f3c83533704cd991bdfdec", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.4.17 or ~> 1.5.2", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14", [hex: :phoenix_html, repo: "hexpm", optional: false]}], "hexpm", "c6309a7da2e779cb9cdf2fb603d75f38f49ef324bedc7a81825998bd1744ff8a"},
"phoenix_live_view": {:git, "https://github.com/phoenixframework/phoenix_live_view", "a622fed5e496561eb05a7fce5423d238c2597142", [branch: "master"]},
"phoenix_pubsub": {:hex, :phoenix_pubsub, "2.0.0", "a1ae76717bb168cdeb10ec9d92d1480fec99e3080f011402c0a2d68d47395ffb", [:mix], [], "hexpm", "c52d948c4f261577b9c6fa804be91884b381a7f8f18450c5045975435350f771"},
"plug": {:hex, :plug, "1.10.2", "0079345cfdf9e17da3858b83eb46bc54beb91554c587b96438f55c1477af5a86", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7898d0eb4767efb3b925fd7f9d1870d15e66e9c33b89c58d8d2ad89aa75ab3c1"},
"plug_cowboy": {:hex, :plug_cowboy, "2.2.2", "7a09aa5d10e79b92d332a288f21cc49406b1b994cbda0fde76160e7f4cc890ea", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e82364b29311dbad3753d588febd7e5ef05062cd6697d8c231e0e007adab3727"},