shadowfacts
/
frenzy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update things, fix warnings

This commit is contained in:
Shadowfacts 2023-06-25 13:45:59 -07:00
parent 4e484cac92
commit 53cbe0a7e9
22 changed files with 45 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
config/config.exs
View File

@ -5,7 +5,7 @@
# is restricted to this project.
# General application configuration
use Mix.Config
import Config
config :frenzy,
ecto_repos: [Frenzy.Repo]
@ -33,6 +33,8 @@ config :logger, truncate: :infinity
config :frenzy, sentry_enabled: false
config :frenzy, external_readability: false
config :frenzy, env: config_env()
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
import_config "#{config_env()}.exs"

2
config/dev.exs
View File

@ -1,4 +1,4 @@
use Mix.Config
import Config
# For development, we disable any cache and enable
# debugging and code reloading.

2
config/prod.exs
View File

@ -1,4 +1,4 @@
use Mix.Config
import Config
# For production, don't forget to configure the url host
# to something meaningful, Phoenix uses this information

2
config/test.exs
View File

@ -1,4 +1,4 @@
use Mix.Config
import Config
# We don't run a server during test. If one is required,
# you can enable the server option below.

7
lib/frenzy/builtin_extractor.ex
View File

@ -3,7 +3,7 @@ defmodule Frenzy.BuiltinExtractor do
alias Frenzy.Network
require Logger
@external_url Application.get_env(:frenzy, :external_readability_url)
@external_url Application.compile_env(:frenzy, :external_readability_url)
def start_link(state) do
GenServer.start_link(__MODULE__, :ok, state)
@ -28,7 +28,7 @@ defmodule Frenzy.BuiltinExtractor do
true
_ ->
Logger.warn("Could not reach external readability for healthcheck, disabling")
Logger.warning("Could not reach external readability for healthcheck, disabling")
false
end
else
@ -49,7 +49,8 @@ defmodule Frenzy.BuiltinExtractor do
case Network.http_post(uri, html, headers: [{"content-type", "text/html"}]) do
{:ok, %Tesla.Env{status: 200, body: body}} ->
{:reply, Floki.parse(body), state}
{:ok, doc} = Floki.parse_document(body)
{:reply, doc, state}
{:ok, %Tesla.Env{status: status}} ->
Logger.error("External readability failed, got HTTP #{status}")

4
lib/frenzy/pipeline/conditional_stage.ex
View File

@ -17,7 +17,7 @@ defmodule Frenzy.Pipeline.ConditionalStage do
@impl Stage
def apply(opts, item_params) do
Logger.warn("Received invalid conditional opts: #{inspect(opts)}")
Logger.warning("Received invalid conditional opts: #{inspect(opts)}")
{:ok, item_params}
end
@ -30,7 +30,7 @@ defmodule Frenzy.Pipeline.ConditionalStage do
end
defp test_condition(condition, _item_params) do
Logger.warn("Received invalid condition: #{inspect(condition)}")
Logger.warning("Received invalid condition: #{inspect(condition)}")
false
end

5
lib/frenzy/pipeline/extractor/ars_technica.ex
View File

@ -34,7 +34,8 @@ defmodule Frenzy.Pipeline.Extractor.ArsTechnica do
if next_page_url != nil do
with body when not is_nil(body) <- fetch_page(next_page_url),
next_pages when is_list(next_pages) <- get_pages_from_tree(Floki.parse(body)) do
{:ok, doc} <- Floki.parse_document(body),
next_pages when is_list(next_pages) <- get_pages_from_tree(doc) do
[content] ++ next_pages
else
_ ->
@ -63,7 +64,7 @@ defmodule Frenzy.Pipeline.Extractor.ArsTechnica do
body
{:ok, %Tesla.Env{status: code}} ->
Logger.warn("Unexpected HTTP code #{code} getting Ars Technica page #{url}")
Logger.warning("Unexpected HTTP code #{code} getting Ars Technica page #{url}")
nil
{:error, reason} ->

2
lib/frenzy/pipeline/extractor/macstories.ex
View File

@ -15,7 +15,7 @@ defmodule Frenzy.Pipeline.Extractor.MacStories do
# some images have full size links, strip those out
|> Floki.filter_out("a.view-full-size")
# rewrite non-standard images captions to <figure>/<figcaption>
|> Floki.map(&rewrite_element/1)
|> Floki.find_and_update("div, p", &rewrite_element/1)
{:ok, content_elem}

2
lib/frenzy/pipeline/extractor/util.ex
View File

@ -5,7 +5,7 @@ defmodule Frenzy.Pipeline.Extractor.Util do
"""
@spec strip_wp_lazy_loading(Floki.html_tree()) :: Floki.html_tree()
def strip_wp_lazy_loading(tree) do
Floki.map(tree, fn
Floki.find_and_update(tree, "img.jetpack-lazy-image", fn
{"img", attrs} = el ->
class = Enum.find(attrs, fn {k, _} -> k == "class" end)

2
lib/frenzy/pipeline/filter_engine.ex
View File

@ -133,7 +133,7 @@ defmodule Frenzy.Pipeline.FilterEngine do
defp get_property(%{content: content, content_type: "text/html"}, "content") do
content
|> Floki.parse()
|> Floki.parse_fragment()
|> Floki.text()
end

2
lib/frenzy/pipeline/gemini_scrape_stage.ex
View File

@ -8,7 +8,7 @@ defmodule Frenzy.Pipeline.GeminiScrapeStage do
def apply(opts, %{url: url} = item_params) do
case get_content(url, opts) do
{:error, reason} ->
Logger.warn("Unable to get Gemini content for #{url}: #{reason}")
Logger.warning("Unable to get Gemini content for #{url}: #{reason}")
{:ok, item_params}
{content, content_type} ->

11
lib/frenzy/pipeline/scrape_stage.ex
View File

@ -12,7 +12,7 @@ defmodule Frenzy.Pipeline.ScrapeStage do
{:ok, %{item_params | content: content}}
{:error, reason} ->
Logger.warn("Unable to get article content for #{url}: #{reason}")
Logger.warning("Unable to get article content for #{url}: #{reason}")
{:ok, item_params}
end
end
@ -92,7 +92,7 @@ defmodule Frenzy.Pipeline.ScrapeStage do
{:ok, BuiltinExtractor.article(url, body)}
module_name ->
html_tree = Floki.parse(body)
{:ok, html_tree} = Floki.parse_document(body)
try do
apply(String.to_existing_atom("Elixir." <> module_name), :extract, [html_tree])
@ -134,7 +134,12 @@ defmodule Frenzy.Pipeline.ScrapeStage do
value -> value
end
html = Floki.map(html, rewrite_image_urls(convert_to_data_uris, URI.parse(url)))
html =
Floki.find_and_update(
html,
"img",
rewrite_image_urls(convert_to_data_uris, URI.parse(url))
)
{:ok, Floki.raw_html(html)}

2
lib/frenzy/task/create_item.ex
View File

@ -109,7 +109,7 @@ defmodule Frenzy.Task.CreateItem do
{:error, changeset} ->
with [feed_id: {_, list}] <- changeset.errors,
true <- {:constraint_name, "items_feed_guid_index"} in list do
Logger.warn("Did not insert duplicate item for #{item_params.guid}")
Logger.warning("Did not insert duplicate item for #{item_params.guid}")
else
_ ->
Logger.error("Error inserting item #{item_params.guid}")

2
lib/frenzy/task/fetch_favicon.ex
View File

@ -59,7 +59,7 @@ defmodule Frenzy.Task.FetchFavicon do
@spec extract_favicon_url(page_url :: String.t(), body :: term()) :: String.t()
defp extract_favicon_url(page_url, body) do
html_tree = Floki.parse(body)
{:ok, html_tree} = Floki.parse_document(body)
case Floki.find(html_tree, "link[rel=icon]") do
[] ->

4
lib/frenzy/update_feeds.ex
View File

@ -77,7 +77,7 @@ defmodule Frenzy.UpdateFeeds do
update_feed(feed)
rescue
error ->
Logger.warn(
Logger.warning(
"Encountered error updating feed #{feed.id} #{feed.feed_url}: #{inspect(error)}"
)
@ -119,7 +119,7 @@ defmodule Frenzy.UpdateFeeds do
update_feed_http(feed, retry_count)
%URI{scheme: scheme} ->
Logger.warn("Unhandled scheme for feed: #{scheme}")
Logger.warning("Unhandled scheme for feed: #{scheme}")
end
end

8
lib/frenzy_web/live/configure_stage/conditional_stage_live.html.heex
View File

@ -1,5 +1,5 @@
<div id={@id}>
<%= if Mix.env == :dev do %>
<%= if Application.fetch_env!(:frenzy, :env) == :dev do %>
<pre><%= Jason.encode!(@opts, pretty: true) %></pre>
<% end %>
@ -19,7 +19,7 @@
<div class="card-body">
<% component = component_module(@opts["stage"]) %>
<%= unless is_nil(component) do %>
<%= live_component(@socket, component, index: @index, id: "#{@id}-conditional", stage: @stage, keypath: @keypath ++ ["opts"]) %>
<%= live_component(component, index: @index, id: "#{@id}-conditional", stage: @stage, keypath: @keypath ++ ["opts"]) %>
<% end %>
</div>
</div>
@ -45,7 +45,7 @@
</div>
<% end %>
<%= live_component @socket, FrenzyWeb.FilterLive, id: "##{@id}-filter", parent_id: @id, filter: @opts["condition"] %>
<%= live_component FrenzyWeb.FilterLive, id: "##{@id}-filter", parent_id: @id, filter: @opts["condition"] %>
</div>
</div>
<% else %>
@ -61,7 +61,7 @@
</div>
</div>
<div class="card-body">
<%= live_component @socket, FrenzyWeb.FilterRuleLive, id: "##{@id}-rule", parent_id: @id, rule: @opts["condition"], index: :no_index %>
<%= live_component FrenzyWeb.FilterRuleLive, id: "##{@id}-rule", parent_id: @id, rule: @opts["condition"], index: :no_index %>
</div>
</div>
<% end %>

4
lib/frenzy_web/live/configure_stage/filter_stage_live.html.heex
View File

@ -1,6 +1,6 @@
<div id={@id}>
<%= if Mix.env == :dev do %>
<%= if Application.fetch_env!(:frenzy, :env) == :dev do %>
<pre><%= Jason.encode!(@opts, pretty: true) %></pre>
<% end %>
<%= live_component @socket, FrenzyWeb.FilterLive, id: "#{@id}-filter", parent_id: @id, filter: @opts %>
<%= live_component FrenzyWeb.FilterLive, id: "#{@id}-filter", parent_id: @id, filter: @opts %>
</div>

2
lib/frenzy_web/live/configure_stage/scrape_stage_live.html.heex
View File

@ -1,5 +1,5 @@
<div id={@id}>
<%= if Mix.env == :dev do %>
<%= if Application.fetch_env!(:frenzy, :env) == :dev do %>
<pre><%= Jason.encode!(@opts, pretty: true) %></pre>
<% end %>
<%= form_for @opts, "#", [as: :opts, phx_change: :update_stage, phx_target: @myself], fn f -> %>

4
lib/frenzy_web/live/edit_pipeline_live.ex
View File

@ -106,13 +106,13 @@ defmodule FrenzyWeb.EditPipelineLive do
end
end
def component_for(socket, %{"module_name" => module} = stage, index) do
def component_for(%{"module_name" => module} = stage, index) do
case component_module(module) do
nil ->
nil
component ->
live_component(socket, component,
live_component(component,
index: index,
id: "stage-#{index}",
stage: stage,

2
lib/frenzy_web/live/edit_pipeline_live.html.heex
View File

@ -19,7 +19,7 @@
</div>
</div>
<div class="card-body">
<%= component_for(@socket, stage, index) %>
<%= component_for(stage, index) %>
</div>
</div>
<% end %>

4
lib/frenzy_web/live/filter/filter_live.html.heex
View File

@ -27,11 +27,11 @@
</div>
<div class="card-body">
<%= live_component @socket, FrenzyWeb.FilterRuleLive, id: "#{@id}-rule-#{index}", parent_id: @parent_id, rule: rule, index: index %>
<%= live_component FrenzyWeb.FilterRuleLive, id: "#{@id}-rule-#{index}", parent_id: @parent_id, rule: rule, index: index %>
</div>
</div>
<% end %>
<%= form_for :rule, "#", [class: "mt-2", phx_submit: :add_rule, phx_target: "##{@parent_id}"], fn f -> %>
<%= form_for :rule, "#", [class: "mt-2", phx_submit: :add_rule, phx_target: "##{@parent_id}"], fn _f -> %>
<%= submit "Add Rule", class: "btn btn-primary" %>
<% end %>
</div>

2
mix.lock
View File

@ -51,7 +51,7 @@
"saxy": {:hex, :saxy, "0.6.0", "cdb2f2fcd8133d1f3f8b0cf6a131ee1ca348dca613de266e9a239db850c4a093", [:mix], [], "hexpm"},
"sentry": {:hex, :sentry, "8.0.5", "5ca922b9238a50c7258b52f47364b2d545beda5e436c7a43965b34577f1ef61f", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.3", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "4972839fdbf52e886d7b3e694c8adf421f764f2fa79036b88fb4742049bd4b7c"},
"socket": {:hex, :socket, "0.3.13", "98a2ab20ce17f95fb512c5cadddba32b57273e0d2dba2d2e5f976c5969d0c632", [:mix], [], "hexpm", "f82ea9833ef49dde272e6568ab8aac657a636acb4cf44a7de8a935acb8957c2e"},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
"telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"},
"tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"},
"timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"},