From 53cbe0a7e9e87e4e86ae5c7804ecba2e6bf7f555 Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Sun, 25 Jun 2023 13:45:59 -0700 Subject: [PATCH] Update things, fix warnings --- config/config.exs | 6 ++++-- config/dev.exs | 2 +- config/prod.exs | 2 +- config/test.exs | 2 +- lib/frenzy/builtin_extractor.ex | 7 ++++--- lib/frenzy/pipeline/conditional_stage.ex | 4 ++-- lib/frenzy/pipeline/extractor/ars_technica.ex | 5 +++-- lib/frenzy/pipeline/extractor/macstories.ex | 2 +- lib/frenzy/pipeline/extractor/util.ex | 2 +- lib/frenzy/pipeline/filter_engine.ex | 2 +- lib/frenzy/pipeline/gemini_scrape_stage.ex | 2 +- lib/frenzy/pipeline/scrape_stage.ex | 11 ++++++++--- lib/frenzy/task/create_item.ex | 2 +- lib/frenzy/task/fetch_favicon.ex | 2 +- lib/frenzy/update_feeds.ex | 4 ++-- .../configure_stage/conditional_stage_live.html.heex | 8 ++++---- .../live/configure_stage/filter_stage_live.html.heex | 4 ++-- .../live/configure_stage/scrape_stage_live.html.heex | 2 +- lib/frenzy_web/live/edit_pipeline_live.ex | 4 ++-- lib/frenzy_web/live/edit_pipeline_live.html.heex | 2 +- lib/frenzy_web/live/filter/filter_live.html.heex | 4 ++-- mix.lock | 2 +- 22 files changed, 45 insertions(+), 36 deletions(-) diff --git a/config/config.exs b/config/config.exs index 5dc4544..3ccbc82 100644 --- a/config/config.exs +++ b/config/config.exs @@ -5,7 +5,7 @@ # is restricted to this project. # General application configuration -use Mix.Config +import Config config :frenzy, ecto_repos: [Frenzy.Repo] @@ -33,6 +33,8 @@ config :logger, truncate: :infinity config :frenzy, sentry_enabled: false config :frenzy, external_readability: false +config :frenzy, env: config_env() + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. -import_config "#{Mix.env()}.exs" +import_config "#{config_env()}.exs" diff --git a/config/dev.exs b/config/dev.exs index 5bd93c7..35a78b4 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # For development, we disable any cache and enable # debugging and code reloading. diff --git a/config/prod.exs b/config/prod.exs index 41c1600..728e0c6 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # For production, don't forget to configure the url host # to something meaningful, Phoenix uses this information diff --git a/config/test.exs b/config/test.exs index 1d6e4c4..5468565 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config # We don't run a server during test. If one is required, # you can enable the server option below. diff --git a/lib/frenzy/builtin_extractor.ex b/lib/frenzy/builtin_extractor.ex index b77b5f8..c8d284d 100644 --- a/lib/frenzy/builtin_extractor.ex +++ b/lib/frenzy/builtin_extractor.ex @@ -3,7 +3,7 @@ defmodule Frenzy.BuiltinExtractor do alias Frenzy.Network require Logger - @external_url Application.get_env(:frenzy, :external_readability_url) + @external_url Application.compile_env(:frenzy, :external_readability_url) def start_link(state) do GenServer.start_link(__MODULE__, :ok, state) @@ -28,7 +28,7 @@ defmodule Frenzy.BuiltinExtractor do true _ -> - Logger.warn("Could not reach external readability for healthcheck, disabling") + Logger.warning("Could not reach external readability for healthcheck, disabling") false end else @@ -49,7 +49,8 @@ defmodule Frenzy.BuiltinExtractor do case Network.http_post(uri, html, headers: [{"content-type", "text/html"}]) do {:ok, %Tesla.Env{status: 200, body: body}} -> - {:reply, Floki.parse(body), state} + {:ok, doc} = Floki.parse_document(body) + {:reply, doc, state} {:ok, %Tesla.Env{status: status}} -> Logger.error("External readability failed, got HTTP #{status}") diff --git a/lib/frenzy/pipeline/conditional_stage.ex b/lib/frenzy/pipeline/conditional_stage.ex index b60c4d9..11c72d4 100644 --- a/lib/frenzy/pipeline/conditional_stage.ex +++ b/lib/frenzy/pipeline/conditional_stage.ex @@ -17,7 +17,7 @@ defmodule Frenzy.Pipeline.ConditionalStage do @impl Stage def apply(opts, item_params) do - Logger.warn("Received invalid conditional opts: #{inspect(opts)}") + Logger.warning("Received invalid conditional opts: #{inspect(opts)}") {:ok, item_params} end @@ -30,7 +30,7 @@ defmodule Frenzy.Pipeline.ConditionalStage do end defp test_condition(condition, _item_params) do - Logger.warn("Received invalid condition: #{inspect(condition)}") + Logger.warning("Received invalid condition: #{inspect(condition)}") false end diff --git a/lib/frenzy/pipeline/extractor/ars_technica.ex b/lib/frenzy/pipeline/extractor/ars_technica.ex index f696560..b89e5c9 100644 --- a/lib/frenzy/pipeline/extractor/ars_technica.ex +++ b/lib/frenzy/pipeline/extractor/ars_technica.ex @@ -34,7 +34,8 @@ defmodule Frenzy.Pipeline.Extractor.ArsTechnica do if next_page_url != nil do with body when not is_nil(body) <- fetch_page(next_page_url), - next_pages when is_list(next_pages) <- get_pages_from_tree(Floki.parse(body)) do + {:ok, doc} <- Floki.parse_document(body), + next_pages when is_list(next_pages) <- get_pages_from_tree(doc) do [content] ++ next_pages else _ -> @@ -63,7 +64,7 @@ defmodule Frenzy.Pipeline.Extractor.ArsTechnica do body {:ok, %Tesla.Env{status: code}} -> - Logger.warn("Unexpected HTTP code #{code} getting Ars Technica page #{url}") + Logger.warning("Unexpected HTTP code #{code} getting Ars Technica page #{url}") nil {:error, reason} -> diff --git a/lib/frenzy/pipeline/extractor/macstories.ex b/lib/frenzy/pipeline/extractor/macstories.ex index f7c8f15..41ff1b5 100644 --- a/lib/frenzy/pipeline/extractor/macstories.ex +++ b/lib/frenzy/pipeline/extractor/macstories.ex @@ -15,7 +15,7 @@ defmodule Frenzy.Pipeline.Extractor.MacStories do # some images have full size links, strip those out |> Floki.filter_out("a.view-full-size") # rewrite non-standard images captions to
/
- |> Floki.map(&rewrite_element/1) + |> Floki.find_and_update("div, p", &rewrite_element/1) {:ok, content_elem} diff --git a/lib/frenzy/pipeline/extractor/util.ex b/lib/frenzy/pipeline/extractor/util.ex index f257a83..f818071 100644 --- a/lib/frenzy/pipeline/extractor/util.ex +++ b/lib/frenzy/pipeline/extractor/util.ex @@ -5,7 +5,7 @@ defmodule Frenzy.Pipeline.Extractor.Util do """ @spec strip_wp_lazy_loading(Floki.html_tree()) :: Floki.html_tree() def strip_wp_lazy_loading(tree) do - Floki.map(tree, fn + Floki.find_and_update(tree, "img.jetpack-lazy-image", fn {"img", attrs} = el -> class = Enum.find(attrs, fn {k, _} -> k == "class" end) diff --git a/lib/frenzy/pipeline/filter_engine.ex b/lib/frenzy/pipeline/filter_engine.ex index 9709f5f..122cc1e 100644 --- a/lib/frenzy/pipeline/filter_engine.ex +++ b/lib/frenzy/pipeline/filter_engine.ex @@ -133,7 +133,7 @@ defmodule Frenzy.Pipeline.FilterEngine do defp get_property(%{content: content, content_type: "text/html"}, "content") do content - |> Floki.parse() + |> Floki.parse_fragment() |> Floki.text() end diff --git a/lib/frenzy/pipeline/gemini_scrape_stage.ex b/lib/frenzy/pipeline/gemini_scrape_stage.ex index e5feb9a..afce982 100644 --- a/lib/frenzy/pipeline/gemini_scrape_stage.ex +++ b/lib/frenzy/pipeline/gemini_scrape_stage.ex @@ -8,7 +8,7 @@ defmodule Frenzy.Pipeline.GeminiScrapeStage do def apply(opts, %{url: url} = item_params) do case get_content(url, opts) do {:error, reason} -> - Logger.warn("Unable to get Gemini content for #{url}: #{reason}") + Logger.warning("Unable to get Gemini content for #{url}: #{reason}") {:ok, item_params} {content, content_type} -> diff --git a/lib/frenzy/pipeline/scrape_stage.ex b/lib/frenzy/pipeline/scrape_stage.ex index f637df2..16581ee 100644 --- a/lib/frenzy/pipeline/scrape_stage.ex +++ b/lib/frenzy/pipeline/scrape_stage.ex @@ -12,7 +12,7 @@ defmodule Frenzy.Pipeline.ScrapeStage do {:ok, %{item_params | content: content}} {:error, reason} -> - Logger.warn("Unable to get article content for #{url}: #{reason}") + Logger.warning("Unable to get article content for #{url}: #{reason}") {:ok, item_params} end end @@ -92,7 +92,7 @@ defmodule Frenzy.Pipeline.ScrapeStage do {:ok, BuiltinExtractor.article(url, body)} module_name -> - html_tree = Floki.parse(body) + {:ok, html_tree} = Floki.parse_document(body) try do apply(String.to_existing_atom("Elixir." <> module_name), :extract, [html_tree]) @@ -134,7 +134,12 @@ defmodule Frenzy.Pipeline.ScrapeStage do value -> value end - html = Floki.map(html, rewrite_image_urls(convert_to_data_uris, URI.parse(url))) + html = + Floki.find_and_update( + html, + "img", + rewrite_image_urls(convert_to_data_uris, URI.parse(url)) + ) {:ok, Floki.raw_html(html)} diff --git a/lib/frenzy/task/create_item.ex b/lib/frenzy/task/create_item.ex index 99f31bc..420a414 100644 --- a/lib/frenzy/task/create_item.ex +++ b/lib/frenzy/task/create_item.ex @@ -109,7 +109,7 @@ defmodule Frenzy.Task.CreateItem do {:error, changeset} -> with [feed_id: {_, list}] <- changeset.errors, true <- {:constraint_name, "items_feed_guid_index"} in list do - Logger.warn("Did not insert duplicate item for #{item_params.guid}") + Logger.warning("Did not insert duplicate item for #{item_params.guid}") else _ -> Logger.error("Error inserting item #{item_params.guid}") diff --git a/lib/frenzy/task/fetch_favicon.ex b/lib/frenzy/task/fetch_favicon.ex index 6327252..576b101 100644 --- a/lib/frenzy/task/fetch_favicon.ex +++ b/lib/frenzy/task/fetch_favicon.ex @@ -59,7 +59,7 @@ defmodule Frenzy.Task.FetchFavicon do @spec extract_favicon_url(page_url :: String.t(), body :: term()) :: String.t() defp extract_favicon_url(page_url, body) do - html_tree = Floki.parse(body) + {:ok, html_tree} = Floki.parse_document(body) case Floki.find(html_tree, "link[rel=icon]") do [] -> diff --git a/lib/frenzy/update_feeds.ex b/lib/frenzy/update_feeds.ex index 69b5373..f3e0c7a 100644 --- a/lib/frenzy/update_feeds.ex +++ b/lib/frenzy/update_feeds.ex @@ -77,7 +77,7 @@ defmodule Frenzy.UpdateFeeds do update_feed(feed) rescue error -> - Logger.warn( + Logger.warning( "Encountered error updating feed #{feed.id} #{feed.feed_url}: #{inspect(error)}" ) @@ -119,7 +119,7 @@ defmodule Frenzy.UpdateFeeds do update_feed_http(feed, retry_count) %URI{scheme: scheme} -> - Logger.warn("Unhandled scheme for feed: #{scheme}") + Logger.warning("Unhandled scheme for feed: #{scheme}") end end diff --git a/lib/frenzy_web/live/configure_stage/conditional_stage_live.html.heex b/lib/frenzy_web/live/configure_stage/conditional_stage_live.html.heex index 815e890..d92e5b3 100644 --- a/lib/frenzy_web/live/configure_stage/conditional_stage_live.html.heex +++ b/lib/frenzy_web/live/configure_stage/conditional_stage_live.html.heex @@ -1,5 +1,5 @@
- <%= if Mix.env == :dev do %> + <%= if Application.fetch_env!(:frenzy, :env) == :dev do %> 
<%= Jason.encode!(@opts, pretty: true) %>
<% end %> @@ -19,7 +19,7 @@
<% component = component_module(@opts["stage"]) %> <%= unless is_nil(component) do %> - <%= live_component(@socket, component, index: @index, id: "#{@id}-conditional", stage: @stage, keypath: @keypath ++ ["opts"]) %> + <%= live_component(component, index: @index, id: "#{@id}-conditional", stage: @stage, keypath: @keypath ++ ["opts"]) %> <% end %>
@@ -45,7 +45,7 @@ <% end %> - <%= live_component @socket, FrenzyWeb.FilterLive, id: "##{@id}-filter", parent_id: @id, filter: @opts["condition"] %> + <%= live_component FrenzyWeb.FilterLive, id: "##{@id}-filter", parent_id: @id, filter: @opts["condition"] %> <% else %> @@ -61,7 +61,7 @@
- <%= live_component @socket, FrenzyWeb.FilterRuleLive, id: "##{@id}-rule", parent_id: @id, rule: @opts["condition"], index: :no_index %> + <%= live_component FrenzyWeb.FilterRuleLive, id: "##{@id}-rule", parent_id: @id, rule: @opts["condition"], index: :no_index %>
<% end %> diff --git a/lib/frenzy_web/live/configure_stage/filter_stage_live.html.heex b/lib/frenzy_web/live/configure_stage/filter_stage_live.html.heex index 26566c4..9a4a116 100644 --- a/lib/frenzy_web/live/configure_stage/filter_stage_live.html.heex +++ b/lib/frenzy_web/live/configure_stage/filter_stage_live.html.heex @@ -1,6 +1,6 @@
- <%= if Mix.env == :dev do %> + <%= if Application.fetch_env!(:frenzy, :env) == :dev do %> 
<%= Jason.encode!(@opts, pretty: true) %>
<% end %> - <%= live_component @socket, FrenzyWeb.FilterLive, id: "#{@id}-filter", parent_id: @id, filter: @opts %> + <%= live_component FrenzyWeb.FilterLive, id: "#{@id}-filter", parent_id: @id, filter: @opts %>
diff --git a/lib/frenzy_web/live/configure_stage/scrape_stage_live.html.heex b/lib/frenzy_web/live/configure_stage/scrape_stage_live.html.heex index b9f48bb..91e552a 100644 --- a/lib/frenzy_web/live/configure_stage/scrape_stage_live.html.heex +++ b/lib/frenzy_web/live/configure_stage/scrape_stage_live.html.heex @@ -1,5 +1,5 @@
- <%= if Mix.env == :dev do %> + <%= if Application.fetch_env!(:frenzy, :env) == :dev do %> 
<%= Jason.encode!(@opts, pretty: true) %>
<% end %> <%= form_for @opts, "#", [as: :opts, phx_change: :update_stage, phx_target: @myself], fn f -> %> diff --git a/lib/frenzy_web/live/edit_pipeline_live.ex b/lib/frenzy_web/live/edit_pipeline_live.ex index 758dba6..93c1bab 100644 --- a/lib/frenzy_web/live/edit_pipeline_live.ex +++ b/lib/frenzy_web/live/edit_pipeline_live.ex @@ -106,13 +106,13 @@ defmodule FrenzyWeb.EditPipelineLive do end end - def component_for(socket, %{"module_name" => module} = stage, index) do + def component_for(%{"module_name" => module} = stage, index) do case component_module(module) do nil -> nil component -> - live_component(socket, component, + live_component(component, index: index, id: "stage-#{index}", stage: stage, diff --git a/lib/frenzy_web/live/edit_pipeline_live.html.heex b/lib/frenzy_web/live/edit_pipeline_live.html.heex index 566c5cf..26c781c 100644 --- a/lib/frenzy_web/live/edit_pipeline_live.html.heex +++ b/lib/frenzy_web/live/edit_pipeline_live.html.heex @@ -19,7 +19,7 @@
- <%= component_for(@socket, stage, index) %> + <%= component_for(stage, index) %>
<% end %> diff --git a/lib/frenzy_web/live/filter/filter_live.html.heex b/lib/frenzy_web/live/filter/filter_live.html.heex index e1ebffe..c76ab1c 100644 --- a/lib/frenzy_web/live/filter/filter_live.html.heex +++ b/lib/frenzy_web/live/filter/filter_live.html.heex @@ -27,11 +27,11 @@
- <%= live_component @socket, FrenzyWeb.FilterRuleLive, id: "#{@id}-rule-#{index}", parent_id: @parent_id, rule: rule, index: index %> + <%= live_component FrenzyWeb.FilterRuleLive, id: "#{@id}-rule-#{index}", parent_id: @parent_id, rule: rule, index: index %>
<% end %> - <%= form_for :rule, "#", [class: "mt-2", phx_submit: :add_rule, phx_target: "##{@parent_id}"], fn f -> %> + <%= form_for :rule, "#", [class: "mt-2", phx_submit: :add_rule, phx_target: "##{@parent_id}"], fn _f -> %> <%= submit "Add Rule", class: "btn btn-primary" %> <% end %> diff --git a/mix.lock b/mix.lock index 02df266..5042d22 100644 --- a/mix.lock +++ b/mix.lock @@ -51,7 +51,7 @@ "saxy": {:hex, :saxy, "0.6.0", "cdb2f2fcd8133d1f3f8b0cf6a131ee1ca348dca613de266e9a239db850c4a093", [:mix], [], "hexpm"}, "sentry": {:hex, :sentry, "8.0.5", "5ca922b9238a50c7258b52f47364b2d545beda5e436c7a43965b34577f1ef61f", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.3", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "4972839fdbf52e886d7b3e694c8adf421f764f2fa79036b88fb4742049bd4b7c"}, "socket": {:hex, :socket, "0.3.13", "98a2ab20ce17f95fb512c5cadddba32b57273e0d2dba2d2e5f976c5969d0c632", [:mix], [], "hexpm", "f82ea9833ef49dde272e6568ab8aac657a636acb4cf44a7de8a935acb8957c2e"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "f354efb2400dd7a80fd9eb6c8419068c4f632da4ac47f3d8822d6e33f08bc852"},