From 2086a31537d46bc872ef6347a0e2410e1689dde3 Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Sun, 26 Jan 2020 21:46:49 -0500 Subject: [PATCH] Update HTTPoison, improve handling for URIs with no scheme --- lib/frenzy/task/fetch_favicon.ex | 33 ++++++++++++++++++++++++-------- mix.exs | 2 +- mix.lock | 6 +++--- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/lib/frenzy/task/fetch_favicon.ex b/lib/frenzy/task/fetch_favicon.ex index 8e77b3f..0a9cbaf 100644 --- a/lib/frenzy/task/fetch_favicon.ex +++ b/lib/frenzy/task/fetch_favicon.ex @@ -101,15 +101,11 @@ defmodule Frenzy.Task.FetchFavicon do end defp fetch_favicon_data(favicon_url, site_url) when is_binary(favicon_url) do + # handle relative URIs, set default scheme if not provided absolute_url = - case URI.parse(favicon_url) do - %URI{host: nil, path: path} -> - # relative path - %URI{URI.parse(site_url) | path: path} |> URI.to_string() - - _ -> - favicon_url - end + favicon_url + |> URI.parse() + |> resolve_uri(site_url) case HTTP.get(absolute_url) do {:ok, %HTTPoison.Response{body: body}} -> @@ -122,6 +118,27 @@ defmodule Frenzy.Task.FetchFavicon do defp fetch_favicon_data(_, _), do: {:error, "No or invalid href for link"} + defp resolve_uri(%URI{host: nil, path: path}, site_url) do + %URI{URI.parse(site_url) | path: path} + |> resolve_uri(site_url) + end + + defp resolve_uri(%URI{scheme: nil} = uri, site_url) do + scheme = + case URI.parse(site_url) do + %URI{scheme: scheme} when scheme != "" -> + scheme + + _ -> + "https" + end + + %URI{uri | scheme: scheme} + |> resolve_uri(site_url) + end + + defp resolve_uri(uri, _), do: uri + # from https://github.com/elixir-plug/plug/blob/v1.8.3/lib/plug/request_id.ex#L60 defp generate_task_id() do binary = << diff --git a/mix.exs b/mix.exs index 49b556c..e0d806b 100644 --- a/mix.exs +++ b/mix.exs @@ -43,7 +43,7 @@ defmodule Frenzy.MixProject do {:gettext, "~> 0.11"}, {:jason, "~> 1.0"}, {:plug_cowboy, "~> 2.0"}, - {:httpoison, "~> 1.4"}, + {:httpoison, "~> 1.6.2"}, {:feed_parser, git: "https://git.shadowfacts.net/shadowfacts/feed_parser.git", branch: "master"}, {:timex, "~> 3.6"}, diff --git a/mix.lock b/mix.lock index b986128..45b4490 100644 --- a/mix.lock +++ b/mix.lock @@ -19,9 +19,9 @@ "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"}, "floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm"}, "gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm"}, - "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, + "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, - "httpoison": {:hex, :httpoison, "1.4.0", "e0b3c2ad6fa573134e42194d13e925acfa8f89d138bc621ffb7b1989e6d22e73", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "idna": {:hex, :idna, "6.0.0", "689c46cbcdf3524c44d5f3dde8001f364cd7608a99556d8fbd8239a5798d4c10", [:rebar3], [{:unicode_util_compat, "0.4.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, @@ -42,7 +42,7 @@ "ranch": {:hex, :ranch, "1.7.0", "9583f47160ca62af7f8d5db11454068eaa32b56eeadf984d4f46e61a076df5f2", [:rebar3], [], "hexpm"}, "readability": {:git, "https://github.com/shadowfacts/readability.git", "71fa17caaf8103ef213e2c7dde4b447a48669122", [branch: "master"]}, "saxy": {:hex, :saxy, "0.6.0", "cdb2f2fcd8133d1f3f8b0cf6a131ee1ca348dca613de266e9a239db850c4a093", [:mix], [], "hexpm"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, + "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.5", "6eaf7ad16cb568bb01753dbbd7a95ff8b91c7979482b95f38443fe2c8852a79b", [:make, :mix, :rebar3], [], "hexpm"}, "telemetry": {:hex, :telemetry, "0.2.0", "5b40caa3efe4deb30fb12d7cd8ed4f556f6d6bd15c374c2366772161311ce377", [:mix], [], "hexpm"}, "timex": {:hex, :timex, "3.6.1", "efdf56d0e67a6b956cc57774353b0329c8ab7726766a11547e529357ffdc1d56", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5 or ~> 1.0.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "1.0.1", "f6027a331af7d837471248e62733c6ebee86a72e57c613aa071ebb1f750fc71a", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},