shadowfacts
/
frenzy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Switch to Mojito for HTTP requests

This commit is contained in:
Shadowfacts 2020-09-11 19:15:19 -04:00
parent 533eae8680
commit 1beff21fc5
Signed by: shadowfacts
GPG Key ID: 94A5AB95422746E5
7 changed files with 31 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
lib/frenzy/network.ex
View File

@ -3,18 +3,18 @@ defmodule Frenzy.Network do
@http_redirect_codes [301, 302]
@spec http_get(String.t(), Keyword.t()) :: {:ok, HTTPoison.Response.t()} | {:error, term()}
def http_get(url, opts \\ []) do
case HTTPoison.get(url, opts) do
{:ok, %HTTPoison.Response{status_code: 200} = response} ->
@spec http_get(String.t()) :: {:ok, Mojito.Response.t()} | {:error, term()}
def http_get(url) do
case Mojito.request(:get, url, [], "", transport_opts: [cacertfile: CAStore.file_path()]) do
{:ok, %Mojito.Response{status_code: 200} = response} ->
{:ok, response}
{:ok, %HTTPoison.Response{status_code: status_code, headers: headers}}
{:ok, %Mojito.Response{status_code: status_code, headers: headers}}
when status_code in @http_redirect_codes ->
headers
|> Enum.find(fn {name, _value} -> name == "Location" end)
|> Enum.find(fn {name, _value} -> name == "location" end)
|> case do
{"Location", new_url} ->
{"location", new_url} ->
new_url =
case URI.parse(new_url) do
%URI{host: nil, path: path} ->
@ -26,23 +26,23 @@ defmodule Frenzy.Network do
end
Logger.debug("Got 301 redirect from #{url} to #{new_url}")
http_get(new_url, opts)
http_get(new_url)
_ ->
{:error, "Missing Location header for redirect"}
end
{:ok, %HTTPoison.Response{status_code: 403}} ->
{:ok, %Mojito.Response{status_code: 403}} ->
{:error, "403 Forbidden"}
{:ok, %HTTPoison.Response{status_code: 404}} ->
{:ok, %Mojito.Response{status_code: 404}} ->
{:error, "404 Not Found"}
{:ok, %HTTPoison.Response{status_code: status_code}} ->
{:ok, %Mojito.Response{status_code: status_code}} ->
{:error, "HTTP #{status_code}"}
{:error, %HTTPoison.Error{reason: reason}} ->
{:error, reason}
{:error, error} ->
{:error, error}
end
end

6
lib/frenzy/pipeline/scrape_stage.ex
View File

@ -80,9 +80,9 @@ defmodule Frenzy.Pipeline.ScrapeStage do
defp get_article_content(_url, _opts), do: {:error, "URL must be a non-empty string"}
@spec handle_response(String.t(), HTTPoison.Response.t(), map()) ::
@spec handle_response(String.t(), Mojito.Response.t(), map()) ::
{:ok, String.t()} | {:error, String.t()}
defp handle_response(url, %HTTPoison.Response{body: body}, opts) do
defp handle_response(url, %Mojito.Response{body: body}, opts) do
case opts["extractor"] do
"builtin" ->
{:ok, Readability.article(body)}
@ -143,7 +143,7 @@ defmodule Frenzy.Pipeline.ScrapeStage do
absolute_url = URI.merge(site_uri, src) |> to_string()
case Network.http_get(absolute_url) do
{:ok, %HTTPoison.Response{body: body, headers: headers}} ->
{:ok, %Mojito.Response{body: body, headers: headers}} ->
{"Content-Type", content_type} =
Enum.find(headers, fn {header, _value} -> header == "Content-Type" end)

12
lib/frenzy/task/fetch_favicon.ex
View File

@ -46,13 +46,9 @@ defmodule Frenzy.Task.FetchFavicon do
defp fetch_favicon_url_from_webpage(url) when is_binary(url) do
case Network.http_get(url) do
{:ok, %HTTPoison.Response{body: body, status_code: code}} when code in 200..299 ->
{:ok, %Mojito.Response{body: body, status_code: code}} when code in 200..299 ->
extract_favicon_url(url, body)
{:ok, %HTTPoison.Response{status_code: code}} ->
Logger.debug("Unhandled HTTP code #{code} for '#{url}'")
nil
{:error, reason} ->
Logger.debug("Error fetching webpage for favicon: #{inspect(reason)}")
nil
@ -113,13 +109,9 @@ defmodule Frenzy.Task.FetchFavicon do
Logger.debug("Fetching favicon from: '#{favicon_url}'")
case Network.http_get(favicon_url) do
{:ok, %HTTPoison.Response{body: body, status_code: code}} when code in 200..299 ->
{:ok, %Mojito.Response{body: body, status_code: code}} when code in 200..299 ->
{:ok, "data:image/png;base64,#{Base.encode64(body)}"}
{:ok, %HTTPoison.Response{status_code: code}} ->
Logger.debug("Unhandled HTTP code #{code} for '#{favicon_url}'")
:error
{:error, reason} ->
Logger.debug("Error fetching favicon: #{inspect(reason)}")
:error

4
lib/frenzy/update_feeds.ex
View File

@ -104,14 +104,14 @@ defmodule Frenzy.UpdateFeeds do
defp update_feed_http(feed) do
case Network.http_get(feed.feed_url) do
{:ok,
%HTTPoison.Response{
%Mojito.Response{
status_code: 200,
body: body,
headers: headers
}} ->
{_, content_type} =
headers
|> Enum.find(fn {k, _v} -> k == "Content-Type" end)
|> Enum.find(fn {k, _v} -> k == "content-type" end)
content_type =
content_type

8
mix.exs
View File

@ -20,7 +20,7 @@ defmodule Frenzy.MixProject do
def application do
[
mod: {Frenzy.Application, []},
extra_applications: [:logger, :runtime_tools, :httpoison, :readability]
extra_applications: [:logger, :runtime_tools, :readability]
]
end
@ -43,12 +43,12 @@ defmodule Frenzy.MixProject do
{:gettext, "~> 0.11"},
{:jason, "~> 1.0"},
{:plug_cowboy, "~> 2.3"},
{:httpoison, "~> 1.6.2"},
{:hackney, "~> 1.16"},
{:mojito, "~> 0.7.3"},
{:feed_parser,
git: "https://git.shadowfacts.net/shadowfacts/feed_parser.git", branch: "master"},
{:timex, "~> 3.6"},
{:readability, git: "https://github.com/shadowfacts/readability.git", branch: "master"},
{:readability,
git: "https://git.shadowfacts.net/shadowfacts/readability.git", branch: "master"},
{:bcrypt_elixir, "~> 2.0"},
{:dialyxir, "~> 1.0.0-rc.6", only: :dev, runtime: false},
{:xml_builder, "~> 2.1.1"},

8
mix.lock
View File

@ -1,6 +1,7 @@
%{
"basic_auth": {:hex, :basic_auth, "2.2.4", "d8c748237870dd1df3bc5c0f1ab4f1fad6270c75472d7e62b19302ec59e92a79", [:mix], [{:plug, "~> 0.14 or ~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
"bcrypt_elixir": {:hex, :bcrypt_elixir, "2.0.1", "1061e2114aaac554c12e5c1e4608bf4aadaca839f30d1b85224272facd5e6427", [:make, :mix], [{:comeonin, "~> 5.1", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "64f174c76ea5edfcc471dfb7762280a20e29fe446baa02dc75c7d14251581e93"},
"castore": {:hex, :castore, "0.1.7", "1ca19eee705cde48c9e809e37fdd0730510752cc397745e550f6065a56a701e9", [:mix], [], "hexpm", "a2ae2c13d40e9c308387f1aceb14786dca019ebc2a11484fb2a9f797ea0aa0d8"},
"certifi": {:hex, :certifi, "2.5.2", "b7cfeae9d2ed395695dd8201c57a2d019c0c43ecaf8b8bcb9320b40d6662f340", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "3b3b5f36493004ac3455966991eaf6e768ce9884693d9968055aeeeb1e575040"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"comeonin": {:hex, :comeonin, "5.1.1", "0abd6bae41acc01c369bb3eafe46399f301bf4e1bacebafdb89252bbb8a1a32d", [:mix], [], "hexpm", "b77aef9eb7ec7a4c01cc3d0683332796052ab71067d858d5dacde967427de0a3"},
@ -15,20 +16,20 @@
"elixir_make": {:hex, :elixir_make, "0.5.2", "96a28c79f5b8d34879cd95ebc04d2a0d678cfbbd3e74c43cb63a76adf0ee8054", [:mix], [], "hexpm", "382eeea8e02dfe6c468f6729b6cf20fe5b14390671d38c7363e59621c7ab4efc"},
"erlex": {:hex, :erlex, "0.2.4", "23791959df45fe8f01f388c6f7eb733cc361668cbeedd801bf491c55a029917b", [:mix], [], "hexpm", "4a12ebc7cd8f24f2d0fce93d279fa34eb5068e0e885bb841d558c4d83c52c439"},
"feed_parser": {:git, "https://git.shadowfacts.net/shadowfacts/feed_parser.git", "8c42d4587328698e8d29d2ad562e478abb146f75", [branch: "master"]},
"fiet": {:git, "https://github.com/shadowfacts/fiet.git", "bf117bc30a6355a189d05a562127cfaf9e0187ae", [branch: "master"]},
"file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm", "0d50da6b04c58e101a3793b1600f9a03b86e3a8057b192ac1766013d35706fa6"},
"floki": {:hex, :floki, "0.23.0", "956ab6dba828c96e732454809fb0bd8d43ce0979b75f34de6322e73d4c917829", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "e680b5ef0b61ce02faa7137db8d1714903a5552be4c89fb57293b8770e7f49c2"},
"gemini": {:git, "https://git.shadowfacts.net/shadowfacts/gemini-ex.git", "322c233e18b2868c1fdc4ac14c3ededdd5c458f0", [branch: "main"]},
"gettext": {:hex, :gettext, "0.17.0", "abe21542c831887a2b16f4c94556db9c421ab301aee417b7c4fbde7fbdbe01ec", [:mix], [], "hexpm", "e0b8598e802676c81e66b061a2148c37c03886b24a3ca86a1f98ed40693b94b3"},
"hackney": {:hex, :hackney, "1.16.0", "5096ac8e823e3a441477b2d187e30dd3fff1a82991a806b2003845ce72ce2d84", [:rebar3], [{:certifi, "2.5.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.1", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.0", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.6", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3bf0bebbd5d3092a3543b783bf065165fa5d3ad4b899b836810e513064134e18"},
"html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm", "3e3d7156a272950373ce5a4018b1490bea26676f8d6a7d409f6fac8568b8cb9a"},
"httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"},
"idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"},
"jason": {:hex, :jason, "1.2.1", "12b22825e22f468c02eb3e4b9985f3d0cb8dc40b9bd704730efa11abd2708c44", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b659b8571deedf60f79c5a608e15414085fa141344e2716fbd6988a084b5f993"},
"metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
"mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"},
"mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
"mint": {:hex, :mint, "1.1.0", "1fd0189edd9e3ffdbd7fcd8bc3835902b987a63ec6c4fd1aa8c2a56e2165f252", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bfd316c3789340b682d5679a8116bcf2112e332447bdc20c1d62909ee45f48d"},
"mochiweb": {:hex, :mochiweb, "2.18.0", "eb55f1db3e6e960fac4e6db4e2db9ec3602cc9f30b86cd1481d56545c3145d2e", [:rebar3], [], "hexpm"},
"mojito": {:hex, :mojito, "0.7.3", "7356f3b7697d79520a243b48cf0bf8bd1152b2e9cdb6ff7cf22cd0769f32dd40", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "433479d8ef1c882fafe864ac6d7b08249f321fc46bdcc8db78691bc1ddcf234a"},
"parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"},
"phoenix": {:hex, :phoenix, "1.5.3", "bfe0404e48ea03dfe17f141eff34e1e058a23f15f109885bbdcf62be303b49ff", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.13", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.1.2 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8e16febeb9640d8b33895a691a56481464b82836d338bb3a23125cd7b6157c25"},
"phoenix_ecto": {:hex, :phoenix_ecto, "4.1.0", "a044d0756d0464c5a541b4a0bf4bcaf89bffcaf92468862408290682c73ae50d", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "c5e666a341ff104d0399d8f0e4ff094559b2fde13a5985d4cb5023b2c2ac558b"},
@ -40,9 +41,10 @@
"plug_cowboy": {:hex, :plug_cowboy, "2.3.0", "149a50e05cb73c12aad6506a371cd75750c0b19a32f81866e1a323dda9e0e99d", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bc595a1870cef13f9c1e03df56d96804db7f702175e4ccacdb8fc75c02a7b97e"},
"plug_crypto": {:hex, :plug_crypto, "1.1.2", "bdd187572cc26dbd95b87136290425f2b580a116d3fb1f564216918c9730d227", [:mix], [], "hexpm", "6b8b608f895b6ffcfad49c37c7883e8df98ae19c6a28113b02aa1e9c5b22d6b5"},
"poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"},
"poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
"postgrex": {:hex, :postgrex, "0.15.5", "aec40306a622d459b01bff890fa42f1430dac61593b122754144ad9033a2152f", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "ed90c81e1525f65a2ba2279dbcebf030d6d13328daa2f8088b9661eb9143af7f"},
"ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},
"readability": {:git, "https://github.com/shadowfacts/readability.git", "71fa17caaf8103ef213e2c7dde4b447a48669122", [branch: "master"]},
"readability": {:git, "https://git.shadowfacts.net/shadowfacts/readability.git", "1538ca2a8c3c3b6b041c9e86b560f368c4547896", [branch: "master"]},
"saxy": {:hex, :saxy, "0.6.0", "cdb2f2fcd8133d1f3f8b0cf6a131ee1ca348dca613de266e9a239db850c4a093", [:mix], [], "hexpm"},
"socket": {:hex, :socket, "0.3.13", "98a2ab20ce17f95fb512c5cadddba32b57273e0d2dba2d2e5f976c5969d0c632", [:mix], [], "hexpm", "f82ea9833ef49dde272e6568ab8aac657a636acb4cf44a7de8a935acb8957c2e"},
"ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},

3
test/support/conn_case.ex
View File

@ -18,7 +18,8 @@ defmodule FrenzyWeb.ConnCase do
using do
quote do
# Import conveniences for testing with connections
use Phoenix.ConnTest
import Plug.Conn
import Phoenix.ConnTest
alias FrenzyWeb.Router.Helpers, as: Routes
# The default endpoint for testing