From 33d1cac5e1970cd99aaac0a869922954db844f71 Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Wed, 31 Mar 2021 15:30:17 -0400 Subject: [PATCH] Recover from errors in custom extractors --- lib/frenzy/pipeline/scrape_stage.ex | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/frenzy/pipeline/scrape_stage.ex b/lib/frenzy/pipeline/scrape_stage.ex index 906fd18..e147868 100644 --- a/lib/frenzy/pipeline/scrape_stage.ex +++ b/lib/frenzy/pipeline/scrape_stage.ex @@ -89,7 +89,19 @@ defmodule Frenzy.Pipeline.ScrapeStage do module_name -> html_tree = Floki.parse(body) - apply(String.to_existing_atom("Elixir." <> module_name), :extract, [html_tree]) + + try do + apply(String.to_existing_atom("Elixir." <> module_name), :extract, [html_tree]) + rescue + e -> + Logger.error( + "Encountered error extracting article content from '#{url}' with #{module_name}, falling back to default" + ) + + Logger.error(Exception.format(:error, e, __STACKTRACE__)) + + {:ok, Readability.article(body)} + end end |> case do {:ok, html} ->