shadowfacts
/
frenzy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: shadowfacts:2dda6d7f46f3fe86b6fb6ba00835181d244abc22
Branches Tags
shadowfacts:main
shadowfacts:master
..
compare: shadowfacts:fec640a37a0fdffb73577725f9976bf154cad435
Branches Tags
shadowfacts:main
shadowfacts:master

3 Commits
2dda6d7f46 ... fec640a37a

Author SHA1 Message Date
Shadowfacts fec640a37a Improve The Verge extractor 2023-07-12 22:11:31 -07:00
Shadowfacts b0089083db Fix inline script tags not being removed 2023-07-12 20:50:17 -07:00
Shadowfacts 54c6a1ca0a Remove extraneous <header> 2023-07-12 20:46:25 -07:00
3 changed files with 59 additions and 4 deletions
Show Stats Expand all files Collapse all files

57
lib/frenzy/pipeline/extractor/verge.ex
View File

@ -16,8 +16,10 @@ defmodule Frenzy.Pipeline.Extractor.TheVerge do
html_tree html_tree
|> Floki.find("article#content > div:not(.duet--article--lede)") |> Floki.find("article#content > div:not(.duet--article--lede)")
|> Floki.filter_out( |> Floki.filter_out(
".duet--layout--rail, .duet--article--article-pullquote, .duet--article--comments-join-the-conversation, .duet--recirculation--related-list, .duet--article--comments-button" ".hidden, .duet--layout--rail, .duet--article--article-pullquote, .duet--article--comments-join-the-conversation, .duet--recirculation--related-list, .duet--article--comments-button, .duet--article--share-buttons"
) )
|> Readability.Helper.remove_attrs("style")
|> Floki.traverse_and_update(&rewrite/1)
{:ok, image ++ content} {:ok, image ++ content}
end end
@ -54,4 +56,57 @@ defmodule Frenzy.Pipeline.Extractor.TheVerge do
[] []
end end
end end
defp rewrite({tag, attrs, children} = el) do
cond do
is_empty_gif(el) ->
nil
is_gallery(el) ->
images =
Floki.find(children, ":not(noscript) > img")
|> Enum.map(fn el ->
[src] = Floki.attribute(el, "src")
src
end)
{
"div",
[
{"style",
"display: flex; flex-direction: row; overflow-x: auto; scroll-snap-type: x mandatory;;"}
],
Enum.map(images, fn src ->
{
"img",
[
{"src", src},
{"loading", "lazy"},
{"style", "max-width: 85%; max-height: 30vh; scroll-snap-align: start;"}
],
[]
}
end)
}
true ->
el
end
end
defp rewrite(other), do: other
defp is_gallery(el) do
case Floki.attribute(el, "class") do
[classes] -> String.contains?(classes, "duet--article--gallery")
_ -> false
end
end
defp is_empty_gif(el) do
case Floki.attribute(el, "src") do
[src] -> String.starts_with?(src, "data:image/gif;")
_ -> false
end
end
end end

5
lib/frenzy/pipeline/scrape_stage.ex
View File

@ -135,8 +135,9 @@ defmodule Frenzy.Pipeline.ScrapeStage do
end end
html = html =
Floki.find_and_update( html
html, |> Floki.filter_out("script")
|> Floki.find_and_update(
"img", "img",
rewrite_image_urls(convert_to_data_uris, URI.parse(url)) rewrite_image_urls(convert_to_data_uris, URI.parse(url))
) )

1
lib/frenzy_web/templates/layout/root.html.eex
View File

@ -13,7 +13,6 @@
<script defer phx-track-static type="text/javascript" src="<%= Routes.static_path(@conn, "/assets/app.js") %>"></script> <script defer phx-track-static type="text/javascript" src="<%= Routes.static_path(@conn, "/assets/app.js") %>"></script>
</head> </head>
<body> <body>
<header>
<main class="main mt-4" role="main"> <main class="main mt-4" role="main">
<div class="container"> <div class="container">
<label class="sidebar-toggle" for="show-sidebar"> <label class="sidebar-toggle" for="show-sidebar">