shadowfacts
/
frenzy
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Compare commits

base: shadowfacts:fec640a37a0fdffb73577725f9976bf154cad435
Branches Tags
shadowfacts:main
shadowfacts:master
..
compare: shadowfacts:2dda6d7f46f3fe86b6fb6ba00835181d244abc22
Branches Tags
shadowfacts:main
shadowfacts:master

No commits in common. "fec640a37a0fdffb73577725f9976bf154cad435" and "2dda6d7f46f3fe86b6fb6ba00835181d244abc22" have entirely different histories.
fec640a37a ... 2dda6d7f46

3 changed files with 4 additions and 59 deletions
Show Stats Expand all files Collapse all files

57
lib/frenzy/pipeline/extractor/verge.ex
View File

@ -16,10 +16,8 @@ defmodule Frenzy.Pipeline.Extractor.TheVerge do
html_tree html_tree
|> Floki.find("article#content > div:not(.duet--article--lede)") |> Floki.find("article#content > div:not(.duet--article--lede)")
|> Floki.filter_out( |> Floki.filter_out(
".hidden, .duet--layout--rail, .duet--article--article-pullquote, .duet--article--comments-join-the-conversation, .duet--recirculation--related-list, .duet--article--comments-button, .duet--article--share-buttons" ".duet--layout--rail, .duet--article--article-pullquote, .duet--article--comments-join-the-conversation, .duet--recirculation--related-list, .duet--article--comments-button"
) )
|> Readability.Helper.remove_attrs("style")
|> Floki.traverse_and_update(&rewrite/1)
{:ok, image ++ content} {:ok, image ++ content}
end end
@ -56,57 +54,4 @@ defmodule Frenzy.Pipeline.Extractor.TheVerge do
[] []
end end
end end
defp rewrite({tag, attrs, children} = el) do
cond do
is_empty_gif(el) ->
nil
is_gallery(el) ->
images =
Floki.find(children, ":not(noscript) > img")
|> Enum.map(fn el ->
[src] = Floki.attribute(el, "src")
src
end)
{
"div",
[
{"style",
"display: flex; flex-direction: row; overflow-x: auto; scroll-snap-type: x mandatory;;"}
],
Enum.map(images, fn src ->
{
"img",
[
{"src", src},
{"loading", "lazy"},
{"style", "max-width: 85%; max-height: 30vh; scroll-snap-align: start;"}
],
[]
}
end)
}
true ->
el
end
end
defp rewrite(other), do: other
defp is_gallery(el) do
case Floki.attribute(el, "class") do
[classes] -> String.contains?(classes, "duet--article--gallery")
_ -> false
end
end
defp is_empty_gif(el) do
case Floki.attribute(el, "src") do
[src] -> String.starts_with?(src, "data:image/gif;")
_ -> false
end
end
end end

5
lib/frenzy/pipeline/scrape_stage.ex
View File

@ -135,9 +135,8 @@ defmodule Frenzy.Pipeline.ScrapeStage do
end end
html = html =
html Floki.find_and_update(
|> Floki.filter_out("script") html,
|> Floki.find_and_update(
"img", "img",
rewrite_image_urls(convert_to_data_uris, URI.parse(url)) rewrite_image_urls(convert_to_data_uris, URI.parse(url))
) )

1
lib/frenzy_web/templates/layout/root.html.eex
View File

@ -13,6 +13,7 @@
<script defer phx-track-static type="text/javascript" src="<%= Routes.static_path(@conn, "/assets/app.js") %>"></script> <script defer phx-track-static type="text/javascript" src="<%= Routes.static_path(@conn, "/assets/app.js") %>"></script>
</head> </head>
<body> <body>
<header>
<main class="main mt-4" role="main"> <main class="main mt-4" role="main">
<div class="container"> <div class="container">
<label class="sidebar-toggle" for="show-sidebar"> <label class="sidebar-toggle" for="show-sidebar">