Add Feed last_updated

This commit is contained in:
Shadowfacts 2019-09-01 16:11:13 -04:00
parent e0dd659436
commit 79fafb99c6
Signed by: shadowfacts
GPG Key ID: 94A5AB95422746E5
11 changed files with 39 additions and 7 deletions

View File

@ -3,11 +3,12 @@ defmodule FeedParser.Feed do
A feed. Contains some information about the site it originates from and a list of items it contains. A feed. Contains some information about the site it originates from and a list of items it contains.
""" """
defstruct [:site_url, :title, :image_url, :items] defstruct [:site_url, :title, :image_url, :last_updated, :items]
@type t() :: %__MODULE__{ @type t() :: %__MODULE__{
site_url: String.t(), site_url: String.t(),
title: String.t(), title: String.t(),
last_updated: DateTime.t() | nil,
image_url: String.t() | nil, image_url: String.t() | nil,
items: [FeedParser.Item.t()] items: [FeedParser.Item.t()]
} }

View File

@ -34,6 +34,14 @@ defmodule FeedParser.Parser.Atom do
link = attr('/feed/link/@href', feed) link = attr('/feed/link/@href', feed)
icon = text('/feed/icon/text()', feed) icon = text('/feed/icon/text()', feed)
updated =
text('/feed/updated/text()', feed)
|> Timex.parse("{ISO:Extended}")
|> case do
{:ok, date} -> date
_ -> nil
end
items = items =
:xmerl_xpath.string('/feed/entry', feed) :xmerl_xpath.string('/feed/entry', feed)
|> Enum.map(fn entry -> |> Enum.map(fn entry ->
@ -65,6 +73,7 @@ defmodule FeedParser.Parser.Atom do
site_url: link, site_url: link,
title: title, title: title,
image_url: icon, image_url: icon,
last_updated: updated,
items: items items: items
}} }}
end end

View File

@ -59,6 +59,7 @@ defmodule FeedParser.Parser.JSONFeed do
site_url: home_page_url, site_url: home_page_url,
title: title, title: title,
image_url: icon, image_url: icon,
last_updated: nil,
items: items items: items
}} }}
end end

View File

@ -35,6 +35,14 @@ defmodule FeedParser.Parser.RSS2 do
link = text('/channel/link/text()', channel) link = text('/channel/link/text()', channel)
image = text('/channel/image/url/text()', channel) image = text('/channel/image/url/text()', channel)
last_updated =
text('/channel/lastBuildDate/text()', channel)
|> Timex.parse("{RFC1123}")
|> case do
{:ok, date} -> date
_ -> nil
end
items = items =
:xmerl_xpath.string('/channel/item', channel) :xmerl_xpath.string('/channel/item', channel)
|> Enum.map(fn item -> |> Enum.map(fn item ->
@ -65,6 +73,7 @@ defmodule FeedParser.Parser.RSS2 do
site_url: link, site_url: link,
title: title, title: title,
image_url: image, image_url: image,
last_updated: last_updated,
items: items items: items
}} }}
end end

View File

@ -23,6 +23,14 @@ defmodule FeedParser.Parser.RSSInJSON do
title = channel["title"] title = channel["title"]
link = channel["link"] link = channel["link"]
last_updated =
channel["lastBuildDate"]
|> Timex.parse("{RFC1123}")
|> case do
{:ok, date} -> date
_ -> nil
end
image = image =
case channel do case channel do
%{"image" => %{"url" => url}} -> url %{"image" => %{"url" => url}} -> url
@ -59,6 +67,7 @@ defmodule FeedParser.Parser.RSSInJSON do
site_url: link, site_url: link,
title: title, title: title,
image_url: image, image_url: image,
last_updated: last_updated,
items: items items: items
}} }}
end end

View File

@ -5,8 +5,8 @@
<link>http://liftoff.msfc.nasa.gov/</link> <link>http://liftoff.msfc.nasa.gov/</link>
<description>Liftoff to Space Exploration.</description> <description>Liftoff to Space Exploration.</description>
<language>en-us</language> <language>en-us</language>
<pubDate>Tue, 10 Jun 2003 04:00:00 GMT</pubDate> <pubDate>Tue, 10 Jun 2003 04:00:00 UTC</pubDate>
<lastBuildDate>Tue, 10 Jun 2003 09:41:01 GMT</lastBuildDate> <lastBuildDate>Tue, 10 Jun 2003 09:41:01 UTC</lastBuildDate>
<docs>http://blogs.law.harvard.edu/tech/rss</docs> <docs>http://blogs.law.harvard.edu/tech/rss</docs>
<generator>Weblog Editor 2.0</generator> <generator>Weblog Editor 2.0</generator>
<managingEditor>editor@example.com</managingEditor> <managingEditor>editor@example.com</managingEditor>

View File

@ -6,8 +6,8 @@
"title": "Scripting News", "title": "Scripting News",
"link": "http://scripting.com/", "link": "http://scripting.com/",
"description": "Scripting News, the weblog started in 1994 that bootstrapped the blogging revolution. 🚀", "description": "Scripting News, the weblog started in 1994 that bootstrapped the blogging revolution. 🚀",
"pubDate": "Sat, 31 Aug 2019 18:04:35 GMT", "pubDate": "Sat, 31 Aug 2019 18:04:35 UTC",
"lastBuildDate": "Sat, 31 Aug 2019 22:49:49 GMT", "lastBuildDate": "Sat, 31 Aug 2019 22:49:49 UTC",
"language": "en-us", "language": "en-us",
"copyright": "&copy; 1994-2019 <a href=\"http://davewiner.com/\">Dave Winer</a>.", "copyright": "&copy; 1994-2019 <a href=\"http://davewiner.com/\">Dave Winer</a>.",
"generator": "oldSchool v0.5.29", "generator": "oldSchool v0.5.29",

View File

@ -15,6 +15,7 @@ defmodule FeedParser.Parser.AtomTest do
assert {:ok, %FeedParser.Feed{} = feed} = Atom.parse_feed(parsed_data) assert {:ok, %FeedParser.Feed{} = feed} = Atom.parse_feed(parsed_data)
assert feed.site_url == "http://example.org/" assert feed.site_url == "http://example.org/"
assert feed.title == "Example Feed" assert feed.title == "Example Feed"
assert feed.last_updated == ~U[2003-12-13 18:30:02Z]
assert [%FeedParser.Item{} = item] = feed.items assert [%FeedParser.Item{} = item] = feed.items
assert item.guid == "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a" assert item.guid == "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a"
assert item.title == "Atom-Powered Robots Run Amok" assert item.title == "Atom-Powered Robots Run Amok"

View File

@ -14,7 +14,7 @@ defmodule FeedParser.Parser.JSONFeedTest do
assert {:ok, %FeedParser.Feed{} = feed} = JSONFeed.parse_feed(parsed_data) assert {:ok, %FeedParser.Feed{} = feed} = JSONFeed.parse_feed(parsed_data)
assert feed.title == "My Example Feed" assert feed.title == "My Example Feed"
assert feed.site_url == "https://example.org/" assert feed.site_url == "https://example.org/"
IO.inspect(feed.items) assert feed.last_updated == nil
assert [%FeedParser.Item{} = item2, %FeedParser.Item{} = item1] = feed.items assert [%FeedParser.Item{} = item2, %FeedParser.Item{} = item1] = feed.items
assert item2.guid == "2" assert item2.guid == "2"
assert item2.content == "This is a second item." assert item2.content == "This is a second item."

View File

@ -15,6 +15,7 @@ defmodule FeedParser.Parser.RSS2Test do
assert {:ok, %FeedParser.Feed{} = feed} = RSS2.parse_feed(parsed_data) assert {:ok, %FeedParser.Feed{} = feed} = RSS2.parse_feed(parsed_data)
assert feed.title == "Liftoff News" assert feed.title == "Liftoff News"
assert feed.site_url == "http://liftoff.msfc.nasa.gov/" assert feed.site_url == "http://liftoff.msfc.nasa.gov/"
assert feed.last_updated == ~U[2003-06-10 09:41:01Z]
assert [%FeedParser.Item{} = item] = feed.items assert [%FeedParser.Item{} = item] = feed.items
assert item.title == "Star City" assert item.title == "Star City"
assert item.url == "http://liftoff.msfc.nasa.gov/news/2003/news-starcity.asp" assert item.url == "http://liftoff.msfc.nasa.gov/news/2003/news-starcity.asp"

View File

@ -14,6 +14,7 @@ defmodule FeedParser.Parser.RSSInJSONTest do
assert {:ok, %FeedParser.Feed{} = feed} = RSSInJSON.parse_feed(parsed_data) assert {:ok, %FeedParser.Feed{} = feed} = RSSInJSON.parse_feed(parsed_data)
assert feed.title == "Scripting News" assert feed.title == "Scripting News"
assert feed.site_url == "http://scripting.com/" assert feed.site_url == "http://scripting.com/"
assert feed.last_updated == ~U[2019-08-31 22:49:49Z]
assert [%FeedParser.Item{} = item] = feed.items assert [%FeedParser.Item{} = item] = feed.items
assert item.url == "http://scripting.com/2019/08/31.html#a151142" assert item.url == "http://scripting.com/2019/08/31.html#a151142"
assert item.guid == "http://scripting.com/2019/08/31.html#a151142" assert item.guid == "http://scripting.com/2019/08/31.html#a151142"