From 79fafb99c66c2448fd97fadb9045ef1721e70dfb Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Sun, 1 Sep 2019 16:11:13 -0400 Subject: [PATCH] Add Feed last_updated --- lib/feed.ex | 3 ++- lib/parser/atom.ex | 9 +++++++++ lib/parser/jsonfeed.ex | 1 + lib/parser/rss2.ex | 9 +++++++++ lib/parser/rssinjson.ex | 9 +++++++++ test/fixtures/rss2/feed.xml | 4 ++-- test/fixtures/rssinjson/feed.json | 6 +++--- test/parser/atom_test.exs | 1 + test/parser/jsonfeed_test.exs | 2 +- test/parser/rss2_test.exs | 1 + test/parser/rssinjson_test.exs | 1 + 11 files changed, 39 insertions(+), 7 deletions(-) diff --git a/lib/feed.ex b/lib/feed.ex index b0fe8db..362373a 100644 --- a/lib/feed.ex +++ b/lib/feed.ex @@ -3,11 +3,12 @@ defmodule FeedParser.Feed do A feed. Contains some information about the site it originates from and a list of items it contains. """ - defstruct [:site_url, :title, :image_url, :items] + defstruct [:site_url, :title, :image_url, :last_updated, :items] @type t() :: %__MODULE__{ site_url: String.t(), title: String.t(), + last_updated: DateTime.t() | nil, image_url: String.t() | nil, items: [FeedParser.Item.t()] } diff --git a/lib/parser/atom.ex b/lib/parser/atom.ex index 40bae42..c4ff0b1 100644 --- a/lib/parser/atom.ex +++ b/lib/parser/atom.ex @@ -34,6 +34,14 @@ defmodule FeedParser.Parser.Atom do link = attr('/feed/link/@href', feed) icon = text('/feed/icon/text()', feed) + updated = + text('/feed/updated/text()', feed) + |> Timex.parse("{ISO:Extended}") + |> case do + {:ok, date} -> date + _ -> nil + end + items = :xmerl_xpath.string('/feed/entry', feed) |> Enum.map(fn entry -> @@ -65,6 +73,7 @@ defmodule FeedParser.Parser.Atom do site_url: link, title: title, image_url: icon, + last_updated: updated, items: items }} end diff --git a/lib/parser/jsonfeed.ex b/lib/parser/jsonfeed.ex index c0b4708..ac6375a 100644 --- a/lib/parser/jsonfeed.ex +++ b/lib/parser/jsonfeed.ex @@ -59,6 +59,7 @@ defmodule FeedParser.Parser.JSONFeed do site_url: home_page_url, title: title, image_url: icon, + last_updated: nil, items: items }} end diff --git a/lib/parser/rss2.ex b/lib/parser/rss2.ex index 61e9ff2..b8c9f4f 100644 --- a/lib/parser/rss2.ex +++ b/lib/parser/rss2.ex @@ -35,6 +35,14 @@ defmodule FeedParser.Parser.RSS2 do link = text('/channel/link/text()', channel) image = text('/channel/image/url/text()', channel) + last_updated = + text('/channel/lastBuildDate/text()', channel) + |> Timex.parse("{RFC1123}") + |> case do + {:ok, date} -> date + _ -> nil + end + items = :xmerl_xpath.string('/channel/item', channel) |> Enum.map(fn item -> @@ -65,6 +73,7 @@ defmodule FeedParser.Parser.RSS2 do site_url: link, title: title, image_url: image, + last_updated: last_updated, items: items }} end diff --git a/lib/parser/rssinjson.ex b/lib/parser/rssinjson.ex index 5607bbc..f4966f7 100644 --- a/lib/parser/rssinjson.ex +++ b/lib/parser/rssinjson.ex @@ -23,6 +23,14 @@ defmodule FeedParser.Parser.RSSInJSON do title = channel["title"] link = channel["link"] + last_updated = + channel["lastBuildDate"] + |> Timex.parse("{RFC1123}") + |> case do + {:ok, date} -> date + _ -> nil + end + image = case channel do %{"image" => %{"url" => url}} -> url @@ -59,6 +67,7 @@ defmodule FeedParser.Parser.RSSInJSON do site_url: link, title: title, image_url: image, + last_updated: last_updated, items: items }} end diff --git a/test/fixtures/rss2/feed.xml b/test/fixtures/rss2/feed.xml index 37c6144..f990759 100644 --- a/test/fixtures/rss2/feed.xml +++ b/test/fixtures/rss2/feed.xml @@ -5,8 +5,8 @@ http://liftoff.msfc.nasa.gov/ Liftoff to Space Exploration. en-us - Tue, 10 Jun 2003 04:00:00 GMT - Tue, 10 Jun 2003 09:41:01 GMT + Tue, 10 Jun 2003 04:00:00 UTC + Tue, 10 Jun 2003 09:41:01 UTC http://blogs.law.harvard.edu/tech/rss Weblog Editor 2.0 editor@example.com diff --git a/test/fixtures/rssinjson/feed.json b/test/fixtures/rssinjson/feed.json index 7d0e8e6..f3f343c 100644 --- a/test/fixtures/rssinjson/feed.json +++ b/test/fixtures/rssinjson/feed.json @@ -6,8 +6,8 @@ "title": "Scripting News", "link": "http://scripting.com/", "description": "Scripting News, the weblog started in 1994 that bootstrapped the blogging revolution. 🚀", - "pubDate": "Sat, 31 Aug 2019 18:04:35 GMT", - "lastBuildDate": "Sat, 31 Aug 2019 22:49:49 GMT", + "pubDate": "Sat, 31 Aug 2019 18:04:35 UTC", + "lastBuildDate": "Sat, 31 Aug 2019 22:49:49 UTC", "language": "en-us", "copyright": "© 1994-2019 Dave Winer.", "generator": "oldSchool v0.5.29", @@ -55,4 +55,4 @@ ] } } -} \ No newline at end of file +} diff --git a/test/parser/atom_test.exs b/test/parser/atom_test.exs index 2ffa451..75f7428 100644 --- a/test/parser/atom_test.exs +++ b/test/parser/atom_test.exs @@ -15,6 +15,7 @@ defmodule FeedParser.Parser.AtomTest do assert {:ok, %FeedParser.Feed{} = feed} = Atom.parse_feed(parsed_data) assert feed.site_url == "http://example.org/" assert feed.title == "Example Feed" + assert feed.last_updated == ~U[2003-12-13 18:30:02Z] assert [%FeedParser.Item{} = item] = feed.items assert item.guid == "urn:uuid:1225c695-cfb8-4ebb-aaaa-80da344efa6a" assert item.title == "Atom-Powered Robots Run Amok" diff --git a/test/parser/jsonfeed_test.exs b/test/parser/jsonfeed_test.exs index b4401b8..ee5e5c9 100644 --- a/test/parser/jsonfeed_test.exs +++ b/test/parser/jsonfeed_test.exs @@ -14,7 +14,7 @@ defmodule FeedParser.Parser.JSONFeedTest do assert {:ok, %FeedParser.Feed{} = feed} = JSONFeed.parse_feed(parsed_data) assert feed.title == "My Example Feed" assert feed.site_url == "https://example.org/" - IO.inspect(feed.items) + assert feed.last_updated == nil assert [%FeedParser.Item{} = item2, %FeedParser.Item{} = item1] = feed.items assert item2.guid == "2" assert item2.content == "This is a second item." diff --git a/test/parser/rss2_test.exs b/test/parser/rss2_test.exs index 61035d7..5bf94c7 100644 --- a/test/parser/rss2_test.exs +++ b/test/parser/rss2_test.exs @@ -15,6 +15,7 @@ defmodule FeedParser.Parser.RSS2Test do assert {:ok, %FeedParser.Feed{} = feed} = RSS2.parse_feed(parsed_data) assert feed.title == "Liftoff News" assert feed.site_url == "http://liftoff.msfc.nasa.gov/" + assert feed.last_updated == ~U[2003-06-10 09:41:01Z] assert [%FeedParser.Item{} = item] = feed.items assert item.title == "Star City" assert item.url == "http://liftoff.msfc.nasa.gov/news/2003/news-starcity.asp" diff --git a/test/parser/rssinjson_test.exs b/test/parser/rssinjson_test.exs index d69db97..e941be1 100644 --- a/test/parser/rssinjson_test.exs +++ b/test/parser/rssinjson_test.exs @@ -14,6 +14,7 @@ defmodule FeedParser.Parser.RSSInJSONTest do assert {:ok, %FeedParser.Feed{} = feed} = RSSInJSON.parse_feed(parsed_data) assert feed.title == "Scripting News" assert feed.site_url == "http://scripting.com/" + assert feed.last_updated == ~U[2019-08-31 22:49:49Z] assert [%FeedParser.Item{} = item] = feed.items assert item.url == "http://scripting.com/2019/08/31.html#a151142" assert item.guid == "http://scripting.com/2019/08/31.html#a151142"