commit d91604a51960c3e031a9053f37f501419347fcd3
Author: keepcosmos
]*>[ \n\r\t]*){2,}/i,
+ replaceFontsRe: ~r/<(\/?)font[^>]*>/i,
+ trimRe: ~r/^\s+|\s+$/,
+ normalizeRe: ~r/\s{2,}/,
+ killBreaksRe: ~r/(
(\s| ?)*){1,}/,
+ videoRe: ~r/http:\/\/(www\.)?(youtube|vimeo)\.com/i
+ ]
+
+ def html do
+ page
+ |> String.replace(@regexes[:replaceBrsRe], "
")
+ |> String.replace(@regexes[:replaceFontsRe], "<\1span>")
+ |> Floki.find("html")
+ |> Floki.filter_out(:comment)
+ end
+
+ def title do
+ html |> Floki.find("title") |> Floki.text
+ end
+
+ def content do
+ html
+ |> Floki.filter_out("script")
+ |> Floki.filter_out("style")
+ end
+
+ def page do
+ {:ok, f} = File.read("test/features/nytimes.html")
+ f
+ end
+
+ def default_options do
+ @default_options
+ end
+
+ def regexes do
+ @regexes
+ end
+end
diff --git a/lib/readability.ex b/lib/readability.ex
new file mode 100644
index 0000000..fc7121b
--- /dev/null
+++ b/lib/readability.ex
@@ -0,0 +1,10 @@
+defmodule Readability do
+ alias Readability.TitleFinder
+
+ @type html_tree :: tuple | list
+
+ def title(html) when is_binary(html), do: parse(html) |> title
+ def title(html_tree), do: TitleFinder.title(html_tree)
+
+ defp parse(raw_html), do: Floki.parse(raw_html)
+end
diff --git a/lib/test.js b/lib/test.js
new file mode 100644
index 0000000..53b4051
--- /dev/null
+++ b/lib/test.js
@@ -0,0 +1,1835 @@
+/*eslint-env es6:false*/
+/*
+ * Copyright (c) 2010 Arc90 Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * This code is heavily based on Arc90's readability.js (1.7.1) script
+ * available at: http://code.google.com/p/arc90labs-readability
+ */
+var root = this;
+
+/**
+ * Public constructor.
+ * @param {Object} uri The URI descriptor object.
+ * @param {HTMLDocument} doc The document to parse.
+ * @param {Object} options The options object.
+ */
+var Readability = function(uri, doc, options) {
+ options = options || {};
+
+ this._uri = uri;
+ this._doc = doc;
+ this._biggestFrame = false;
+ this._articleByline = null;
+ this._articleDir = null;
+
+ // Configureable options
+ this._debug = !!options.debug;
+ this._maxElemsToParse = options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE;
+ this._nbTopCandidates = options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES;
+ this._maxPages = options.maxPages || this.DEFAULT_MAX_PAGES;
+
+ // Start with all flags set
+ this._flags = this.FLAG_STRIP_UNLIKELYS |
+ this.FLAG_WEIGHT_CLASSES |
+ this.FLAG_CLEAN_CONDITIONALLY;
+
+ // The list of pages we've parsed in this call of readability,
+ // for autopaging. As a key store for easier searching.
+ this._parsedPages = {};
+
+ // A list of the ETag headers of pages we've parsed, in case they happen to match,
+ // we'll know it's a duplicate.
+ this._pageETags = {};
+
+ // Make an AJAX request for each page and append it to the document.
+ this._curPageNum = 1;
+
+ var logEl;
+
+ // Control whether log messages are sent to the console
+ if (this._debug) {
+ logEl = function(e) {
+ var rv = e.nodeName + " ";
+ if (e.nodeType == e.TEXT_NODE) {
+ return rv + '("' + e.textContent + '")';
+ }
+ var classDesc = e.className && ("." + e.className.replace(/ /g, "."));
+ var elDesc = "";
+ if (e.id)
+ elDesc = "(#" + e.id + classDesc + ")";
+ else if (classDesc)
+ elDesc = "(" + classDesc + ")";
+ return rv + elDesc;
+ };
+ this.log = function () {
+ if ("dump" in root) {
+ var msg = Array.prototype.map.call(arguments, function(x) {
+ return (x && x.nodeName) ? logEl(x) : x;
+ }).join(" ");
+ dump("Reader: (Readability) " + msg + "\n");
+ } else if ("console" in root) {
+ var args = ["Reader: (Readability) "].concat(arguments);
+ console.log.apply(console, args);
+ }
+ };
+ } else {
+ this.log = function () {};
+ }
+}
+
+Readability.prototype = {
+ FLAG_STRIP_UNLIKELYS: 0x1,
+ FLAG_WEIGHT_CLASSES: 0x2,
+ FLAG_CLEAN_CONDITIONALLY: 0x4,
+
+ // Max number of nodes supported by this parser. Default: 0 (no limit)
+ DEFAULT_MAX_ELEMS_TO_PARSE: 0,
+
+ // The number of top candidates to consider when analysing how
+ // tight the competition is among candidates.
+ DEFAULT_N_TOP_CANDIDATES: 5,
+
+ // The maximum number of pages to loop through before we call
+ // it quits and just show a link.
+ DEFAULT_MAX_PAGES: 5,
+
+ // Element tags to score by default.
+ DEFAULT_TAGS_TO_SCORE: "section,h2,h3,h4,h5,h6,p,td,pre".toUpperCase().split(","),
+
+ // All of the regular expressions in use within readability.
+ // Defined up here so we don't instantiate them repeatedly in loops.
+ REGEXPS: {
+ unlikelyCandidates: /banner|combx|comment|community|disqus|extra|foot|header|menu|modal|related|remark|rss|share|shoutbox|sidebar|skyscraper|sponsor|ad-break|agegate|pagination|pager|popup/i,
+ okMaybeItsACandidate: /and|article|body|column|main|shadow/i,
+ positive: /article|body|content|entry|hentry|main|page|pagination|post|text|blog|story/i,
+ negative: /hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|modal|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget/i,
+ extraneous: /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,
+ byline: /byline|author|dateline|writtenby/i,
+ replaceFonts: /<(\/?)font[^>]*>/gi,
+ normalize: /\s{2,}/g,
+ videos: /\/\/(www\.)?(dailymotion|youtube|youtube-nocookie|player\.vimeo)\.com/i,
+ nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,
+ prevLink: /(prev|earl|old|new|<|«)/i,
+ whitespace: /^\s*$/,
+ hasContent: /\S$/,
+ },
+
+ DIV_TO_P_ELEMS: [ "A", "BLOCKQUOTE", "DL", "DIV", "IMG", "OL", "P", "PRE", "TABLE", "UL", "SELECT" ],
+
+ ALTER_TO_DIV_EXCEPTIONS: ["DIV", "ARTICLE", "SECTION", "P"],
+
+ /**
+ * Run any post-process modifications to article content as necessary.
+ *
+ * @param Element
+ * @return void
+ **/
+ _postProcessContent: function(articleContent) {
+ // Readability cannot open relative uris so we convert them to absolute uris.
+ this._fixRelativeUris(articleContent);
+ },
+
+ /**
+ * Iterate over a NodeList, which doesn't natively fully implement the Array
+ * interface.
+ *
+ * For convenience, the current object context is applied to the provided
+ * iterate function.
+ *
+ * @param NodeList nodeList The NodeList.
+ * @param Function fn The iterate function.
+ * @return void
+ */
+ _forEachNode: function(nodeList, fn) {
+ return Array.prototype.forEach.call(nodeList, fn, this);
+ },
+
+ /**
+ * Iterate over a NodeList, return true if any of the provided iterate
+ * function calls returns true, false otherwise.
+ *
+ * For convenience, the current object context is applied to the
+ * provided iterate function.
+ *
+ * @param NodeList nodeList The NodeList.
+ * @param Function fn The iterate function.
+ * @return Boolean
+ */
+ _someNode: function(nodeList, fn) {
+ return Array.prototype.some.call(nodeList, fn, this);
+ },
+
+ /**
+ * Concat all nodelists passed as arguments.
+ *
+ * @return ...NodeList
+ * @return Array
+ */
+ _concatNodeLists: function() {
+ var slice = Array.prototype.slice;
+ var args = slice.call(arguments);
+ var nodeLists = args.map(function(list) {
+ return slice.call(list);
+ });
+ return Array.prototype.concat.apply([], nodeLists);
+ },
+
+ _getAllNodesWithTag: function(node, tagNames) {
+ if (node.querySelectorAll) {
+ return node.querySelectorAll(tagNames.join(','));
+ }
+ return [].concat.apply([], tagNames.map(function(tag) {
+ return node.getElementsByTagName(tag);
+ }));
+ },
+
+ /**
+ * Converts each and uri in the given element to an absolute URI,
+ * ignoring #ref URIs.
+ *
+ * @param Element
+ * @return void
+ */
+ _fixRelativeUris: function(articleContent) {
+ var scheme = this._uri.scheme;
+ var prePath = this._uri.prePath;
+ var pathBase = this._uri.pathBase;
+
+ function toAbsoluteURI(uri) {
+ // If this is already an absolute URI, return it.
+ if (/^[a-zA-Z][a-zA-Z0-9\+\-\.]*:/.test(uri))
+ return uri;
+
+ // Scheme-rooted relative URI.
+ if (uri.substr(0, 2) == "//")
+ return scheme + "://" + uri.substr(2);
+
+ // Prepath-rooted relative URI.
+ if (uri[0] == "/")
+ return prePath + uri;
+
+ // Dotslash relative URI.
+ if (uri.indexOf("./") === 0)
+ return pathBase + uri.slice(2);
+
+ // Ignore hash URIs:
+ if (uri[0] == "#")
+ return uri;
+
+ // Standard relative URI; add entire path. pathBase already includes a
+ // trailing "/".
+ return pathBase + uri;
+ }
+
+ var links = articleContent.getElementsByTagName("a");
+ this._forEachNode(links, function(link) {
+ var href = link.getAttribute("href");
+ if (href) {
+ // Replace links with javascript: URIs with text content, since
+ // they won't work after scripts have been removed from the page.
+ if (href.indexOf("javascript:") === 0) {
+ var text = this._doc.createTextNode(link.textContent);
+ link.parentNode.replaceChild(text, link);
+ } else {
+ link.setAttribute("href", toAbsoluteURI(href));
+ }
+ }
+ });
+
+ var imgs = articleContent.getElementsByTagName("img");
+ this._forEachNode(imgs, function(img) {
+ var src = img.getAttribute("src");
+ if (src) {
+ img.setAttribute("src", toAbsoluteURI(src));
+ }
+ });
+ },
+
+ /**
+ * Get the article title as an H1.
+ *
+ * @return void
+ **/
+ _getArticleTitle: function() {
+ var doc = this._doc;
+ var curTitle = "";
+ var origTitle = "";
+
+ try {
+ curTitle = origTitle = doc.title;
+
+ // If they had an element with id "title" in their HTML
+ if (typeof curTitle !== "string")
+ curTitle = origTitle = this._getInnerText(doc.getElementsByTagName('title')[0]);
+ } catch(e) {}
+
+ if (curTitle.match(/ [\|\-] /)) {
+ curTitle = origTitle.replace(/(.*)[\|\-] .*/gi,'$1');
+
+ if (curTitle.split(' ').length < 3)
+ curTitle = origTitle.replace(/[^\|\-]*[\|\-](.*)/gi,'$1');
+ } else if (curTitle.indexOf(': ') !== -1) {
+ // Check if we have an heading containing this exact string, so we
+ // could assume it's the full title.
+ var headings = this._concatNodeLists(
+ doc.getElementsByTagName('h1'),
+ doc.getElementsByTagName('h2')
+ );
+ var match = this._someNode(headings, function(heading) {
+ return heading.textContent === curTitle;
+ });
+
+ // If we don't, let's extract the title out of the original title string.
+ if (!match) {
+ curTitle = origTitle.- substring(origTitle.lastIndexOf(':') + 1);
+
+ // If the title is now too short, try the first colon instead:
+ if (curTitle.split(' ').length < 3)
+ curTitle = origTitle.substring(origTitle.indexOf(':') + 1);
+ }
+ } else if (curTitle.length > 150 || curTitle.length < 15) {
+ var hOnes = doc.getElementsByTagName('h1');
+
+ if (hOnes.length === 1)
+ curTitle = this._getInnerText(hOnes[0]);
+ }
+
+ curTitle = curTitle.trim();
+
+ if (curTitle.split(' ').length <= 4)
+ curTitle = origTitle;
+
+ return curTitle;
+ },
+
+ /**
+ * Prepare the HTML document for readability to scrape it.
+ * This includes things like stripping javascript, CSS, and handling terrible markup.
+ *
+ * @return void
+ **/
+ _prepDocument: function() {
+ var doc = this._doc;
+
+ // Remove all style tags in head
+ this._forEachNode(doc.getElementsByTagName("style"), function(styleNode) {
+ styleNode.parentNode.removeChild(styleNode);
+ });
+
+ if (doc.body) {
+ this._replaceBrs(doc.body);
+ }
+
+ this._forEachNode(doc.getElementsByTagName("font"), function(fontNode) {
+ this._setNodeTag(fontNode, "SPAN");
+ });
+ },
+
+ /**
+ * Finds the next element, starting from the given node, and ignoring
+ * whitespace in between. If the given node is an element, the same node is
+ * returned.
+ */
+ _nextElement: function (node) {
+ var next = node;
+ while (next
+ && (next.nodeType != Node.ELEMENT_NODE)
+ && this.REGEXPS.whitespace.test(next.textContent)) {
+ next = next.nextSibling;
+ }
+ return next;
+ },
+
+ /**
+ * Replaces 2 or more successive .
+ * Whitespace between abc block.
+ var replaced = false;
+
+ // If we find a later).
+ while ((next = this._nextElement(next)) && (next.tagName == "BR")) {
+ replaced = true;
+ var brSibling = next.nextSibling;
+ next.parentNode.removeChild(next);
+ next = brSibling;
+ }
+
+ // If we removed a . Add
+ // all sibling nodes as children of the until we hit another .
+ if (next.tagName == "BR") {
+ var nextElem = this._nextElement(next);
+ if (nextElem && nextElem.tagName == "BR")
+ break;
+ }
+
+ // Otherwise, make this node a child of the new .
+ var sibling = next.nextSibling;
+ p.appendChild(next);
+ next = sibling;
+ }
+ }
+ });
+ },
+
+ _setNodeTag: function (node, tag) {
+ this.log("_setNodeTag", node, tag);
+ if (node.__JSDOMParser__) {
+ node.localName = tag.toLowerCase();
+ node.tagName = tag.toUpperCase();
+ return node;
+ }
+
+ var replacement = node.ownerDocument.createElement(tag);
+ while (node.firstChild) {
+ replacement.appendChild(node.firstChild);
+ }
+ node.parentNode.replaceChild(replacement, node);
+ if (node.readability)
+ replacement.readability = node.readability;
+
+ for (var i = 0; i < node.attributes.length; i++) {
+ replacement.setAttribute(node.attributes[i].name, node.attributes[i].value);
+ }
+ return replacement;
+ },
+
+ /**
+ * Prepare the article node for display. Clean out any inline styles,
+ * iframes, forms, strip extraneous tags, etc.
+ *
+ * @param Element
+ * @return void
+ **/
+ _prepArticle: function(articleContent) {
+ this._cleanStyles(articleContent);
+
+ // Clean out junk from the article content
+ this._cleanConditionally(articleContent, "form");
+ this._clean(articleContent, "object");
+ this._clean(articleContent, "embed");
+ this._clean(articleContent, "h1");
+ this._clean(articleContent, "footer");
+
+ // If there is only one h2, they are probably using it as a header
+ // and not a subheader, so remove it since we already have a header.
+ if (articleContent.getElementsByTagName('h2').length === 1)
+ this._clean(articleContent, "h2");
+
+ this._clean(articleContent, "iframe");
+ this._cleanHeaders(articleContent);
+
+ // Do these last as the previous stuff may have removed junk
+ // that will affect these
+ this._cleanConditionally(articleContent, "table");
+ this._cleanConditionally(articleContent, "ul");
+ this._cleanConditionally(articleContent, "div");
+
+ // Remove extra paragraphs
+ this._forEachNode(articleContent.getElementsByTagName('p'), function(paragraph) {
+ var imgCount = paragraph.getElementsByTagName('img').length;
+ var embedCount = paragraph.getElementsByTagName('embed').length;
+ var objectCount = paragraph.getElementsByTagName('object').length;
+ // At this point, nasty iframes have been removed, only remain embedded video ones.
+ var iframeCount = paragraph.getElementsByTagName('iframe').length;
+ var totalCount = imgCount + embedCount + objectCount + iframeCount;
+
+ if (totalCount === 0 && !this._getInnerText(paragraph, false))
+ paragraph.parentNode.removeChild(paragraph);
+ });
+
+ this._forEachNode(articleContent.getElementsByTagName("br"), function(br) {
+ var next = this._nextElement(br.nextSibling);
+ if (next && next.tagName == "P")
+ br.parentNode.removeChild(br);
+ });
+ },
+
+ /**
+ * Initialize a node with the readability object. Also checks the
+ * className/id for special names to add to its score.
+ *
+ * @param Element
+ * @return void
+ **/
+ _initializeNode: function(node) {
+ node.readability = {"contentScore": 0};
+
+ switch(node.tagName) {
+ case 'DIV':
+ node.readability.contentScore += 5;
+ break;
+
+ case 'PRE':
+ case 'TD':
+ case 'BLOCKQUOTE':
+ node.readability.contentScore += 3;
+ break;
+
+ case 'ADDRESS':
+ case 'OL':
+ case 'UL':
+ case 'DL':
+ case 'DD':
+ case 'DT':
+ case 'LI':
+ case 'FORM':
+ node.readability.contentScore -= 3;
+ break;
+
+ case 'H1':
+ case 'H2':
+ case 'H3':
+ case 'H4':
+ case 'H5':
+ case 'H6':
+ case 'TH':
+ node.readability.contentScore -= 5;
+ break;
+ }
+
+ node.readability.contentScore += this._getClassWeight(node);
+ },
+
+ _removeAndGetNext: function(node) {
+ var nextNode = this._getNextNode(node, true);
+ node.parentNode.removeChild(node);
+ return nextNode;
+ },
+
+ /**
+ * Traverse the DOM from node to node, starting at the node passed in.
+ * Pass true for the second parameter to indicate this node itself
+ * (and its kids) are going away, and we want the next node over.
+ *
+ * Calling this in a loop will traverse the DOM depth-first.
+ */
+ _getNextNode: function(node, ignoreSelfAndKids) {
+ // First check for kids if those aren't being ignored
+ if (!ignoreSelfAndKids && node.firstElementChild) {
+ return node.firstElementChild;
+ }
+ // Then for siblings...
+ if (node.nextElementSibling) {
+ return node.nextElementSibling;
+ }
+ // And finally, move up the parent chain *and* find a sibling
+ // (because this is depth-first traversal, we will have already
+ // seen the parent nodes themselves).
+ do {
+ node = node.parentNode;
+ } while (node && !node.nextElementSibling);
+ return node && node.nextElementSibling;
+ },
+
+ /**
+ * Like _getNextNode, but for DOM implementations with no
+ * firstElementChild/nextElementSibling functionality...
+ */
+ _getNextNodeNoElementProperties: function(node, ignoreSelfAndKids) {
+ function nextSiblingEl(n) {
+ do {
+ n = n.nextSibling;
+ } while (n && n.nodeType !== n.ELEMENT_NODE);
+ return n;
+ }
+ // First check for kids if those aren't being ignored
+ if (!ignoreSelfAndKids && node.children[0]) {
+ return node.children[0];
+ }
+ // Then for siblings...
+ var next = nextSiblingEl(node);
+ if (next) {
+ return next;
+ }
+ // And finally, move up the parent chain *and* find a sibling
+ // (because this is depth-first traversal, we will have already
+ // seen the parent nodes themselves).
+ do {
+ node = node.parentNode;
+ if (node)
+ next = nextSiblingEl(node);
+ } while (node && !next);
+ return node && next;
+ },
+
+ _checkByline: function(node, matchString) {
+ if (this._articleByline) {
+ return false;
+ }
+
+ if (node.getAttribute !== undefined) {
+ var rel = node.getAttribute("rel");
+ }
+
+ if ((rel === "author" || this.REGEXPS.byline.test(matchString)) && this._isValidByline(node.textContent)) {
+ this._articleByline = node.textContent.trim();
+ return true;
+ }
+
+ return false;
+ },
+
+ _getNodeAncestors: function(node, maxDepth) {
+ maxDepth = maxDepth || 0;
+ var i = 0, ancestors = [];
+ while (node.parentNode) {
+ ancestors.push(node.parentNode)
+ if (maxDepth && ++i === maxDepth)
+ break;
+ node = node.parentNode;
+ }
+ return ancestors;
+ },
+
+ /***
+ * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
+ * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
+ *
+ * @param page a document to run upon. Needs to be a full document, complete with body.
+ * @return Element
+ **/
+ _grabArticle: function (page) {
+ this.log("**** grabArticle ****");
+ var doc = this._doc;
+ var isPaging = (page !== null ? true: false);
+ page = page ? page : this._doc.body;
+
+ // We can't grab an article if we don't have a page!
+ if (!page) {
+ this.log("No body found in document. Abort.");
+ return null;
+ }
+
+ var pageCacheHtml = page.innerHTML;
+
+ // Check if any "dir" is set on the toplevel document element
+ this._articleDir = doc.documentElement.getAttribute("dir");
+
+ while (true) {
+ var stripUnlikelyCandidates = this._flagIsActive(this.FLAG_STRIP_UNLIKELYS);
+
+ // First, node prepping. Trash nodes that look cruddy (like ones with the
+ // class name "comment", etc), and turn divs into P tags where they have been
+ // used inappropriately (as in, where they contain no other block level elements.)
+ var elementsToScore = [];
+ var node = this._doc.documentElement;
+
+ while (node) {
+ var matchString = node.className + " " + node.id;
+
+ // Check to see if this node is a byline, and remove it if it is.
+ if (this._checkByline(node, matchString)) {
+ node = this._removeAndGetNext(node);
+ continue;
+ }
+
+ // Remove unlikely candidates
+ if (stripUnlikelyCandidates) {
+ if (this.REGEXPS.unlikelyCandidates.test(matchString) &&
+ !this.REGEXPS.okMaybeItsACandidate.test(matchString) &&
+ node.tagName !== "BODY" &&
+ node.tagName !== "A") {
+ this.log("Removing unlikely candidate - " + matchString);
+ node = this._removeAndGetNext(node);
+ continue;
+ }
+ }
+
+ if (this.DEFAULT_TAGS_TO_SCORE.indexOf(node.tagName) !== -1) {
+ elementsToScore.push(node);
+ }
+
+ // Turn all divs that don't have children block level elements into p's
+ if (node.tagName === "DIV") {
+ // Sites like http://mobile.slate.com encloses each paragraph with a DIV
+ // element. DIVs with only a P element inside and no text content can be
+ // safely converted into plain P elements to avoid confusing the scoring
+ // algorithm with DIVs with are, in practice, paragraphs.
+ if (this._hasSinglePInsideElement(node)) {
+ var newNode = node.children[0];
+ node.parentNode.replaceChild(newNode, node);
+ node = newNode;
+ } else if (!this._hasChildBlockElement(node)) {
+ node = this._setNodeTag(node, "P");
+ elementsToScore.push(node);
+ } else {
+ // EXPERIMENTAL
+ this._forEachNode(node.childNodes, function(childNode) {
+ if (childNode.nodeType === Node.TEXT_NODE) {
+ var p = doc.createElement('p');
+ p.textContent = childNode.textContent;
+ p.style.display = 'inline';
+ p.className = 'readability-styled';
+ node.replaceChild(p, childNode);
+ }
+ });
+ }
+ }
+ node = this._getNextNode(node);
+ }
+
+ /**
+ * Loop through all paragraphs, and assign a score to them based on how content-y they look.
+ * Then add their score to their parent node.
+ *
+ * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
+ **/
+ var candidates = [];
+ this._forEachNode(elementsToScore, function(elementToScore) {
+ if (!elementToScore.parentNode || typeof(elementToScore.parentNode.tagName) === 'undefined')
+ return;
+
+ // If this paragraph is less than 25 characters, don't even count it.
+ var innerText = this._getInnerText(elementToScore);
+ if (innerText.length < 25)
+ return;
+
+ // Exclude nodes with no ancestor.
+ var ancestors = this._getNodeAncestors(elementToScore, 3);
+ if (ancestors.length === 0)
+ return;
+
+ var contentScore = 0;
+
+ // Add a point for the paragraph itself as a base.
+ contentScore += 1;
+
+ // Add points for any commas within this paragraph.
+ contentScore += innerText.split(',').length;
+
+ // For every 100 characters in this paragraph, add another point. Up to 3 points.
+ contentScore += Math.min(Math.floor(innerText.length / 100), 3);
+
+ // Initialize and score ancestors.
+ this._forEachNode(ancestors, function(ancestor, level) {
+ if (!ancestor.tagName)
+ return;
+
+ if (typeof(ancestor.readability) === 'undefined') {
+ this._initializeNode(ancestor);
+ candidates.push(ancestor);
+ }
+
+ // Node score divider:
+ // - parent: 1 (no division)
+ // - grandparent: 2
+ // - great grandparent+: ancestor level * 3
+ if (level === 0)
+ var scoreDivider = 1;
+ else if (level === 1)
+ scoreDivider = 2;
+ else
+ scoreDivider = level * 3;
+ ancestor.readability.contentScore += contentScore / scoreDivider;
+ });
+ });
+
+ // After we've calculated scores, loop through all of the possible
+ // candidate nodes we found and find the one with the highest score.
+ var topCandidates = [];
+ for (var c = 0, cl = candidates.length; c < cl; c += 1) {
+ var candidate = candidates[c];
+
+ // Scale the final candidates score based on link density. Good content
+ // should have a relatively small link density (5% or less) and be mostly
+ // unaffected by this operation.
+ var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate));
+ candidate.readability.contentScore = candidateScore;
+
+ this.log('Candidate:', candidate, "with score " + candidateScore);
+
+ for (var t = 0; t < this._nbTopCandidates; t++) {
+ var aTopCandidate = topCandidates[t];
+
+ if (!aTopCandidate || candidateScore > aTopCandidate.readability.contentScore) {
+ topCandidates.splice(t, 0, candidate);
+ if (topCandidates.length > this._nbTopCandidates)
+ topCandidates.pop();
+ break;
+ }
+ }
+ }
+
+ var topCandidate = topCandidates[0] || null;
+ var neededToCreateTopCandidate = false;
+
+ // If we still have no top candidate, just use the body as a last resort.
+ // We also have to copy the body node so it is something we can modify.
+ if (topCandidate === null || topCandidate.tagName === "BODY") {
+ // Move all of the page's children into topCandidate
+ topCandidate = doc.createElement("DIV");
+ neededToCreateTopCandidate = true;
+ // Move everything (not just elements, also text nodes etc.) into the container
+ // so we even include text directly in the body:
+ var kids = page.childNodes;
+ while (kids.length) {
+ this.log("Moving child out:", kids[0]);
+ topCandidate.appendChild(kids[0]);
+ }
+
+ page.appendChild(topCandidate);
+
+ this._initializeNode(topCandidate);
+ } else if (topCandidate) {
+ // Because of our bonus system, parents of candidates might have scores
+ // themselves. They get half of the node. There won't be nodes with higher
+ // scores than our topCandidate, but if we see the score going *up* in the first
+ // few steps up the tree, that's a decent sign that there might be more content
+ // lurking in other places that we want to unify in. The sibling stuff
+ // below does some of that - but only if we've looked high enough up the DOM
+ // tree.
+ var parentOfTopCandidate = topCandidate.parentNode;
+ var lastScore = topCandidate.readability.contentScore;
+ // The scores shouldn't get too low.
+ var scoreThreshold = lastScore / 3;
+ while (parentOfTopCandidate && parentOfTopCandidate.readability) {
+ var parentScore = parentOfTopCandidate.readability.contentScore;
+ if (parentScore < scoreThreshold)
+ break;
+ if (parentScore > lastScore) {
+ // Alright! We found a better parent to use.
+ topCandidate = parentOfTopCandidate;
+ break;
+ }
+ lastScore = parentOfTopCandidate.readability.contentScore;
+ parentOfTopCandidate = parentOfTopCandidate.parentNode;
+ }
+ }
+
+ // Now that we have the top candidate, look through its siblings for content
+ // that might also be related. Things like preambles, content split by ads
+ // that we removed, etc.
+ var articleContent = doc.createElement("DIV");
+ if (isPaging)
+ articleContent.id = "readability-content";
+
+ var siblingScoreThreshold = Math.max(10, topCandidate.readability.contentScore * 0.2);
+ var siblings = topCandidate.parentNode.children;
+
+ for (var s = 0, sl = siblings.length; s < sl; s++) {
+ var sibling = siblings[s];
+ var append = false;
+
+ this.log("Looking at sibling node:", sibling, sibling.readability ? ("with score " + sibling.readability.contentScore) : '');
+ this.log("Sibling has score", sibling.readability ? sibling.readability.contentScore : 'Unknown');
+
+ if (sibling === topCandidate) {
+ append = true;
+ } else {
+ var contentBonus = 0;
+
+ // Give a bonus if sibling nodes and top candidates have the example same classname
+ if (sibling.className === topCandidate.className && topCandidate.className !== "")
+ contentBonus += topCandidate.readability.contentScore * 0.2;
+
+ if (sibling.readability &&
+ ((sibling.readability.contentScore + contentBonus) >= siblingScoreThreshold)) {
+ append = true;
+ } else if (sibling.nodeName === "P") {
+ var linkDensity = this._getLinkDensity(sibling);
+ var nodeContent = this._getInnerText(sibling);
+ var nodeLength = nodeContent.length;
+
+ if (nodeLength > 80 && linkDensity < 0.25) {
+ append = true;
+ } else if (nodeLength < 80 && nodeLength > 0 && linkDensity === 0 &&
+ nodeContent.search(/\.( |$)/) !== -1) {
+ append = true;
+ }
+ }
+ }
+
+ if (append) {
+ this.log("Appending node:", sibling);
+
+ if (this.ALTER_TO_DIV_EXCEPTIONS.indexOf(sibling.nodeName) === -1) {
+ // We have a node that isn't a common block level element, like a form or td tag.
+ // Turn it into a div so it doesn't get filtered out later by accident.
+ this.log("Altering sibling:", sibling, 'to div.');
+
+ sibling = this._setNodeTag(sibling, "DIV");
+ }
+
+ articleContent.appendChild(sibling);
+ // siblings is a reference to the children array, and
+ // sibling is removed from the array when we call appendChild().
+ // As a result, we must revisit this index since the nodes
+ // have been shifted.
+ s -= 1;
+ sl -= 1;
+ }
+ }
+
+ if (this._debug)
+ this.log("Article content pre-prep: " + articleContent.innerHTML);
+ // So we have all of the content that we need. Now we clean it up for presentation.
+ this._prepArticle(articleContent);
+ if (this._debug)
+ this.log("Article content post-prep: " + articleContent.innerHTML);
+
+ if (this._curPageNum === 1) {
+ if (neededToCreateTopCandidate) {
+ // We already created a fake div thing, and there wouldn't have been any siblings left
+ // for the previous loop, so there's no point trying to create a new div, and then
+ // move all the children over. Just assign IDs and class names here. No need to append
+ // because that already happened anyway.
+ topCandidate.id = "readability-page-1";
+ topCandidate.className = "page";
+ } else {
+ var div = doc.createElement("DIV");
+ div.id = "readability-page-1";
+ div.className = "page";
+ var children = articleContent.childNodes;
+ while (children.length) {
+ div.appendChild(children[0]);
+ }
+ articleContent.appendChild(div);
+ }
+ }
+
+ if (this._debug)
+ this.log("Article content after paging: " + articleContent.innerHTML);
+
+ // Now that we've gone through the full algorithm, check to see if
+ // we got any meaningful content. If we didn't, we may need to re-run
+ // grabArticle with different flags set. This gives us a higher likelihood of
+ // finding the content, and the sieve approach gives us a higher likelihood of
+ // finding the -right- content.
+ if (this._getInnerText(articleContent, true).length < 500) {
+ page.innerHTML = pageCacheHtml;
+
+ if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) {
+ this._removeFlag(this.FLAG_STRIP_UNLIKELYS);
+ } else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {
+ this._removeFlag(this.FLAG_WEIGHT_CLASSES);
+ } else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {
+ this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);
+ } else {
+ return null;
+ }
+ } else {
+ return articleContent;
+ }
+ }
+ },
+
+ /**
+ * Check whether the input string could be a byline.
+ * This verifies that the input is a string, and that the length
+ * is less than 100 chars.
+ *
+ * @param possibleByline {string} - a string to check whether its a byline.
+ * @return Boolean - whether the input string is a byline.
+ */
+ _isValidByline: function(byline) {
+ if (typeof byline == 'string' || byline instanceof String) {
+ byline = byline.trim();
+ return (byline.length > 0) && (byline.length < 100);
+ }
+ return false;
+ },
+
+ /**
+ * Attempts to get excerpt and byline metadata for the article.
+ *
+ * @return Object with optional "excerpt" and "byline" properties
+ */
+ _getArticleMetadata: function() {
+ var metadata = {};
+ var values = {};
+ var metaElements = this._doc.getElementsByTagName("meta");
+
+ // Match "description", or Twitter's "twitter:description" (Cards)
+ // in name attribute.
+ var namePattern = /^\s*((twitter)\s*:\s*)?(description|title)\s*$/gi;
+
+ // Match Facebook's Open Graph title & description properties.
+ var propertyPattern = /^\s*og\s*:\s*(description|title)\s*$/gi;
+
+ // Find description tags.
+ this._forEachNode(metaElements, function(element) {
+ var elementName = element.getAttribute("name");
+ var elementProperty = element.getAttribute("property");
+
+ if ([elementName, elementProperty].indexOf("author") !== -1) {
+ metadata.byline = element.getAttribute("content");
+ return;
+ }
+
+ var name = null;
+ if (namePattern.test(elementName)) {
+ name = elementName;
+ } else if (propertyPattern.test(elementProperty)) {
+ name = elementProperty;
+ }
+
+ if (name) {
+ var content = element.getAttribute("content");
+ if (content) {
+ // Convert to lowercase and remove any whitespace
+ // so we can match below.
+ name = name.toLowerCase().replace(/\s/g, '');
+ values[name] = content.trim();
+ }
+ }
+ });
+
+ if ("description" in values) {
+ metadata.excerpt = values["description"];
+ } else if ("og:description" in values) {
+ // Use facebook open graph description.
+ metadata.excerpt = values["og:description"];
+ } else if ("twitter:description" in values) {
+ // Use twitter cards description.
+ metadata.excerpt = values["twitter:description"];
+ }
+
+ if ("og:title" in values) {
+ // Use facebook open graph title.
+ metadata.title = values["og:title"];
+ } else if ("twitter:title" in values) {
+ // Use twitter cards title.
+ metadata.title = values["twitter:title"];
+ }
+
+ return metadata;
+ },
+
+ /**
+ * Removes script tags from the document.
+ *
+ * @param Element
+ **/
+ _removeScripts: function(doc) {
+ this._forEachNode(doc.getElementsByTagName('script'), function(scriptNode) {
+ scriptNode.nodeValue = "";
+ scriptNode.removeAttribute('src');
+
+ if (scriptNode.parentNode)
+ scriptNode.parentNode.removeChild(scriptNode);
+ });
+ this._forEachNode(doc.getElementsByTagName('noscript'), function(noscriptNode) {
+ if (noscriptNode.parentNode)
+ noscriptNode.parentNode.removeChild(noscriptNode);
+ });
+ },
+
+ /**
+ * Check if this node has only whitespace and a single P element
+ * Returns false if the DIV node contains non-empty text nodes
+ * or if it contains no P or more than 1 element.
+ *
+ * @param Element
+ **/
+ _hasSinglePInsideElement: function(element) {
+ // There should be exactly 1 element child which is a P:
+ if (element.children.length != 1 || element.children[0].tagName !== "P") {
+ return false;
+ }
+
+ // And there should be no text nodes with real content
+ return !this._someNode(element.childNodes, function(node) {
+ return node.nodeType === Node.TEXT_NODE &&
+ this.REGEXPS.hasContent.test(node.textContent);
+ });
+ },
+
+ /**
+ * Determine whether element has any children block level elements.
+ *
+ * @param Element
+ */
+ _hasChildBlockElement: function (element) {
+ return this._someNode(element.childNodes, function(node) {
+ return this.DIV_TO_P_ELEMS.indexOf(node.tagName) !== -1 ||
+ this._hasChildBlockElement(node);
+ });
+ },
+
+ /**
+ * Get the inner text of a node - cross browser compatibly.
+ * This also strips out any excess whitespace to be found.
+ *
+ * @param Element
+ * @param Boolean normalizeSpaces (default: true)
+ * @return string
+ **/
+ _getInnerText: function(e, normalizeSpaces) {
+ normalizeSpaces = (typeof normalizeSpaces === 'undefined') ? true : normalizeSpaces;
+ var textContent = e.textContent.trim();
+
+ if (normalizeSpaces) {
+ return textContent.replace(this.REGEXPS.normalize, " ");
+ } else {
+ return textContent;
+ }
+ },
+
+ /**
+ * Get the number of times a string s appears in the node e.
+ *
+ * @param Element
+ * @param string - what to split on. Default is ","
+ * @return number (integer)
+ **/
+ _getCharCount: function(e,s) {
+ s = s || ",";
+ return this._getInnerText(e).split(s).length - 1;
+ },
+
+ /**
+ * Remove the style attribute on every e and under.
+ * TODO: Test if getElementsByTagName(*) is faster.
+ *
+ * @param Element
+ * @return void
+ **/
+ _cleanStyles: function(e) {
+ e = e || this._doc;
+ if (!e)
+ return;
+ var cur = e.firstChild;
+
+ // Remove any root styles, if we're able.
+ if (typeof e.removeAttribute === 'function' && e.className !== 'readability-styled')
+ e.removeAttribute('style');
+
+ // Go until there are no more child nodes
+ while (cur !== null) {
+ if (cur.nodeType === cur.ELEMENT_NODE) {
+ // Remove style attribute(s) :
+ if (cur.className !== "readability-styled")
+ cur.removeAttribute("style");
+
+ this._cleanStyles(cur);
+ }
+
+ cur = cur.nextSibling;
+ }
+ },
+
+ /**
+ * Get the density of links as a percentage of the content
+ * This is the amount of text that is inside a link divided by the total text in the node.
+ *
+ * @param Element
+ * @return number (float)
+ **/
+ _getLinkDensity: function(element) {
+ var textLength = this._getInnerText(element).length;
+ if (textLength === 0)
+ return 0;
+
+ var linkLength = 0;
+
+ // XXX implement _reduceNodeList?
+ this._forEachNode(element.getElementsByTagName("a"), function(linkNode) {
+ linkLength += this._getInnerText(linkNode).length;
+ });
+
+ return linkLength / textLength;
+ },
+
+ /**
+ * Find a cleaned up version of the current URL, to use for comparing links for possible next-pageyness.
+ *
+ * @author Dan Lacy
+ * @return string the base url
+ **/
+ _findBaseUrl: function() {
+ var uri = this._uri;
+ var noUrlParams = uri.path.split("?")[0];
+ var urlSlashes = noUrlParams.split("/").reverse();
+ var cleanedSegments = [];
+ var possibleType = "";
+
+ for (var i = 0, slashLen = urlSlashes.length; i < slashLen; i += 1) {
+ var segment = urlSlashes[i];
+
+ // Split off and save anything that looks like a file type.
+ if (segment.indexOf(".") !== -1) {
+ possibleType = segment.split(".")[1];
+
+ // If the type isn't alpha-only, it's probably not actually a file extension.
+ if (!possibleType.match(/[^a-zA-Z]/))
+ segment = segment.split(".")[0];
+ }
+
+ // EW-CMS specific segment replacement. Ugly.
+ // Example: http://www.ew.com/ew/article/0,,20313460_20369436,00.html
+ if (segment.indexOf(',00') !== -1)
+ segment = segment.replace(',00', '');
+
+ // If our first or second segment has anything looking like a page number, remove it.
+ if (segment.match(/((_|-)?p[a-z]*|(_|-))[0-9]{1,2}$/i) && ((i === 1) || (i === 0)))
+ segment = segment.replace(/((_|-)?p[a-z]*|(_|-))[0-9]{1,2}$/i, "");
+
+ var del = false;
+
+ // If this is purely a number, and it's the first or second segment,
+ // it's probably a page number. Remove it.
+ if (i < 2 && segment.match(/^\d{1,2}$/))
+ del = true;
+
+ // If this is the first segment and it's just "index", remove it.
+ if (i === 0 && segment.toLowerCase() === "index")
+ del = true;
+
+ // If our first or second segment is smaller than 3 characters,
+ // and the first segment was purely alphas, remove it.
+ if (i < 2 && segment.length < 3 && !urlSlashes[0].match(/[a-z]/i))
+ del = true;
+
+ // If it's not marked for deletion, push it to cleanedSegments.
+ if (!del)
+ cleanedSegments.push(segment);
+ }
+
+ // This is our final, cleaned, base article URL.
+ return uri.scheme + "://" + uri.host + cleanedSegments.reverse().join("/");
+ },
+
+ /**
+ * Look for any paging links that may occur within the document.
+ *
+ * @param body
+ * @return object (array)
+ **/
+ _findNextPageLink: function(elem) {
+ var uri = this._uri;
+ var possiblePages = {};
+ var allLinks = elem.getElementsByTagName('a');
+ var articleBaseUrl = this._findBaseUrl();
+
+ // Loop through all links, looking for hints that they may be next-page links.
+ // Things like having "page" in their textContent, className or id, or being a child
+ // of a node with a page-y className or id.
+ //
+ // Also possible: levenshtein distance? longest common subsequence?
+ //
+ // After we do that, assign each page a score, and
+ for (var i = 0, il = allLinks.length; i < il; i += 1) {
+ var link = allLinks[i];
+ var linkHref = allLinks[i].href.replace(/#.*$/, '').replace(/\/$/, '');
+
+ // If we've already seen this page, ignore it.
+ if (linkHref === "" ||
+ linkHref === articleBaseUrl ||
+ linkHref === uri.spec ||
+ linkHref in this._parsedPages) {
+ continue;
+ }
+
+ // If it's on a different domain, skip it.
+ if (uri.host !== linkHref.split(/\/+/g)[1])
+ continue;
+
+ var linkText = this._getInnerText(link);
+
+ // If the linkText looks like it's not the next page, skip it.
+ if (linkText.match(this.REGEXPS.extraneous) || linkText.length > 25)
+ continue;
+
+ // If the leftovers of the URL after removing the base URL don't contain
+ // any digits, it's certainly not a next page link.
+ var linkHrefLeftover = linkHref.replace(articleBaseUrl, '');
+ if (!linkHrefLeftover.match(/\d/))
+ continue;
+
+ if (!(linkHref in possiblePages)) {
+ possiblePages[linkHref] = {"score": 0, "linkText": linkText, "href": linkHref};
+ } else {
+ possiblePages[linkHref].linkText += ' | ' + linkText;
+ }
+
+ var linkObj = possiblePages[linkHref];
+
+ // If the articleBaseUrl isn't part of this URL, penalize this link. It could
+ // still be the link, but the odds are lower.
+ // Example: http://www.actionscript.org/resources/articles/745/1/JavaScript-and-VBScript-Injection-in-ActionScript-3/Page1.html
+ if (linkHref.indexOf(articleBaseUrl) !== 0)
+ linkObj.score -= 25;
+
+ var linkData = linkText + ' ' + link.className + ' ' + link.id;
+ if (linkData.match(this.REGEXPS.nextLink))
+ linkObj.score += 50;
+
+ if (linkData.match(/pag(e|ing|inat)/i))
+ linkObj.score += 25;
+
+ if (linkData.match(/(first|last)/i)) {
+ // -65 is enough to negate any bonuses gotten from a > or » in the text,
+ // If we already matched on "next", last is probably fine.
+ // If we didn't, then it's bad. Penalize.
+ if (!linkObj.linkText.match(this.REGEXPS.nextLink))
+ linkObj.score -= 65;
+ }
+
+ if (linkData.match(this.REGEXPS.negative) || linkData.match(this.REGEXPS.extraneous))
+ linkObj.score -= 50;
+
+ if (linkData.match(this.REGEXPS.prevLink))
+ linkObj.score -= 200;
+
+ // If a parentNode contains page or paging or paginat
+ var parentNode = link.parentNode;
+ var positiveNodeMatch = false;
+ var negativeNodeMatch = false;
+
+ while (parentNode) {
+ var parentNodeClassAndId = parentNode.className + ' ' + parentNode.id;
+
+ if (!positiveNodeMatch && parentNodeClassAndId && parentNodeClassAndId.match(/pag(e|ing|inat)/i)) {
+ positiveNodeMatch = true;
+ linkObj.score += 25;
+ }
+
+ if (!negativeNodeMatch && parentNodeClassAndId && parentNodeClassAndId.match(this.REGEXPS.negative)) {
+ // If this is just something like "footer", give it a negative.
+ // If it's something like "body-and-footer", leave it be.
+ if (!parentNodeClassAndId.match(this.REGEXPS.positive)) {
+ linkObj.score -= 25;
+ negativeNodeMatch = true;
+ }
+ }
+
+ parentNode = parentNode.parentNode;
+ }
+
+ // If the URL looks like it has paging in it, add to the score.
+ // Things like /page/2/, /pagenum/2, ?p=3, ?page=11, ?pagination=34
+ if (linkHref.match(/p(a|g|ag)?(e|ing|ination)?(=|\/)[0-9]{1,2}/i) || linkHref.match(/(page|paging)/i))
+ linkObj.score += 25;
+
+ // If the URL contains negative values, give a slight decrease.
+ if (linkHref.match(this.REGEXPS.extraneous))
+ linkObj.score -= 15;
+
+ /**
+ * Minor punishment to anything that doesn't match our current URL.
+ * NOTE: I'm finding this to cause more harm than good where something is exactly 50 points.
+ * Dan, can you show me a counterexample where this is necessary?
+ * if (linkHref.indexOf(window.location.href) !== 0) {
+ * linkObj.score -= 1;
+ * }
+ **/
+
+ // If the link text can be parsed as a number, give it a minor bonus, with a slight
+ // bias towards lower numbered pages. This is so that pages that might not have 'next'
+ // in their text can still get scored, and sorted properly by score.
+ var linkTextAsNumber = parseInt(linkText, 10);
+ if (linkTextAsNumber) {
+ // Punish 1 since we're either already there, or it's probably
+ // before what we want anyways.
+ if (linkTextAsNumber === 1) {
+ linkObj.score -= 10;
+ } else {
+ linkObj.score += Math.max(0, 10 - linkTextAsNumber);
+ }
+ }
+ }
+
+ // Loop thrugh all of our possible pages from above and find our top
+ // candidate for the next page URL. Require at least a score of 50, which
+ // is a relatively high confidence that this page is the next link.
+ var topPage = null;
+ for (var page in possiblePages) {
+ if (possiblePages.hasOwnProperty(page)) {
+ if (possiblePages[page].score >= 50 &&
+ (!topPage || topPage.score < possiblePages[page].score))
+ topPage = possiblePages[page];
+ }
+ }
+
+ if (topPage) {
+ var nextHref = topPage.href.replace(/\/$/,'');
+
+ this.log('NEXT PAGE IS ' + nextHref);
+ this._parsedPages[nextHref] = true;
+ return nextHref;
+ } else {
+ return null;
+ }
+ },
+
+ _successfulRequest: function(request) {
+ return (request.status >= 200 && request.status < 300) ||
+ request.status === 304 ||
+ (request.status === 0 && request.responseText);
+ },
+
+ _ajax: function(url, options) {
+ var request = new XMLHttpRequest();
+
+ function respondToReadyState(readyState) {
+ if (request.readyState === 4) {
+ if (this._successfulRequest(request)) {
+ if (options.success)
+ options.success(request);
+ } else {
+ if (options.error)
+ options.error(request);
+ }
+ }
+ }
+
+ if (typeof options === 'undefined')
+ options = {};
+
+ request.onreadystatechange = respondToReadyState;
+
+ request.open('get', url, true);
+ request.setRequestHeader('Accept', 'text/html');
+
+ try {
+ request.send(options.postBody);
+ } catch (e) {
+ if (options.error)
+ options.error();
+ }
+
+ return request;
+ },
+
+ _appendNextPage: function(nextPageLink) {
+ var doc = this._doc;
+ this._curPageNum += 1;
+
+ var articlePage = doc.createElement("DIV");
+ articlePage.id = 'readability-page-' + this._curPageNum;
+ articlePage.className = 'page';
+ articlePage.innerHTML = ' §
elements with a single
elements are ignored. For example:
+ *
bar
abc
bar
elements have been found and replaced with a
+ //
chain, remove the
s until we hit another element
+ // or non-whitespace. This leaves behind the first
in the chain
+ // (which will be replaced with a
chain, replace the remaining
with a
+ // chain.
+ if (replaced) {
+ var p = this._doc.createElement("p");
+ br.parentNode.replaceChild(p, br);
+
+ next = p.nextSibling;
+ while (next) {
+ // If we've hit another
, we're done adding children to this
]*>[ \n\r\t]*){2,}/i,
+ :replaceFontsRe => /<(\/?)font[^>]*>/i,
+ :trimRe => /^\s+|\s+$/,
+ :normalizeRe => /\s{2,}/,
+ :killBreaksRe => /(
(\s| ?)*){1,}/,
+ :videoRe => /http:\/\/(www\.)?(youtube|vimeo)\.com/i
+ }
+
+ attr_accessor :options, :html, :best_candidate, :candidates, :best_candidate_has_image
+
+ def initialize(input, options = {})
+ @options = DEFAULT_OPTIONS.merge(options)
+ @input = input
+
+ if RUBY_VERSION =~ /^(1\.9|2)/ && !@options[:encoding]
+ @input = GuessHtmlEncoding.encode(@input, @options[:html_headers]) unless @options[:do_not_guess_encoding]
+ @options[:encoding] = @input.encoding.to_s
+ end
+
+ @input = @input.gsub(REGEXES[:replaceBrsRe], '
').gsub(REGEXES[:replaceFontsRe], '<\1span>') + @remove_unlikely_candidates = @options[:remove_unlikely_candidates] + @weight_classes = @options[:weight_classes] + @clean_conditionally = @options[:clean_conditionally] + @best_candidate_has_image = true + make_html + handle_exclusions!(@options[:whitelist], @options[:blacklist]) + end + + def images(content=nil, reload=false) + begin + require 'fastimage' + rescue LoadError + raise "Please install fastimage in order to use the #images feature." + end + + @best_candidate_has_image = false if reload + + prepare_candidates + list_images = [] + tested_images = [] + content = @best_candidate[:elem] unless reload + + return list_images if content.nil? + elements = content.css("img").map(&:attributes) + + elements.each do |element| + next unless element["src"] + + url = element["src"].value + height = element["height"].nil? ? 0 : element["height"].value.to_i + width = element["width"].nil? ? 0 : element["width"].value.to_i + + if url =~ /\Ahttps?:\/\//i && (height.zero? || width.zero?) + image = get_image_size(url) + next unless image + else + image = {:width => width, :height => height} + end + + image[:format] = File.extname(url).gsub(".", "") + + if tested_images.include?(url) + debug("Image was tested: #{url}") + next + end + + tested_images.push(url) + if image_meets_criteria?(image) + list_images << url + else + debug("Image discarded: #{url} - height: #{image[:height]} - width: #{image[:width]} - format: #{image[:format]}") + end + end + + (list_images.empty? and content != @html) ? images(@html, true) : list_images + end + + def images_with_fqdn_uris!(source_uri) + images_with_fqdn_uris(@html, source_uri) + end + + def images_with_fqdn_uris(document = @html.dup, source_uri) + uri = URI.parse(source_uri) + host = uri.host + scheme = uri.scheme + port = uri.port # defaults to 80 + + base = "#{scheme}://#{host}:#{port}/" + + images = [] + document.css("img").each do |elem| + begin + elem['src'] = URI.join(base,elem['src']).to_s if URI.parse(elem['src']).host == nil + images << elem['src'].to_s + rescue URI::InvalidURIError => exc + elem.remove + end + end + + images(document,true) + end + + def get_image_size(url) + w, h = FastImage.size(url) + raise "Couldn't get size." if w.nil? || h.nil? + {:width => w, :height => h} + rescue => e + debug("Image error: #{e}") + nil + end + + def image_meets_criteria?(image) + return false if options[:ignore_image_format].include?(image[:format].downcase) + image[:width] >= (options[:min_image_width] || 0) && image[:height] >= (options[:min_image_height] || 0) + end + + def title + title = @html.css("title").first + title ? title.text : nil + end + + # Look through the @html document looking for the author + # Precedence Information here on the wiki: (TODO attach wiki URL if it is accepted) + # Returns nil if no author is detected + def author + # Let's grab this author: + # + author_elements = @html.xpath('//meta[@name = "dc.creator"]') + unless author_elements.empty? + author_elements.each do |element| + return element['content'].strip if element['content'] + end + end + + # Now let's try to grab this + #
+ # + author_elements = @html.xpath('//*[contains(@class, "vcard")]//*[contains(@class, "fn")]') + unless author_elements.empty? + author_elements.each do |element| + return element.text.strip if element.text + end + end + + # Now let's try to grab this + # Danny Banks (rel) + # TODO: strip out the (rel)? + author_elements = @html.xpath('//a[@rel = "author"]') + unless author_elements.empty? + author_elements.each do |element| + return element.text.strip if element.text + end + end + + author_elements = @html.xpath('//*[@id = "author"]') + unless author_elements.empty? + author_elements.each do |element| + return element.text.strip if element.text + end + end + end + + def content(remove_unlikely_candidates = :default) + @remove_unlikely_candidates = false if remove_unlikely_candidates == false + + prepare_candidates + article = get_article(@candidates, @best_candidate) + + cleaned_article = sanitize(article, @candidates, options) + if article.text.strip.length < options[:retry_length] + if @remove_unlikely_candidates + @remove_unlikely_candidates = false + elsif @weight_classes + @weight_classes = false + elsif @clean_conditionally + @clean_conditionally = false + else + # nothing we can do + return cleaned_article + end + + make_html + content + else + cleaned_article + end + end + + def get_article(candidates, best_candidate) + # Now that we have the top candidate, look through its siblings for content that might also be related. + # Things like preambles, content split by ads that we removed, etc. + + sibling_score_threshold = [10, best_candidate[:content_score] * 0.2].max + output = Nokogiri::XML::Node.new('div', @html) + best_candidate[:elem].parent.children.each do |sibling| + append = false + append = true if sibling == best_candidate[:elem] + append = true if candidates[sibling] && candidates[sibling][:content_score] >= sibling_score_threshold + + if sibling.name.downcase == "p" + link_density = get_link_density(sibling) + node_content = sibling.text + node_length = node_content.length + + append = if node_length > 80 && link_density < 0.25 + true + elsif node_length < 80 && link_density == 0 && node_content =~ /\.( |$)/ + true + end + end + + if append + sibling_dup = sibling.dup # otherwise the state of the document in processing will change, thus creating side effects + sibling_dup.name = "div" unless %w[div p].include?(sibling.name.downcase) + output << sibling_dup + end + end + + output + end + + def select_best_candidate(candidates) + sorted_candidates = candidates.values.sort { |a, b| b[:content_score] <=> a[:content_score] } + + debug("Top 5 candidates:") + sorted_candidates[0...5].each do |candidate| + debug("Candidate #{candidate[:elem].name}##{candidate[:elem][:id]}.#{candidate[:elem][:class]} with score #{candidate[:content_score]}") + end + + best_candidate = sorted_candidates.first || { :elem => @html.css("body").first, :content_score => 0 } + debug("Best candidate #{best_candidate[:elem].name}##{best_candidate[:elem][:id]}.#{best_candidate[:elem][:class]} with score #{best_candidate[:content_score]}") + + best_candidate + end + + def get_link_density(elem) + link_length = elem.css("a").map(&:text).join("").length + text_length = elem.text.length + link_length / text_length.to_f + end + + def class_weight(e) + weight = 0 + return weight unless @weight_classes + + if e[:class] && e[:class] != "" + weight -= 25 if e[:class] =~ REGEXES[:negativeRe] + weight += 25 if e[:class] =~ REGEXES[:positiveRe] + end + + if e[:id] && e[:id] != "" + weight -= 25 if e[:id] =~ REGEXES[:negativeRe] + weight += 25 if e[:id] =~ REGEXES[:positiveRe] + end + + weight + end + + ELEMENT_SCORES = { + 'div' => 5, + 'blockquote' => 3, + 'form' => -3, + 'th' => -5 + }.freeze + + def score_node(elem) + content_score = class_weight(elem) + content_score += ELEMENT_SCORES.fetch(elem.name.downcase, 0) + { :content_score => content_score, :elem => elem } + end + + def debug(str) + puts str if options[:debug] + end + + def sanitize(node, candidates, options = {}) + node.css("h1, h2, h3, h4, h5, h6").each do |header| + header.remove if class_weight(header) < 0 || get_link_density(header) > 0.33 + end + + node.css("form, object, iframe, embed").each do |elem| + elem.remove + end + + if @options[:remove_empty_nodes] + # removetags that have no text content - this will also remove p tags that contain only images. + node.css("p").each do |elem| + elem.remove if elem.content.strip.empty? + end + end + + # Conditionally clean