diff --git a/.eslintignore b/.eslintignore index a34b5181f..80848abc4 100644 --- a/.eslintignore +++ b/.eslintignore @@ -166,6 +166,7 @@ toolkit/components/places/** # Uses preprocessing toolkit/content/contentAreaUtils.js toolkit/components/jsdownloads/src/DownloadIntegration.jsm +toolkit/components/reader/Readerable.jsm toolkit/components/search/nsSearchService.js toolkit/components/url-classifier/** toolkit/components/urlformatter/nsURLFormatter.js diff --git a/browser/base/content/tab-content.js b/browser/base/content/tab-content.js index a99bd5d8b..6729670bd 100644 --- a/browser/base/content/tab-content.js +++ b/browser/base/content/tab-content.js @@ -24,6 +24,8 @@ XPCOMUtils.defineLazyModuleGetter(this, "AboutReader", "resource://gre/modules/AboutReader.jsm"); XPCOMUtils.defineLazyModuleGetter(this, "ReaderMode", "resource://gre/modules/ReaderMode.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "Readerable", + "resource://gre/modules/Readerable.jsm"); XPCOMUtils.defineLazyGetter(this, "SimpleServiceDiscovery", function() { let ssdp = Cu.import("resource://gre/modules/SimpleServiceDiscovery.jsm", {}).SimpleServiceDiscovery; // Register targets @@ -344,7 +346,7 @@ var AboutReaderListener = { * painted is not going to work. */ updateReaderButton: function(forceNonArticle) { - if (!ReaderMode.isEnabledForParseOnLoad || this.isAboutReader || + if (!Readerable.isEnabledForParseOnLoad || this.isAboutReader || !(content.document instanceof content.HTMLDocument) || content.document.mozSyntheticDocument) { return; @@ -385,7 +387,7 @@ var AboutReaderListener = { // Only send updates when there are articles; there's no point updating with // |false| all the time. - if (ReaderMode.isProbablyReaderable(content.document)) { + if (Readerable.isProbablyReaderable(content.document)) { sendAsyncMessage("Reader:UpdateReaderButton", { isArticle: true }); } else if (forceNonArticle) { sendAsyncMessage("Reader:UpdateReaderButton", { isArticle: false }); diff --git a/toolkit/components/reader/JSDOMParser.js b/toolkit/components/reader/JSDOMParser.js index 7d8b225e0..e19172fc6 100644 --- a/toolkit/components/reader/JSDOMParser.js +++ b/toolkit/components/reader/JSDOMParser.js @@ -1,10 +1,4 @@ -/* - * DO NOT MODIFY THIS FILE DIRECTLY! - * - * This is a shared library that is maintained in an external repo: - * https://github.com/mozilla/readability - */ - +/*eslint-env es6:false*/ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this file, * You can obtain one at http://mozilla.org/MPL/2.0/. */ @@ -33,10 +27,6 @@ */ (function (global) { - function error(m) { - dump("JSDOMParser error: " + m + "\n"); - } - // XML only defines these and the numeric ones: var entityTable = { @@ -463,16 +453,15 @@ else this.children.push(newNode); } - } else { + } else if (oldNode.nodeType === Node.ELEMENT_NODE) { // new node is not an element node. // if the old one was, update its element siblings: - if (oldNode.nodeType === Node.ELEMENT_NODE) { - if (oldNode.previousElementSibling) - oldNode.previousElementSibling.nextElementSibling = oldNode.nextElementSibling; - if (oldNode.nextElementSibling) - oldNode.nextElementSibling.previousElementSibling = oldNode.previousElementSibling; - this.children.splice(this.children.indexOf(oldNode), 1); - } + if (oldNode.previousElementSibling) + oldNode.previousElementSibling.nextElementSibling = oldNode.nextElementSibling; + if (oldNode.nextElementSibling) + oldNode.nextElementSibling.previousElementSibling = oldNode.previousElementSibling; + this.children.splice(this.children.indexOf(oldNode), 1); + // If the old node wasn't an element, neither the new nor the old node was an element, // and the children array and its members shouldn't need any updating. } @@ -492,8 +481,8 @@ __JSDOMParser__: true, }; - for (var i in nodeTypes) { - Node[i] = Node.prototype[i] = nodeTypes[i]; + for (var nodeType in nodeTypes) { + Node[nodeType] = Node.prototype[nodeType] = nodeTypes[nodeType]; } var Attribute = function (name, value) { @@ -507,17 +496,9 @@ }, setValue: function(newValue) { this._value = newValue; - delete this._decodedValue; }, - setDecodedValue: function(newValue) { - this._value = encodeHTML(newValue); - this._decodedValue = newValue; - }, - getDecodedValue: function() { - if (typeof this._decodedValue === "undefined") { - this._decodedValue = (this._value && decodeHTML(this._value)) || ""; - } - return this._decodedValue; + getEncodedValue: function() { + return encodeHTML(this._value); }, }; @@ -562,9 +543,10 @@ this._textContent = newText; delete this._innerHTML; }, - } + }; - var Document = function () { + var Document = function (url) { + this.documentURI = url; this.styleSheets = []; this.childNodes = []; this.children = []; @@ -604,9 +586,30 @@ node.textContent = text; return node; }, + + get baseURI() { + if (!this.hasOwnProperty("_baseURI")) { + this._baseURI = this.documentURI; + var baseElements = this.getElementsByTagName("base"); + var href = baseElements[0] && baseElements[0].getAttribute("href"); + if (href) { + try { + this._baseURI = (new URL(href, this._baseURI)).href; + } catch (ex) {/* Just fall back to documentURI */} + } + } + return this._baseURI; + }, }; var Element = function (tag) { + // We use this to find the closing tag. + this._matchingTag = tag; + // We're explicitly a non-namespace aware parser, we just pretend it's all HTML. + var lastColonIndex = tag.lastIndexOf(":"); + if (lastColonIndex != -1) { + tag = tag.substring(lastColonIndex + 1); + } this.attributes = []; this.childNodes = []; this.children = []; @@ -655,6 +658,14 @@ this.setAttribute("src", str); }, + get srcset() { + return this.getAttribute("srcset") || ""; + }, + + set srcset(str) { + this.setAttribute("srcset", str); + }, + get nodeName() { return this.tagName; }, @@ -671,14 +682,14 @@ for (var j = 0; j < child.attributes.length; j++) { var attr = child.attributes[j]; // the attribute value will be HTML escaped. - var val = attr.value; + var val = attr.getEncodedValue(); var quote = (val.indexOf('"') === -1 ? '"' : "'"); - arr.push(" " + attr.name + '=' + quote + val + quote); + arr.push(" " + attr.name + "=" + quote + val + quote); } - if (child.localName in voidElems) { + if (child.localName in voidElems && !child.childNodes.length) { // if this is a self-closing element, end it here - arr.push(">"); + arr.push("/>"); } else { // otherwise, add its children arr.push(">"); @@ -702,12 +713,13 @@ set innerHTML(html) { var parser = new JSDOMParser(); var node = parser.parse(html); - for (var i = this.childNodes.length; --i >= 0;) { + var i; + for (i = this.childNodes.length; --i >= 0;) { this.childNodes[i].parentNode = null; } this.childNodes = node.childNodes; this.children = node.children; - for (var i = this.childNodes.length; --i >= 0;) { + for (i = this.childNodes.length; --i >= 0;) { this.childNodes[i].parentNode = this; } }, @@ -748,8 +760,9 @@ getAttribute: function (name) { for (var i = this.attributes.length; --i >= 0;) { var attr = this.attributes[i]; - if (attr.name === name) - return attr.getDecodedValue(); + if (attr.name === name) { + return attr.value; + } } return undefined; }, @@ -758,11 +771,11 @@ for (var i = this.attributes.length; --i >= 0;) { var attr = this.attributes[i]; if (attr.name === name) { - attr.setDecodedValue(value); + attr.setValue(value); return; } } - this.attributes.push(new Attribute(name, encodeHTML(value))); + this.attributes.push(new Attribute(name, value)); }, removeAttribute: function (name) { @@ -773,7 +786,13 @@ break; } } - } + }, + + hasAttribute: function (name) { + return this.attributes.some(function (attr) { + return attr.name == name; + }); + }, }; var Style = function (node) { @@ -831,7 +850,7 @@ Style.prototype.__defineSetter__(jsName, function (value) { this.setStyle(cssName, value); }); - }) (styleMap[jsName]); + })(styleMap[jsName]); } var JSDOMParser = function () { @@ -849,9 +868,16 @@ // makeElementNode(), which saves us from having to allocate a new array // every time. this.retPair = []; + + this.errorState = ""; }; JSDOMParser.prototype = { + error: function(m) { + dump("JSDOMParser error: " + m + "\n"); + this.errorState += m + "\n"; + }, + /** * Look at the next character without advancing the index. */ @@ -906,14 +932,14 @@ // After a '=', we should see a '"' for the attribute value var c = this.nextChar(); if (c !== '"' && c !== "'") { - error("Error reading attribute " + name + ", expecting '\"'"); + this.error("Error reading attribute " + name + ", expecting '\"'"); return; } // Read the attribute value (and consume the matching quote) var value = this.readString(c); - node.attributes.push(new Attribute(name, value)); + node.attributes.push(new Attribute(name, decodeHTML(value))); return; }, @@ -938,7 +964,7 @@ strBuf.push(c); c = this.nextChar(); } - var tag = strBuf.join(''); + var tag = strBuf.join(""); if (!tag) return false; @@ -949,7 +975,9 @@ while (c !== "/" && c !== ">") { if (c === undefined) return false; - while (whitespace.indexOf(this.html[this.currentChar++]) != -1); + while (whitespace.indexOf(this.html[this.currentChar++]) != -1) { + // Advance cursor to first non-whitespace char. + } this.currentChar--; c = this.nextChar(); if (c !== "/" && c !== ">") { @@ -959,19 +987,19 @@ } // If this is a self-closing tag, read '/>' - var closed = tag in voidElems; + var closed = false; if (c === "/") { closed = true; c = this.nextChar(); if (c !== ">") { - error("expected '>' to close " + tag); + this.error("expected '>' to close " + tag); return false; } } retPair[0] = node; retPair[1] = closed; - return true + return true; }, /** @@ -1013,46 +1041,6 @@ } }, - readScript: function (node) { - while (this.currentChar < this.html.length) { - var c = this.nextChar(); - var nextC = this.peekNext(); - if (c === "<") { - if (nextC === "!" || nextC === "?") { - // We're still before the ! or ? that is starting this comment: - this.currentChar++; - node.appendChild(this.discardNextComment()); - continue; - } - if (nextC === "/" && this.html.substr(this.currentChar, 8 /*"/script>".length */).toLowerCase() == "/script>") { - // Go back before the '<' so we find the end tag. - this.currentChar--; - // Done with this script tag, the caller will close: - return; - } - } - // Either c wasn't a '<' or it was but we couldn't find either a comment - // or a closing script tag, so we should just parse as text until the next one - // comes along: - - var haveTextNode = node.lastChild && node.lastChild.nodeType === Node.TEXT_NODE; - var textNode = haveTextNode ? node.lastChild : new Text(); - var n = this.html.indexOf("<", this.currentChar); - // Decrement this to include the current character *afterwards* so we don't get stuck - // looking for the same < all the time. - this.currentChar--; - if (n === -1) { - textNode.innerHTML += this.html.substring(this.currentChar, this.html.length); - this.currentChar = this.html.length; - } else { - textNode.innerHTML += this.html.substring(this.currentChar, n); - this.currentChar = n; - } - if (!haveTextNode) - node.appendChild(textNode); - } - }, - discardNextComment: function() { if (this.match("--")) { this.discardTo("-->"); @@ -1083,18 +1071,31 @@ return null; // Read any text as Text node + var textNode; if (c !== "<") { --this.currentChar; - var node = new Text(); + textNode = new Text(); var n = this.html.indexOf("<", this.currentChar); if (n === -1) { - node.innerHTML = this.html.substring(this.currentChar, this.html.length); + textNode.innerHTML = this.html.substring(this.currentChar, this.html.length); this.currentChar = this.html.length; } else { - node.innerHTML = this.html.substring(this.currentChar, n); + textNode.innerHTML = this.html.substring(this.currentChar, n); this.currentChar = n; } - return node; + return textNode; + } + + if (this.match("![CDATA[")) { + var endChar = this.html.indexOf("]]>", this.currentChar); + if (endChar === -1) { + this.error("unclosed CDATA section"); + return null; + } + textNode = new Text(); + textNode.textContent = this.html.substring(this.currentChar, endChar); + this.currentChar = endChar + ("]]>").length; + return textNode; } c = this.peekNext(); @@ -1127,14 +1128,10 @@ // If this isn't a void Element, read its child nodes if (!closed) { - if (localName == "script") { - this.readScript(node); - } else { - this.readChildren(node); - } - var closingTag = "" + localName + ">"; + this.readChildren(node); + var closingTag = "" + node._matchingTag + ">"; if (!this.match(closingTag)) { - error("expected '" + closingTag + "'"); + this.error("expected '" + closingTag + "' and got " + this.html.substr(this.currentChar, closingTag.length)); return null; } } @@ -1158,9 +1155,9 @@ /** * Parses an HTML string and returns a JS implementation of the Document. */ - parse: function (html) { + parse: function (html, url) { this.html = html; - var doc = this.doc = new Document(); + var doc = this.doc = new Document(url); this.readChildren(doc); // If this is an HTML document, remove root-level children except for the @@ -1188,4 +1185,4 @@ // Attach JSDOMParser to the global scope global.JSDOMParser = JSDOMParser; -}) (this); +})(this); diff --git a/toolkit/components/reader/Readability-readerable.js b/toolkit/components/reader/Readability-readerable.js new file mode 100644 index 000000000..a813f5fb2 --- /dev/null +++ b/toolkit/components/reader/Readability-readerable.js @@ -0,0 +1,98 @@ +/* eslint-env es6:false */ +/* globals exports */ +/* + * Copyright (c) 2010 Arc90 Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * This code is heavily based on Arc90's readability.js (1.7.1) script + * available at: http://code.google.com/p/arc90labs-readability + */ + +var REGEXPS = { + // NOTE: These two regular expressions are duplicated in + // Readability.js. Please keep both copies in sync. + unlikelyCandidates: /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i, + okMaybeItsACandidate: /and|article|body|column|content|main|shadow/i, +}; + +function isNodeVisible(node) { + // Have to null-check node.style to deal with SVG and MathML nodes. + return (!node.style || node.style.display != "none") && !node.hasAttribute("hidden") + && (!node.hasAttribute("aria-hidden") || node.getAttribute("aria-hidden") != "true"); +} + +/** + * Decides whether or not the document is reader-able without parsing the whole thing. + * + * @return boolean Whether or not we suspect Readability.parse() will suceeed at returning an article object. + */ +function isProbablyReaderable(doc, isVisible) { + if (!isVisible) { + isVisible = isNodeVisible; + } + + var nodes = doc.querySelectorAll("p, pre"); + + // Get
abc
later).
while ((next = this._nextElement(next)) && (next.tagName == "BR")) {
replaced = true;
- var sibling = next.nextSibling;
+ var brSibling = next.nextSibling;
next.parentNode.removeChild(next);
- next = sibling;
+ next = brSibling;
}
// If we removed a
chain, replace the remaining
with a
. Add
@@ -386,16 +501,26 @@ Readability.prototype = {
while (next) {
// If we've hit another
, we're done adding children to this
. if (next.tagName == "BR") { - var nextElem = this._nextElement(next); + var nextElem = this._nextElement(next.nextSibling); if (nextElem && nextElem.tagName == "BR") break; } + if (!this._isPhrasingContent(next)) + break; + // Otherwise, make this node a child of the new
. var sibling = next.nextSibling; p.appendChild(next); next = sibling; } + + while (p.lastChild && this._isWhitespace(p.lastChild)) { + p.removeChild(p.lastChild); + } + + if (p.parentNode.tagName === "P") + this._setNodeTag(p.parentNode, "DIV"); } }); }, @@ -417,7 +542,16 @@ Readability.prototype = { replacement.readability = node.readability; for (var i = 0; i < node.attributes.length; i++) { - replacement.setAttribute(node.attributes[i].name, node.attributes[i].value); + try { + replacement.setAttribute(node.attributes[i].name, node.attributes[i].value); + } catch (ex) { + /* it's possible for setAttribute() to throw if the attribute name + * isn't a valid XML Name. Such attributes can however be parsed from + * source in HTML docs, see https://github.com/whatwg/html/issues/4275, + * so we can hit them here and then throw. We don't care about such + * attributes so we ignore them. + */ + } } return replacement; }, @@ -432,19 +566,58 @@ Readability.prototype = { _prepArticle: function(articleContent) { this._cleanStyles(articleContent); + // Check for data tables before we continue, to avoid removing items in + // those tables, which will often be isolated even though they're + // visually linked to other content-ful elements (text, images, etc.). + this._markDataTables(articleContent); + + this._fixLazyImages(articleContent); + // Clean out junk from the article content this._cleanConditionally(articleContent, "form"); + this._cleanConditionally(articleContent, "fieldset"); this._clean(articleContent, "object"); this._clean(articleContent, "embed"); this._clean(articleContent, "h1"); this._clean(articleContent, "footer"); + this._clean(articleContent, "link"); + this._clean(articleContent, "aside"); - // If there is only one h2, they are probably using it as a header - // and not a subheader, so remove it since we already have a header. - if (articleContent.getElementsByTagName('h2').length === 1) - this._clean(articleContent, "h2"); + // Clean out elements with little content that have "share" in their id/class combinations from final top candidates, + // which means we don't remove the top candidates even they have "share". + + var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD; + + this._forEachNode(articleContent.children, function (topCandidate) { + this._cleanMatchedNodes(topCandidate, function (node, matchString) { + return this.REGEXPS.shareElements.test(matchString) && node.textContent.length < shareElementThreshold; + }); + }); + + // If there is only one h2 and its text content substantially equals article title, + // they are probably using it as a header and not a subheader, + // so remove it since we already extract the title separately. + var h2 = articleContent.getElementsByTagName("h2"); + if (h2.length === 1) { + var lengthSimilarRate = (h2[0].textContent.length - this._articleTitle.length) / this._articleTitle.length; + if (Math.abs(lengthSimilarRate) < 0.5) { + var titlesMatch = false; + if (lengthSimilarRate > 0) { + titlesMatch = h2[0].textContent.includes(this._articleTitle); + } else { + titlesMatch = this._articleTitle.includes(h2[0].textContent); + } + if (titlesMatch) { + this._clean(articleContent, "h2"); + } + } + } this._clean(articleContent, "iframe"); + this._clean(articleContent, "input"); + this._clean(articleContent, "textarea"); + this._clean(articleContent, "select"); + this._clean(articleContent, "button"); this._cleanHeaders(articleContent); // Do these last as the previous stuff may have removed junk @@ -454,23 +627,35 @@ Readability.prototype = { this._cleanConditionally(articleContent, "div"); // Remove extra paragraphs - this._forEachNode(articleContent.getElementsByTagName('p'), function(paragraph) { - var imgCount = paragraph.getElementsByTagName('img').length; - var embedCount = paragraph.getElementsByTagName('embed').length; - var objectCount = paragraph.getElementsByTagName('object').length; + this._removeNodes(articleContent.getElementsByTagName("p"), function (paragraph) { + var imgCount = paragraph.getElementsByTagName("img").length; + var embedCount = paragraph.getElementsByTagName("embed").length; + var objectCount = paragraph.getElementsByTagName("object").length; // At this point, nasty iframes have been removed, only remain embedded video ones. - var iframeCount = paragraph.getElementsByTagName('iframe').length; + var iframeCount = paragraph.getElementsByTagName("iframe").length; var totalCount = imgCount + embedCount + objectCount + iframeCount; - if (totalCount === 0 && !this._getInnerText(paragraph, false)) - paragraph.parentNode.removeChild(paragraph); + return totalCount === 0 && !this._getInnerText(paragraph, false); }); - this._forEachNode(articleContent.getElementsByTagName("br"), function(br) { + this._forEachNode(this._getAllNodesWithTag(articleContent, ["br"]), function(br) { var next = this._nextElement(br.nextSibling); if (next && next.tagName == "P") br.parentNode.removeChild(br); }); + + // Remove single-cell tables + this._forEachNode(this._getAllNodesWithTag(articleContent, ["table"]), function(table) { + var tbody = this._hasSingleTagInsideElement(table, "TBODY") ? table.firstElementChild : table; + if (this._hasSingleTagInsideElement(tbody, "TR")) { + var row = tbody.firstElementChild; + if (this._hasSingleTagInsideElement(row, "TD")) { + var cell = row.firstElementChild; + cell = this._setNodeTag(cell, this._everyNode(cell.childNodes, this._isPhrasingContent) ? "P" : "DIV"); + table.parentNode.replaceChild(cell, table); + } + } + }); }, /** @@ -483,35 +668,35 @@ Readability.prototype = { _initializeNode: function(node) { node.readability = {"contentScore": 0}; - switch(node.tagName) { - case 'DIV': + switch (node.tagName) { + case "DIV": node.readability.contentScore += 5; break; - case 'PRE': - case 'TD': - case 'BLOCKQUOTE': + case "PRE": + case "TD": + case "BLOCKQUOTE": node.readability.contentScore += 3; break; - case 'ADDRESS': - case 'OL': - case 'UL': - case 'DL': - case 'DD': - case 'DT': - case 'LI': - case 'FORM': + case "ADDRESS": + case "OL": + case "UL": + case "DL": + case "DD": + case "DT": + case "LI": + case "FORM": node.readability.contentScore -= 3; break; - case 'H1': - case 'H2': - case 'H3': - case 'H4': - case 'H5': - case 'H6': - case 'TH': + case "H1": + case "H2": + case "H3": + case "H4": + case "H5": + case "H6": + case "TH": node.readability.contentScore -= 5; break; } @@ -550,37 +735,6 @@ Readability.prototype = { return node && node.nextElementSibling; }, - /** - * Like _getNextNode, but for DOM implementations with no - * firstElementChild/nextElementSibling functionality... - */ - _getNextNodeNoElementProperties: function(node, ignoreSelfAndKids) { - function nextSiblingEl(n) { - do { - n = n.nextSibling; - } while (n && n.nodeType !== n.ELEMENT_NODE); - return n; - } - // First check for kids if those aren't being ignored - if (!ignoreSelfAndKids && node.children[0]) { - return node.children[0]; - } - // Then for siblings... - var next = nextSiblingEl(node); - if (next) { - return next; - } - // And finally, move up the parent chain *and* find a sibling - // (because this is depth-first traversal, we will have already - // seen the parent nodes themselves). - do { - node = node.parentNode; - if (node) - next = nextSiblingEl(node); - } while (node && !next); - return node && next; - }, - _checkByline: function(node, matchString) { if (this._articleByline) { return false; @@ -588,9 +742,10 @@ Readability.prototype = { if (node.getAttribute !== undefined) { var rel = node.getAttribute("rel"); + var itemprop = node.getAttribute("itemprop"); } - if ((rel === "author" || this.REGEXPS.byline.test(matchString)) && this._isValidByline(node.textContent)) { + if ((rel === "author" || (itemprop && itemprop.indexOf("author") !== -1) || this.REGEXPS.byline.test(matchString)) && this._isValidByline(node.textContent)) { this._articleByline = node.textContent.trim(); return true; } @@ -602,7 +757,7 @@ Readability.prototype = { maxDepth = maxDepth || 0; var i = 0, ancestors = []; while (node.parentNode) { - ancestors.push(node.parentNode) + ancestors.push(node.parentNode); if (maxDepth && ++i === maxDepth) break; node = node.parentNode; @@ -631,9 +786,6 @@ Readability.prototype = { var pageCacheHtml = page.innerHTML; - // Check if any "dir" is set on the toplevel document element - this._articleDir = doc.documentElement.getAttribute("dir"); - while (true) { var stripUnlikelyCandidates = this._flagIsActive(this.FLAG_STRIP_UNLIKELYS); @@ -646,6 +798,12 @@ Readability.prototype = { while (node) { var matchString = node.className + " " + node.id; + if (!this._isProbablyVisible(node)) { + this.log("Removing hidden node - " + matchString); + node = this._removeAndGetNext(node); + continue; + } + // Check to see if this node is a byline, and remove it if it is. if (this._checkByline(node, matchString)) { node = this._removeAndGetNext(node); @@ -656,6 +814,7 @@ Readability.prototype = { if (stripUnlikelyCandidates) { if (this.REGEXPS.unlikelyCandidates.test(matchString) && !this.REGEXPS.okMaybeItsACandidate.test(matchString) && + !this._hasAncestorTag(node, "table") && node.tagName !== "BODY" && node.tagName !== "A") { this.log("Removing unlikely candidate - " + matchString); @@ -664,34 +823,55 @@ Readability.prototype = { } } + // Remove DIV, SECTION, and HEADER nodes without any content(e.g. text, image, video, or iframe). + if ((node.tagName === "DIV" || node.tagName === "SECTION" || node.tagName === "HEADER" || + node.tagName === "H1" || node.tagName === "H2" || node.tagName === "H3" || + node.tagName === "H4" || node.tagName === "H5" || node.tagName === "H6") && + this._isElementWithoutContent(node)) { + node = this._removeAndGetNext(node); + continue; + } + if (this.DEFAULT_TAGS_TO_SCORE.indexOf(node.tagName) !== -1) { elementsToScore.push(node); } // Turn all divs that don't have children block level elements into p's if (node.tagName === "DIV") { + // Put phrasing content into paragraphs. + var p = null; + var childNode = node.firstChild; + while (childNode) { + var nextSibling = childNode.nextSibling; + if (this._isPhrasingContent(childNode)) { + if (p !== null) { + p.appendChild(childNode); + } else if (!this._isWhitespace(childNode)) { + p = doc.createElement("p"); + node.replaceChild(p, childNode); + p.appendChild(childNode); + } + } else if (p !== null) { + while (p.lastChild && this._isWhitespace(p.lastChild)) { + p.removeChild(p.lastChild); + } + p = null; + } + childNode = nextSibling; + } + // Sites like http://mobile.slate.com encloses each paragraph with a DIV // element. DIVs with only a P element inside and no text content can be // safely converted into plain P elements to avoid confusing the scoring // algorithm with DIVs with are, in practice, paragraphs. - if (this._hasSinglePInsideElement(node)) { + if (this._hasSingleTagInsideElement(node, "P") && this._getLinkDensity(node) < 0.25) { var newNode = node.children[0]; node.parentNode.replaceChild(newNode, node); node = newNode; + elementsToScore.push(node); } else if (!this._hasChildBlockElement(node)) { node = this._setNodeTag(node, "P"); elementsToScore.push(node); - } else { - // EXPERIMENTAL - this._forEachNode(node.childNodes, function(childNode) { - if (childNode.nodeType === Node.TEXT_NODE) { - var p = doc.createElement('p'); - p.textContent = childNode.textContent; - p.style.display = 'inline'; - p.className = 'readability-styled'; - node.replaceChild(p, childNode); - } - }); } } node = this._getNextNode(node); @@ -705,7 +885,7 @@ Readability.prototype = { **/ var candidates = []; this._forEachNode(elementsToScore, function(elementToScore) { - if (!elementToScore.parentNode || typeof(elementToScore.parentNode.tagName) === 'undefined') + if (!elementToScore.parentNode || typeof(elementToScore.parentNode.tagName) === "undefined") return; // If this paragraph is less than 25 characters, don't even count it. @@ -724,17 +904,17 @@ Readability.prototype = { contentScore += 1; // Add points for any commas within this paragraph. - contentScore += innerText.split(',').length; + contentScore += innerText.split(",").length; // For every 100 characters in this paragraph, add another point. Up to 3 points. contentScore += Math.min(Math.floor(innerText.length / 100), 3); // Initialize and score ancestors. this._forEachNode(ancestors, function(ancestor, level) { - if (!ancestor.tagName) + if (!ancestor.tagName || !ancestor.parentNode || typeof(ancestor.parentNode.tagName) === "undefined") return; - if (typeof(ancestor.readability) === 'undefined') { + if (typeof(ancestor.readability) === "undefined") { this._initializeNode(ancestor); candidates.push(ancestor); } @@ -743,7 +923,12 @@ Readability.prototype = { // - parent: 1 (no division) // - grandparent: 2 // - great grandparent+: ancestor level * 3 - var scoreDivider = level === 0 ? 1 : level === 1 ? 2 : level * 3; + if (level === 0) + var scoreDivider = 1; + else if (level === 1) + scoreDivider = 2; + else + scoreDivider = level * 3; ancestor.readability.contentScore += contentScore / scoreDivider; }); }); @@ -760,7 +945,7 @@ Readability.prototype = { var candidateScore = candidate.readability.contentScore * (1 - this._getLinkDensity(candidate)); candidate.readability.contentScore = candidateScore; - this.log('Candidate:', candidate, "with score " + candidateScore); + this.log("Candidate:", candidate, "with score " + candidateScore); for (var t = 0; t < this._nbTopCandidates; t++) { var aTopCandidate = topCandidates[t]; @@ -776,6 +961,7 @@ Readability.prototype = { var topCandidate = topCandidates[0] || null; var neededToCreateTopCandidate = false; + var parentOfTopCandidate; // If we still have no top candidate, just use the body as a last resort. // We also have to copy the body node so it is something we can modify. @@ -795,6 +981,33 @@ Readability.prototype = { this._initializeNode(topCandidate); } else if (topCandidate) { + // Find a better top candidate node if it contains (at least three) nodes which belong to `topCandidates` array + // and whose scores are quite closed with current `topCandidate` node. + var alternativeCandidateAncestors = []; + for (var i = 1; i < topCandidates.length; i++) { + if (topCandidates[i].readability.contentScore / topCandidate.readability.contentScore >= 0.75) { + alternativeCandidateAncestors.push(this._getNodeAncestors(topCandidates[i])); + } + } + var MINIMUM_TOPCANDIDATES = 3; + if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) { + parentOfTopCandidate = topCandidate.parentNode; + while (parentOfTopCandidate.tagName !== "BODY") { + var listsContainingThisAncestor = 0; + for (var ancestorIndex = 0; ancestorIndex < alternativeCandidateAncestors.length && listsContainingThisAncestor < MINIMUM_TOPCANDIDATES; ancestorIndex++) { + listsContainingThisAncestor += Number(alternativeCandidateAncestors[ancestorIndex].includes(parentOfTopCandidate)); + } + if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) { + topCandidate = parentOfTopCandidate; + break; + } + parentOfTopCandidate = parentOfTopCandidate.parentNode; + } + } + if (!topCandidate.readability) { + this._initializeNode(topCandidate); + } + // Because of our bonus system, parents of candidates might have scores // themselves. They get half of the node. There won't be nodes with higher // scores than our topCandidate, but if we see the score going *up* in the first @@ -802,11 +1015,15 @@ Readability.prototype = { // lurking in other places that we want to unify in. The sibling stuff // below does some of that - but only if we've looked high enough up the DOM // tree. - var parentOfTopCandidate = topCandidate.parentNode; + parentOfTopCandidate = topCandidate.parentNode; var lastScore = topCandidate.readability.contentScore; // The scores shouldn't get too low. var scoreThreshold = lastScore / 3; - while (parentOfTopCandidate && parentOfTopCandidate.readability) { + while (parentOfTopCandidate.tagName !== "BODY") { + if (!parentOfTopCandidate.readability) { + parentOfTopCandidate = parentOfTopCandidate.parentNode; + continue; + } var parentScore = parentOfTopCandidate.readability.contentScore; if (parentScore < scoreThreshold) break; @@ -818,6 +1035,17 @@ Readability.prototype = { lastScore = parentOfTopCandidate.readability.contentScore; parentOfTopCandidate = parentOfTopCandidate.parentNode; } + + // If the top candidate is the only child, use parent instead. This will help sibling + // joining logic when adjacent content is actually located in parent's sibling node. + parentOfTopCandidate = topCandidate.parentNode; + while (parentOfTopCandidate.tagName != "BODY" && parentOfTopCandidate.children.length == 1) { + topCandidate = parentOfTopCandidate; + parentOfTopCandidate = topCandidate.parentNode; + } + if (!topCandidate.readability) { + this._initializeNode(topCandidate); + } } // Now that we have the top candidate, look through its siblings for content @@ -828,14 +1056,16 @@ Readability.prototype = { articleContent.id = "readability-content"; var siblingScoreThreshold = Math.max(10, topCandidate.readability.contentScore * 0.2); - var siblings = topCandidate.parentNode.children; + // Keep potential top candidate's parent node to try to get text direction of it later. + parentOfTopCandidate = topCandidate.parentNode; + var siblings = parentOfTopCandidate.children; for (var s = 0, sl = siblings.length; s < sl; s++) { var sibling = siblings[s]; var append = false; - this.log("Looking at sibling node:", sibling, sibling.readability ? ("with score " + sibling.readability.contentScore) : ''); - this.log("Sibling has score", sibling.readability ? sibling.readability.contentScore : 'Unknown'); + this.log("Looking at sibling node:", sibling, sibling.readability ? ("with score " + sibling.readability.contentScore) : ""); + this.log("Sibling has score", sibling.readability ? sibling.readability.contentScore : "Unknown"); if (sibling === topCandidate) { append = true; @@ -856,7 +1086,8 @@ Readability.prototype = { if (nodeLength > 80 && linkDensity < 0.25) { append = true; - } else if (nodeLength < 80 && linkDensity === 0 && nodeContent.search(/\.( |$)/) !== -1) { + } else if (nodeLength < 80 && nodeLength > 0 && linkDensity === 0 && + nodeContent.search(/\.( |$)/) !== -1) { append = true; } } @@ -868,7 +1099,7 @@ Readability.prototype = { if (this.ALTER_TO_DIV_EXCEPTIONS.indexOf(sibling.nodeName) === -1) { // We have a node that isn't a common block level element, like a form or td tag. // Turn it into a div so it doesn't get filtered out later by accident. - this.log("Altering sibling:", sibling, 'to div.'); + this.log("Altering sibling:", sibling, "to div."); sibling = this._setNodeTag(sibling, "DIV"); } @@ -890,47 +1121,78 @@ Readability.prototype = { if (this._debug) this.log("Article content post-prep: " + articleContent.innerHTML); - if (this._curPageNum === 1) { - if (neededToCreateTopCandidate) { - // We already created a fake div thing, and there wouldn't have been any siblings left - // for the previous loop, so there's no point trying to create a new div, and then - // move all the children over. Just assign IDs and class names here. No need to append - // because that already happened anyway. - topCandidate.id = "readability-page-1"; - topCandidate.className = "page"; - } else { - var div = doc.createElement("DIV"); - div.id = "readability-page-1"; - div.className = "page"; - var children = articleContent.childNodes; - while (children.length) { - div.appendChild(children[0]); - } - articleContent.appendChild(div); + if (neededToCreateTopCandidate) { + // We already created a fake div thing, and there wouldn't have been any siblings left + // for the previous loop, so there's no point trying to create a new div, and then + // move all the children over. Just assign IDs and class names here. No need to append + // because that already happened anyway. + topCandidate.id = "readability-page-1"; + topCandidate.className = "page"; + } else { + var div = doc.createElement("DIV"); + div.id = "readability-page-1"; + div.className = "page"; + var children = articleContent.childNodes; + while (children.length) { + div.appendChild(children[0]); } + articleContent.appendChild(div); } if (this._debug) this.log("Article content after paging: " + articleContent.innerHTML); + var parseSuccessful = true; + // Now that we've gone through the full algorithm, check to see if // we got any meaningful content. If we didn't, we may need to re-run // grabArticle with different flags set. This gives us a higher likelihood of // finding the content, and the sieve approach gives us a higher likelihood of // finding the -right- content. - if (this._getInnerText(articleContent, true).length < 500) { + var textLength = this._getInnerText(articleContent, true).length; + if (textLength < this._charThreshold) { + parseSuccessful = false; page.innerHTML = pageCacheHtml; if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) { this._removeFlag(this.FLAG_STRIP_UNLIKELYS); + this._attempts.push({articleContent: articleContent, textLength: textLength}); } else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) { this._removeFlag(this.FLAG_WEIGHT_CLASSES); + this._attempts.push({articleContent: articleContent, textLength: textLength}); } else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) { this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY); + this._attempts.push({articleContent: articleContent, textLength: textLength}); } else { - return null; + this._attempts.push({articleContent: articleContent, textLength: textLength}); + // No luck after removing flags, just return the longest text we found during the different loops + this._attempts.sort(function (a, b) { + return b.textLength - a.textLength; + }); + + // But first check if we actually have something + if (!this._attempts[0].textLength) { + return null; + } + + articleContent = this._attempts[0].articleContent; + parseSuccessful = true; } - } else { + } + + if (parseSuccessful) { + // Find out text direction from ancestors of final top candidate. + var ancestors = [parentOfTopCandidate, topCandidate].concat(this._getNodeAncestors(parentOfTopCandidate)); + this._someNode(ancestors, function(ancestor) { + if (!ancestor.tagName) + return false; + var articleDir = ancestor.getAttribute("dir"); + if (articleDir) { + this._articleDir = articleDir; + return true; + } + return false; + }); return articleContent; } } @@ -945,7 +1207,7 @@ Readability.prototype = { * @return Boolean - whether the input string is a byline. */ _isValidByline: function(byline) { - if (typeof byline == 'string' || byline instanceof String) { + if (typeof byline == "string" || byline instanceof String) { byline = byline.trim(); return (byline.length > 0) && (byline.length < 100); } @@ -962,58 +1224,75 @@ Readability.prototype = { var values = {}; var metaElements = this._doc.getElementsByTagName("meta"); - // Match "description", or Twitter's "twitter:description" (Cards) - // in name attribute. - var namePattern = /^\s*((twitter)\s*:\s*)?(description|title)\s*$/gi; + // property is a space-separated list of values + var propertyPattern = /\s*(dc|dcterm|og|twitter)\s*:\s*(author|creator|description|title|site_name)\s*/gi; - // Match Facebook's Open Graph title & description properties. - var propertyPattern = /^\s*og\s*:\s*(description|title)\s*$/gi; + // name is a single value + var namePattern = /^\s*(?:(dc|dcterm|og|twitter|weibo:(article|webpage))\s*[\.:]\s*)?(author|creator|description|title|site_name)\s*$/i; // Find description tags. this._forEachNode(metaElements, function(element) { var elementName = element.getAttribute("name"); var elementProperty = element.getAttribute("property"); - - if ([elementName, elementProperty].indexOf("author") !== -1) { - metadata.byline = element.getAttribute("content"); + var content = element.getAttribute("content"); + if (!content) { return; } - + var matches = null; var name = null; - if (namePattern.test(elementName)) { - name = elementName; - } else if (propertyPattern.test(elementProperty)) { - name = elementProperty; - } - if (name) { - var content = element.getAttribute("content"); + if (elementProperty) { + matches = elementProperty.match(propertyPattern); + if (matches) { + for (var i = matches.length - 1; i >= 0; i--) { + // Convert to lowercase, and remove any whitespace + // so we can match below. + name = matches[i].toLowerCase().replace(/\s/g, ""); + // multiple authors + values[name] = content.trim(); + } + } + } + if (!matches && elementName && namePattern.test(elementName)) { + name = elementName; if (content) { - // Convert to lowercase and remove any whitespace - // so we can match below. - name = name.toLowerCase().replace(/\s/g, ''); + // Convert to lowercase, remove any whitespace, and convert dots + // to colons so we can match below. + name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":"); values[name] = content.trim(); } } }); - if ("description" in values) { - metadata.excerpt = values["description"]; - } else if ("og:description" in values) { - // Use facebook open graph description. - metadata.excerpt = values["og:description"]; - } else if ("twitter:description" in values) { - // Use twitter cards description. - metadata.excerpt = values["twitter:description"]; + // get title + metadata.title = values["dc:title"] || + values["dcterm:title"] || + values["og:title"] || + values["weibo:article:title"] || + values["weibo:webpage:title"] || + values["title"] || + values["twitter:title"]; + + if (!metadata.title) { + metadata.title = this._getArticleTitle(); } - if ("og:title" in values) { - // Use facebook open graph title. - metadata.title = values["og:title"]; - } else if ("twitter:title" in values) { - // Use twitter cards title. - metadata.title = values["twitter:title"]; - } + // get author + metadata.byline = values["dc:creator"] || + values["dcterm:creator"] || + values["author"]; + + // get description + metadata.excerpt = values["dc:description"] || + values["dcterm:description"] || + values["og:description"] || + values["weibo:article:description"] || + values["weibo:webpage:description"] || + values["description"] || + values["twitter:description"]; + + // get site name + metadata.siteName = values["og:site_name"]; return metadata; }, @@ -1024,39 +1303,42 @@ Readability.prototype = { * @param Element **/ _removeScripts: function(doc) { - this._forEachNode(doc.getElementsByTagName('script'), function(scriptNode) { + this._removeNodes(doc.getElementsByTagName("script"), function(scriptNode) { scriptNode.nodeValue = ""; - scriptNode.removeAttribute('src'); - - if (scriptNode.parentNode) - scriptNode.parentNode.removeChild(scriptNode); - }); - this._forEachNode(doc.getElementsByTagName('noscript'), function(noscriptNode) { - if (noscriptNode.parentNode) - noscriptNode.parentNode.removeChild(noscriptNode); + scriptNode.removeAttribute("src"); + return true; }); + this._removeNodes(doc.getElementsByTagName("noscript")); }, /** - * Check if this node has only whitespace and a single P element + * Check if this node has only whitespace and a single element with given tag * Returns false if the DIV node contains non-empty text nodes - * or if it contains no P or more than 1 element. + * or if it contains no element with given tag or more than 1 element. * * @param Element + * @param string tag of child element **/ - _hasSinglePInsideElement: function(element) { - // There should be exactly 1 element child which is a P: - if (element.children.length != 1 || element.children[0].tagName !== "P") { + _hasSingleTagInsideElement: function(element, tag) { + // There should be exactly 1 element child with given tag + if (element.children.length != 1 || element.children[0].tagName !== tag) { return false; } // And there should be no text nodes with real content return !this._someNode(element.childNodes, function(node) { - return node.nodeType === Node.TEXT_NODE && + return node.nodeType === this.TEXT_NODE && this.REGEXPS.hasContent.test(node.textContent); }); }, + _isElementWithoutContent: function(node) { + return node.nodeType === this.ELEMENT_NODE && + node.textContent.trim().length == 0 && + (node.children.length == 0 || + node.children.length == node.getElementsByTagName("br").length + node.getElementsByTagName("hr").length); + }, + /** * Determine whether element has any children block level elements. * @@ -1069,6 +1351,21 @@ Readability.prototype = { }); }, + /*** + * Determine if a node qualifies as phrasing content. + * https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Content_categories#Phrasing_content + **/ + _isPhrasingContent: function(node) { + return node.nodeType === this.TEXT_NODE || this.PHRASING_ELEMS.indexOf(node.tagName) !== -1 || + ((node.tagName === "A" || node.tagName === "DEL" || node.tagName === "INS") && + this._everyNode(node.childNodes, this._isPhrasingContent)); + }, + + _isWhitespace: function(node) { + return (node.nodeType === this.TEXT_NODE && node.textContent.trim().length === 0) || + (node.nodeType === this.ELEMENT_NODE && node.tagName === "BR"); + }, + /** * Get the inner text of a node - cross browser compatibly. * This also strips out any excess whitespace to be found. @@ -1078,14 +1375,13 @@ Readability.prototype = { * @return string **/ _getInnerText: function(e, normalizeSpaces) { - normalizeSpaces = (typeof normalizeSpaces === 'undefined') ? true : normalizeSpaces; + normalizeSpaces = (typeof normalizeSpaces === "undefined") ? true : normalizeSpaces; var textContent = e.textContent.trim(); if (normalizeSpaces) { return textContent.replace(this.REGEXPS.normalize, " "); - } else { - return textContent; } + return textContent; }, /** @@ -1095,7 +1391,7 @@ Readability.prototype = { * @param string - what to split on. Default is "," * @return number (integer) **/ - _getCharCount: function(e,s) { + _getCharCount: function(e, s) { s = s || ","; return this._getInnerText(e).split(s).length - 1; }, @@ -1108,26 +1404,23 @@ Readability.prototype = { * @return void **/ _cleanStyles: function(e) { - e = e || this._doc; - if (!e) + if (!e || e.tagName.toLowerCase() === "svg") return; - var cur = e.firstChild; - // Remove any root styles, if we're able. - if (typeof e.removeAttribute === 'function' && e.className !== 'readability-styled') - e.removeAttribute('style'); + // Remove `style` and deprecated presentational attributes + for (var i = 0; i < this.PRESENTATIONAL_ATTRIBUTES.length; i++) { + e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[i]); + } - // Go until there are no more child nodes + if (this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.indexOf(e.tagName) !== -1) { + e.removeAttribute("width"); + e.removeAttribute("height"); + } + + var cur = e.firstElementChild; while (cur !== null) { - if (cur.nodeType === cur.ELEMENT_NODE) { - // Remove style attribute(s) : - if (cur.className !== "readability-styled") - cur.removeAttribute("style"); - - this._cleanStyles(cur); - } - - cur = cur.nextSibling; + this._cleanStyles(cur); + cur = cur.nextElementSibling; } }, @@ -1141,7 +1434,7 @@ Readability.prototype = { _getLinkDensity: function(element) { var textLength = this._getInnerText(element).length; if (textLength === 0) - return; + return 0; var linkLength = 0; @@ -1153,371 +1446,6 @@ Readability.prototype = { return linkLength / textLength; }, - /** - * Find a cleaned up version of the current URL, to use for comparing links for possible next-pageyness. - * - * @author Dan Lacy - * @return string the base url - **/ - _findBaseUrl: function() { - var uri = this._uri; - var noUrlParams = uri.path.split("?")[0]; - var urlSlashes = noUrlParams.split("/").reverse(); - var cleanedSegments = []; - var possibleType = ""; - - for (var i = 0, slashLen = urlSlashes.length; i < slashLen; i += 1) { - var segment = urlSlashes[i]; - - // Split off and save anything that looks like a file type. - if (segment.indexOf(".") !== -1) { - possibleType = segment.split(".")[1]; - - // If the type isn't alpha-only, it's probably not actually a file extension. - if (!possibleType.match(/[^a-zA-Z]/)) - segment = segment.split(".")[0]; - } - - // EW-CMS specific segment replacement. Ugly. - // Example: http://www.ew.com/ew/article/0,,20313460_20369436,00.html - if (segment.indexOf(',00') !== -1) - segment = segment.replace(',00', ''); - - // If our first or second segment has anything looking like a page number, remove it. - if (segment.match(/((_|-)?p[a-z]*|(_|-))[0-9]{1,2}$/i) && ((i === 1) || (i === 0))) - segment = segment.replace(/((_|-)?p[a-z]*|(_|-))[0-9]{1,2}$/i, ""); - - var del = false; - - // If this is purely a number, and it's the first or second segment, - // it's probably a page number. Remove it. - if (i < 2 && segment.match(/^\d{1,2}$/)) - del = true; - - // If this is the first segment and it's just "index", remove it. - if (i === 0 && segment.toLowerCase() === "index") - del = true; - - // If our first or second segment is smaller than 3 characters, - // and the first segment was purely alphas, remove it. - if (i < 2 && segment.length < 3 && !urlSlashes[0].match(/[a-z]/i)) - del = true; - - // If it's not marked for deletion, push it to cleanedSegments. - if (!del) - cleanedSegments.push(segment); - } - - // This is our final, cleaned, base article URL. - return uri.scheme + "://" + uri.host + cleanedSegments.reverse().join("/"); - }, - - /** - * Look for any paging links that may occur within the document. - * - * @param body - * @return object (array) - **/ - _findNextPageLink: function(elem) { - var uri = this._uri; - var possiblePages = {}; - var allLinks = elem.getElementsByTagName('a'); - var articleBaseUrl = this._findBaseUrl(); - - // Loop through all links, looking for hints that they may be next-page links. - // Things like having "page" in their textContent, className or id, or being a child - // of a node with a page-y className or id. - // - // Also possible: levenshtein distance? longest common subsequence? - // - // After we do that, assign each page a score, and - for (var i = 0, il = allLinks.length; i < il; i += 1) { - var link = allLinks[i]; - var linkHref = allLinks[i].href.replace(/#.*$/, '').replace(/\/$/, ''); - - // If we've already seen this page, ignore it. - if (linkHref === "" || - linkHref === articleBaseUrl || - linkHref === uri.spec || - linkHref in this._parsedPages) { - continue; - } - - // If it's on a different domain, skip it. - if (uri.host !== linkHref.split(/\/+/g)[1]) - continue; - - var linkText = this._getInnerText(link); - - // If the linkText looks like it's not the next page, skip it. - if (linkText.match(this.REGEXPS.extraneous) || linkText.length > 25) - continue; - - // If the leftovers of the URL after removing the base URL don't contain - // any digits, it's certainly not a next page link. - var linkHrefLeftover = linkHref.replace(articleBaseUrl, ''); - if (!linkHrefLeftover.match(/\d/)) - continue; - - if (!(linkHref in possiblePages)) { - possiblePages[linkHref] = {"score": 0, "linkText": linkText, "href": linkHref}; - } else { - possiblePages[linkHref].linkText += ' | ' + linkText; - } - - var linkObj = possiblePages[linkHref]; - - // If the articleBaseUrl isn't part of this URL, penalize this link. It could - // still be the link, but the odds are lower. - // Example: http://www.actionscript.org/resources/articles/745/1/JavaScript-and-VBScript-Injection-in-ActionScript-3/Page1.html - if (linkHref.indexOf(articleBaseUrl) !== 0) - linkObj.score -= 25; - - var linkData = linkText + ' ' + link.className + ' ' + link.id; - if (linkData.match(this.REGEXPS.nextLink)) - linkObj.score += 50; - - if (linkData.match(/pag(e|ing|inat)/i)) - linkObj.score += 25; - - if (linkData.match(/(first|last)/i)) { - // -65 is enough to negate any bonuses gotten from a > or » in the text, - // If we already matched on "next", last is probably fine. - // If we didn't, then it's bad. Penalize. - if (!linkObj.linkText.match(this.REGEXPS.nextLink)) - linkObj.score -= 65; - } - - if (linkData.match(this.REGEXPS.negative) || linkData.match(this.REGEXPS.extraneous)) - linkObj.score -= 50; - - if (linkData.match(this.REGEXPS.prevLink)) - linkObj.score -= 200; - - // If a parentNode contains page or paging or paginat - var parentNode = link.parentNode; - var positiveNodeMatch = false; - var negativeNodeMatch = false; - - while (parentNode) { - var parentNodeClassAndId = parentNode.className + ' ' + parentNode.id; - - if (!positiveNodeMatch && parentNodeClassAndId && parentNodeClassAndId.match(/pag(e|ing|inat)/i)) { - positiveNodeMatch = true; - linkObj.score += 25; - } - - if (!negativeNodeMatch && parentNodeClassAndId && parentNodeClassAndId.match(this.REGEXPS.negative)) { - // If this is just something like "footer", give it a negative. - // If it's something like "body-and-footer", leave it be. - if (!parentNodeClassAndId.match(this.REGEXPS.positive)) { - linkObj.score -= 25; - negativeNodeMatch = true; - } - } - - parentNode = parentNode.parentNode; - } - - // If the URL looks like it has paging in it, add to the score. - // Things like /page/2/, /pagenum/2, ?p=3, ?page=11, ?pagination=34 - if (linkHref.match(/p(a|g|ag)?(e|ing|ination)?(=|\/)[0-9]{1,2}/i) || linkHref.match(/(page|paging)/i)) - linkObj.score += 25; - - // If the URL contains negative values, give a slight decrease. - if (linkHref.match(this.REGEXPS.extraneous)) - linkObj.score -= 15; - - /** - * Minor punishment to anything that doesn't match our current URL. - * NOTE: I'm finding this to cause more harm than good where something is exactly 50 points. - * Dan, can you show me a counterexample where this is necessary? - * if (linkHref.indexOf(window.location.href) !== 0) { - * linkObj.score -= 1; - * } - **/ - - // If the link text can be parsed as a number, give it a minor bonus, with a slight - // bias towards lower numbered pages. This is so that pages that might not have 'next' - // in their text can still get scored, and sorted properly by score. - var linkTextAsNumber = parseInt(linkText, 10); - if (linkTextAsNumber) { - // Punish 1 since we're either already there, or it's probably - // before what we want anyways. - if (linkTextAsNumber === 1) { - linkObj.score -= 10; - } else { - linkObj.score += Math.max(0, 10 - linkTextAsNumber); - } - } - } - - // Loop thrugh all of our possible pages from above and find our top - // candidate for the next page URL. Require at least a score of 50, which - // is a relatively high confidence that this page is the next link. - var topPage = null; - for (var page in possiblePages) { - if (possiblePages.hasOwnProperty(page)) { - if (possiblePages[page].score >= 50 && - (!topPage || topPage.score < possiblePages[page].score)) - topPage = possiblePages[page]; - } - } - - if (topPage) { - var nextHref = topPage.href.replace(/\/$/,''); - - this.log('NEXT PAGE IS ' + nextHref); - this._parsedPages[nextHref] = true; - return nextHref; - } else { - return null; - } - }, - - _successfulRequest: function(request) { - return (request.status >= 200 && request.status < 300) || - request.status === 304 || - (request.status === 0 && request.responseText); - }, - - _ajax: function(url, options) { - var request = new XMLHttpRequest(); - - function respondToReadyState(readyState) { - if (request.readyState === 4) { - if (this._successfulRequest(request)) { - if (options.success) - options.success(request); - } else { - if (options.error) - options.error(request); - } - } - } - - if (typeof options === 'undefined') - options = {}; - - request.onreadystatechange = respondToReadyState; - - request.open('get', url, true); - request.setRequestHeader('Accept', 'text/html'); - - try { - request.send(options.postBody); - } catch (e) { - if (options.error) - options.error(); - } - - return request; - }, - - _appendNextPage: function(nextPageLink) { - var doc = this._doc; - this._curPageNum += 1; - - var articlePage = doc.createElement("DIV"); - articlePage.id = 'readability-page-' + this._curPageNum; - articlePage.className = 'page'; - articlePage.innerHTML = '
§
'; - - doc.getElementById("readability-content").appendChild(articlePage); - - if (this._curPageNum > this._maxPages) { - var nextPageMarkup = ""; - articlePage.innerHTML = articlePage.innerHTML + nextPageMarkup; - return; - } - - // Now that we've built the article page DOM element, get the page content - // asynchronously and load the cleaned content into the div we created for it. - (function(pageUrl, thisPage) { - this._ajax(pageUrl, { - success: function(r) { - - // First, check to see if we have a matching ETag in headers - if we do, this is a duplicate page. - var eTag = r.getResponseHeader('ETag'); - if (eTag) { - if (eTag in this._pageETags) { - this.log("Exact duplicate page found via ETag. Aborting."); - articlePage.style.display = 'none'; - return; - } else { - this._pageETags[eTag] = 1; - } - } - - // TODO: this ends up doubling up page numbers on NYTimes articles. Need to generically parse those away. - var page = doc.createElement("DIV"); - - // Do some preprocessing to our HTML to make it ready for appending. - // - Remove any script tags. Swap and reswap newlines with a unicode - // character because multiline regex doesn't work in javascript. - // - Turn any noscript tags into divs so that we can parse them. This - // allows us to find any next page links hidden via javascript. - // - Turn all double br's into p's - was handled by prepDocument in the original view. - // Maybe in the future abstract out prepDocument to work for both the original document - // and AJAX-added pages. - var responseHtml = r.responseText.replace(/\n/g,'\uffff').replace(/