Process runs of unmodified characters as characterSequence tokens

Sources/HTMLStreamer/AttributedStringConverter.swift

@@ -49,6 +49,8 @@ public struct AttributedStringConverter<Callbacks: AttributedStringCallbacks> {
             switch token {
             case .character(let c):
                 currentRun.unicodeScalars.append(c)
+            case .characterSequence(let s):
+                currentRun.append(s)
             case .comment:
                 // ignored
                 continue

Sources/HTMLStreamer/Tokenizer.swift

@@ -212,6 +212,7 @@ struct Tokenizer<Chars: IteratorProtocol<Unicode.Scalar>>: IteratorProtocol {
 
 enum Token: Equatable {
     case character(Unicode.Scalar)
+    case characterSequence(String)
     case comment(String)
     case startTag(String, selfClosing: Bool, attributes: [Attribute])
     case endTag(String)
@@ -326,20 +327,44 @@ private enum DoctypeIdentifierQuotation {
 
 private extension Tokenizer {
     mutating func tokenizeData() -> Token? {
-        switch nextChar() {
+        // Optimization: It's common to have runs of characters that are tokenized as-is,
-        case "&":
+        // so try to return them as a single token so the downstream consumer
-            returnState = .data
+        // can avoid repeated work.
-            state = .characterReference
+        var buf = ""
-            return tokenizeCharacterReference()
+        while true {
-        case "<":
+            switch nextChar() {
-            state = .tagOpen
+            case "&":
-            return tokenizeTagOpen()
+                returnState = .data
-        case "\0":
+                state = .characterReference
-            return .character("\0")
+                if buf.isEmpty {
-        case nil:
+                    return tokenizeCharacterReference()
-            return nil // end of fil
+                } else {
-        case .some(let c):
+                    return .characterSequence(buf)
-            return .character(c)
+                }
+            case "<":
+                state = .tagOpen
+                if buf.isEmpty {
+                    return tokenizeTagOpen()
+                } else {
+                    return .characterSequence(buf)
+                }
+            case "\0":
+                if buf.isEmpty {
+                    return .character("\0")
+                } else {
+                    reconsume("\0")
+                    return .characterSequence(buf)
+                }
+            case nil:
+                if buf.isEmpty {
+                    return nil // end of file
+                } else {
+                    return .characterSequence(buf)
+                }
+            case .some(let c):
+                buf.unicodeScalars.append(c)
+                continue
+            }
         }
     }
     

Tests/HTMLStreamerTests/TokenizerTests.swift

@@ -19,10 +19,10 @@ final class TokenizerTests: XCTestCase {
     func testNamedCharacterReferences() {
         XCTAssertEqual(tokenize("&"), [.character("&")])
         // missing-semicolon-after-character-reference:
-        XCTAssertEqual(tokenize("¬in"), [.character("¬"), .character("i"), .character("n")])
+        XCTAssertEqual(tokenize("¬in"), [.character("¬"), .characterSequence("in")])
-        XCTAssertEqual(tokenize("&notin"), [.character("¬"), .character("i"), .character("n")])
+        XCTAssertEqual(tokenize("&notin"), [.character("¬"), .characterSequence("in")])
         // unknown-named-character-reference:
-        XCTAssertEqual(tokenize("&notit;"), [.character("¬"), .character("i"), .character("t"), .character(";")])
+        XCTAssertEqual(tokenize("&notit;"), [.character("¬"), .characterSequence("it;")])
         XCTAssertEqual(tokenize("&asdf"), "&asdf".unicodeScalars.map { .character($0) })
         XCTAssertEqual(tokenize("&a"), "&a".unicodeScalars.map { .character($0) })
         
@@ -71,7 +71,7 @@ final class TokenizerTests: XCTestCase {
     }
     
     func testMultiScalar() {
-        XCTAssertEqual(tokenize("🇺🇸"), [.character("\u{1F1FA}"), .character("\u{1F1F8}")])
+        XCTAssertEqual(tokenize("🇺🇸"), [.characterSequence("\u{1F1FA}\u{1F1F8}")])
     }
     
 }