From 38b1d2949bb0bca96568b0c9ed2b8c35189d34c3 Mon Sep 17 00:00:00 2001
From: Shadowfacts <me@shadowfacts.net>
Date: Sat, 23 Dec 2023 11:48:31 -0500
Subject: [PATCH] Return temporary buffer as a .characterSequence

---
 Sources/HTMLStreamer/Tokenizer.swift         |  6 ++++--
 Tests/HTMLStreamerTests/TokenizerTests.swift | 16 ++++++++--------
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/Sources/HTMLStreamer/Tokenizer.swift b/Sources/HTMLStreamer/Tokenizer.swift
index 81d7a36..ec07217 100644
--- a/Sources/HTMLStreamer/Tokenizer.swift
+++ b/Sources/HTMLStreamer/Tokenizer.swift
@@ -30,11 +30,13 @@ struct Tokenizer<Chars: IteratorProtocol<Unicode.Scalar>>: IteratorProtocol {
     mutating func next() -> Token? {
         switch state {
         case .flushingTemporaryBuffer(let returnState):
+            state = returnState
             if temporaryBuffer == nil || temporaryBuffer!.isEmpty {
-                state = returnState
                 return next()
             } else {
-                return .character(temporaryBuffer!.unicodeScalars.removeFirst())
+                var buffer: String? = nil
+                swap(&buffer, &temporaryBuffer)
+                return .characterSequence(buffer!)
             }
         case .endOfFile:
             return nil
diff --git a/Tests/HTMLStreamerTests/TokenizerTests.swift b/Tests/HTMLStreamerTests/TokenizerTests.swift
index d110e77..1f3ce8d 100644
--- a/Tests/HTMLStreamerTests/TokenizerTests.swift
+++ b/Tests/HTMLStreamerTests/TokenizerTests.swift
@@ -17,22 +17,22 @@ final class TokenizerTests: XCTestCase {
     }
 
     func testNamedCharacterReferences() {
-        XCTAssertEqual(tokenize("&amp;"), [.character("&")])
+        XCTAssertEqual(tokenize("&amp;"), [.characterSequence("&")])
         // missing-semicolon-after-character-reference:
-        XCTAssertEqual(tokenize("&not;in"), [.character("¬"), .characterSequence("in")])
-        XCTAssertEqual(tokenize("&notin"), [.character("¬"), .characterSequence("in")])
+        XCTAssertEqual(tokenize("&not;in"), [.characterSequence("¬"), .characterSequence("in")])
+        XCTAssertEqual(tokenize("&notin"), [.characterSequence("¬"), .characterSequence("in")])
         // unknown-named-character-reference:
-        XCTAssertEqual(tokenize("&notit;"), [.character("¬"), .characterSequence("it;")])
-        XCTAssertEqual(tokenize("&asdf"), "&asdf".unicodeScalars.map { .character($0) })
-        XCTAssertEqual(tokenize("&a"), "&a".unicodeScalars.map { .character($0) })
+        XCTAssertEqual(tokenize("&notit;"), [.characterSequence("¬"), .characterSequence("it;")])
+        XCTAssertEqual(tokenize("&asdf"), [.characterSequence("&asdf")])
+        XCTAssertEqual(tokenize("&a"), [.characterSequence("&a")])
         
         // attribute special case
         XCTAssertEqual(tokenize("<a a='&nota' />"), [.startTag("a", selfClosing: true, attributes: [Attribute(name: "a", value: "&nota")])])
     }
     
     func testNumericCharacterReference() {
-        XCTAssertEqual(tokenize("&#33;"), [.character("!")])
-        XCTAssertEqual(tokenize("&#x21;"), [.character("!")])
+        XCTAssertEqual(tokenize("&#33;"), [.characterSequence("!")])
+        XCTAssertEqual(tokenize("&#x21;"), [.characterSequence("!")])
     }
     
     func testStartTag() {