shadowfacts
/
HTMLStreamer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Return temporary buffer as a .characterSequence

This commit is contained in:
Shadowfacts 2023-12-23 11:48:31 -05:00
parent f585a6b502
commit 38b1d2949b
2 changed files with 12 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Sources/HTMLStreamer/Tokenizer.swift
View File

@ -30,11 +30,13 @@ struct Tokenizer<Chars: IteratorProtocol<Unicode.Scalar>>: IteratorProtocol {
mutating func next() -> Token? {
switch state {
case .flushingTemporaryBuffer(let returnState):
state = returnState
if temporaryBuffer == nil || temporaryBuffer!.isEmpty {
state = returnState
return next()
} else {
return .character(temporaryBuffer!.unicodeScalars.removeFirst())
var buffer: String? = nil
swap(&buffer, &temporaryBuffer)
return .characterSequence(buffer!)
}
case .endOfFile:
return nil

16
Tests/HTMLStreamerTests/TokenizerTests.swift
View File

@ -17,22 +17,22 @@ final class TokenizerTests: XCTestCase {
}
func testNamedCharacterReferences() {
XCTAssertEqual(tokenize("&amp;"), [.character("&")])
XCTAssertEqual(tokenize("&amp;"), [.characterSequence("&")])
// missing-semicolon-after-character-reference:
XCTAssertEqual(tokenize("&not;in"), [.character("¬"), .characterSequence("in")])
XCTAssertEqual(tokenize("&notin"), [.character("¬"), .characterSequence("in")])
XCTAssertEqual(tokenize("&not;in"), [.characterSequence("¬"), .characterSequence("in")])
XCTAssertEqual(tokenize("&notin"), [.characterSequence("¬"), .characterSequence("in")])
// unknown-named-character-reference:
XCTAssertEqual(tokenize("&notit;"), [.character("¬"), .characterSequence("it;")])
XCTAssertEqual(tokenize("&asdf"), "&asdf".unicodeScalars.map { .character($0) })
XCTAssertEqual(tokenize("&a"), "&a".unicodeScalars.map { .character($0) })
XCTAssertEqual(tokenize("&notit;"), [.characterSequence("¬"), .characterSequence("it;")])
XCTAssertEqual(tokenize("&asdf"), [.characterSequence("&asdf")])
XCTAssertEqual(tokenize("&a"), [.characterSequence("&a")])
// attribute special case
XCTAssertEqual(tokenize("<a a='&nota' />"), [.startTag("a", selfClosing: true, attributes: [Attribute(name: "a", value: "&nota")])])
}
func testNumericCharacterReference() {
XCTAssertEqual(tokenize("&#33;"), [.character("!")])
XCTAssertEqual(tokenize("&#x21;"), [.character("!")])
XCTAssertEqual(tokenize("&#33;"), [.characterSequence("!")])
XCTAssertEqual(tokenize("&#x21;"), [.characterSequence("!")])
}
func testStartTag() {