Process runs of unmodified characters as characterSequence tokens

All the chars we care about are a single scalar, so this avoids spending
time on the grapheme breaking algorithm.

Package.swift

@@ -5,6 +5,10 @@ import PackageDescription
 
 let package = Package(
     name: "HTMLStreamer",
+    platforms: [
+        .iOS(.v13),
+        .macOS(.v10_15),
+    ],
     products: [
         // Products define the executables and libraries a package produces, making them visible to other packages.
         .library(

Sources/HTMLStreamer/AttributedStringConverter.swift

@@ -0,0 +1,431 @@
+//
+//  AttributedStringConverter.swift
+//  HTMLStreamer
+//
+//  Created by Shadowfacts on 11/24/23.
+//
+
+#if os(iOS)
+import UIKit
+#elseif os(macOS)
+import AppKit
+#endif
+
+#if os(iOS)
+private typealias PlatformFont = UIFont
+#elseif os(macOS)
+private typealias PlatformFont = NSFont
+#endif
+
+public struct AttributedStringConverter<Callbacks: AttributedStringCallbacks> {
+    private let configuration: AttributedStringConverterConfiguration
+    private var fontCache: [FontTrait: PlatformFont] = [:]
+    
+    private var tokenizer: Tokenizer<String.UnicodeScalarView.Iterator>!
+    private var str: NSMutableAttributedString!
+    
+    private var actionStack: [ElementAction] = []
+    private var styleStack: [Style] = []
+    // The current run of text w/o styles changing
+    private var currentRun: String = ""
+    
+    public init(configuration: AttributedStringConverterConfiguration) where Callbacks == DefaultCallbacks {
+        self.init(configuration: configuration, callbacks: DefaultCallbacks.self)
+    }
+    
+    public init(configuration: AttributedStringConverterConfiguration, callbacks _: Callbacks.Type = Callbacks.self) {
+        self.configuration = configuration
+    }
+    
+    public mutating func convert(html: String) -> NSAttributedString {
+        tokenizer = Tokenizer(chars: html.unicodeScalars.makeIterator())
+        str = NSMutableAttributedString()
+        
+        actionStack = []
+        styleStack = []
+        currentRun = ""
+        
+        while let token = tokenizer.next() {
+            switch token {
+            case .character(let c):
+                currentRun.unicodeScalars.append(c)
+            case .characterSequence(let s):
+                currentRun.append(s)
+            case .comment:
+                // ignored
+                continue
+            case .startTag(let name, let selfClosing, let attributes):
+                let action = Callbacks.elementAction(name: name, attributes: attributes)
+                actionStack.append(action)
+                // self closing tags are ignored since they have no content
+                if !selfClosing {
+                    handleStartTag(name, attributes: attributes)
+                }
+            case .endTag(let name):
+                handleEndTag(name)
+                // if we have a non-default action for the current element, the run finishes here
+                if actionStack.last != .default {
+                    finishRun()
+                }
+                actionStack.removeLast()
+            case .doctype:
+                // ignored
+                continue
+            }
+        }
+        
+        finishRun()
+        
+        return str
+    }
+    
+    private mutating func handleStartTag(_ name: String, attributes: [HTMLStreamer.Attribute]) {
+        switch name {
+        case "br":
+            currentRun.append("\n")
+        case "a":
+            // we need to always insert in attribute, because we need to always have one
+            // to remove from the stack in handleEndTag
+            // but we only need to finish the run if we have a URL, since otherwise
+            // the final attribute run won't be affected
+            let url = attributes.attributeValue(for: "href").flatMap(Callbacks.makeURL(string:))
+            if url != nil {
+                finishRun()
+            }
+            styleStack.append(.link(url))
+        case "em", "i":
+            finishRun()
+            styleStack.append(.italic)
+        case "strong", "b":
+            finishRun()
+            styleStack.append(.bold)
+        case "del":
+            finishRun()
+            styleStack.append(.strikethrough)
+        case "code":
+            finishRun()
+            styleStack.append(.monospace)
+        case "pre":
+            startBlockElement()
+            finishRun()
+            styleStack.append(.monospace)
+        case "blockquote":
+            startBlockElement()
+            finishRun()
+            styleStack.append(.blockquote)
+        case "p":
+            startBlockElement()
+        case "ol":
+            startBlockElement()
+            finishRun()
+            styleStack.append(.orderedList(nextElementOrdinal: 1))
+        case "ul":
+            startBlockElement()
+            finishRun()
+            styleStack.append(.unorderedList)
+        case "li":
+            if str.length != 0 || !currentRun.isEmpty {
+                currentRun.append("\n")
+            }
+            let marker: String
+            if case .orderedList(let nextElementOrdinal) = styleStack.last {
+                marker = orderedTextList.marker(forItemNumber: nextElementOrdinal)
+                styleStack[styleStack.count - 1] = .orderedList(nextElementOrdinal: nextElementOrdinal + 1)
+            } else if case .unorderedList = styleStack.last {
+                marker = unorderedTextList.marker(forItemNumber: 0)
+            } else {
+                break
+            }
+            currentRun.append("\t\(marker)\t")
+        default:
+            break
+        }
+    }
+    
+    private mutating func startBlockElement() {
+        if str.length != 0 || !currentRun.isEmpty {
+            currentRun.append("\n\n")
+        }
+    }
+    
+    private mutating func handleEndTag(_ name: String) {
+        switch name {
+        case "a":
+            if case .link(.some(_)) = styleStack.last {
+                finishRun()
+            }
+            removeLastStyle(.link)
+        case "em", "i":
+            finishRun()
+            removeLastStyle(.italic)
+        case "strong", "b":
+            finishRun()
+            removeLastStyle(.bold)
+        case "del":
+            finishRun()
+            removeLastStyle(.strikethrough)
+        case "code":
+            finishRun()
+            removeLastStyle(.monospace)
+        case "pre":
+            finishRun()
+            removeLastStyle(.monospace)
+        case "blockquote":
+            finishRun()
+            removeLastStyle(.blockquote)
+        case "ol":
+            finishRun()
+            removeLastStyle(.orderedList)
+        case "ul":
+            finishRun()
+            removeLastStyle(.unorderedList)
+        case "li":
+            finishRun()
+        default:
+            break
+        }
+    }
+    
+    // needed to correctly handle mis-nested tags
+    private mutating func removeLastStyle(_ type: Style.StyleType) {
+        var i = styleStack.index(before: styleStack.endIndex)
+        while i >= styleStack.startIndex {
+            if styleStack[i].type == type {
+                styleStack.remove(at: i)
+                return
+            }
+            styleStack.formIndex(before: &i)
+        }
+    }
+    
+    private lazy var blockquoteParagraphStyle: NSParagraphStyle = {
+        let style = configuration.paragraphStyle.mutableCopy() as! NSMutableParagraphStyle
+        style.headIndent = 32
+        style.firstLineHeadIndent = 32
+        return style
+    }()
+    
+    private lazy var listParagraphStyle: NSParagraphStyle = {
+        let style = configuration.paragraphStyle.mutableCopy() as! NSMutableParagraphStyle
+        // I don't like that I can't just use paragraphStyle.textLists, because it makes the list markers
+        // not use the monospace digit font (it seems to just use whatever font attribute is set for the whole thing),
+        // and it doesn't right align the list markers.
+        // Unfortunately, doing it manually means the list markers are incldued in the selectable text.
+        style.headIndent = 32
+        style.firstLineHeadIndent = 0
+        // Use 2 tab stops, one for the list marker, the second for the content.
+        style.tabStops = [NSTextTab(textAlignment: .right, location: 28), NSTextTab(textAlignment: .natural, location: 32)]
+        return style
+    }()
+    
+    private mutating func finishRun() {
+        guard !currentRun.isEmpty else {
+            return
+        }
+        
+        if actionStack.contains(.skip) {
+            currentRun = ""
+            return
+        } else if case .replace(let replacement) = actionStack.first(where: \.isReplace) {
+            currentRun = replacement
+        }
+        
+        var attributes = [NSAttributedString.Key: Any]()
+        var currentFontTraits: FontTrait = []
+        for style in styleStack {
+            switch style {
+            case .bold:
+                currentFontTraits.insert(.bold)
+            case .italic:
+                currentFontTraits.insert(.italic)
+            case .monospace:
+                currentFontTraits.insert(.monospace)
+            case .link(let url):
+                if let url {
+                    attributes[.link] = url
+                }
+            case .strikethrough:
+                attributes[.strikethroughStyle] = NSUnderlineStyle.single.rawValue
+            case .blockquote:
+                attributes[.paragraphStyle] = blockquoteParagraphStyle
+                currentFontTraits.insert(.italic)
+            case .orderedList, .unorderedList:
+                attributes[.paragraphStyle] = listParagraphStyle
+            }
+        }
+        
+        attributes[.font] = getFont(traits: currentFontTraits)
+        
+        if !attributes.keys.contains(.paragraphStyle) {
+            attributes[.paragraphStyle] = configuration.paragraphStyle
+        }
+        
+        str.append(NSAttributedString(string: currentRun, attributes: attributes))
+        currentRun = ""
+    }
+    
+    private mutating func getFont(traits: FontTrait) -> PlatformFont? {
+        if let cached = fontCache[traits] {
+            return cached
+        }
+        
+        let baseFont = traits.contains(.monospace) ? configuration.monospaceFont : configuration.font
+        var descriptor = baseFont.fontDescriptor
+        if traits.contains(.bold) && traits.contains(.italic),
+           let boldItalic = descriptor.withSymbolicTraits([.traitBold, .traitItalic]) {
+            descriptor = boldItalic
+        } else if traits.contains(.bold),
+           let bold = descriptor.withSymbolicTraits(.traitBold) {
+            descriptor = bold
+        } else if traits.contains(.italic),
+           let italic = descriptor.withSymbolicTraits(.traitItalic) {
+            descriptor = italic
+        }
+        let font = PlatformFont(descriptor: descriptor, size: 0)
+        fontCache[traits] = font
+        return font
+    }
+}
+
+public protocol AttributedStringCallbacks {
+    static func makeURL(string: String) -> URL?
+    static func elementAction(name: String, attributes: [Attribute]) -> ElementAction
+}
+
+public enum ElementAction: Equatable {
+    case `default`
+    case skip
+    case replace(String)
+    
+    var isReplace: Bool {
+        if case .replace(_) = self {
+            true
+        } else {
+            false
+        }
+    }
+}
+
+public extension AttributedStringCallbacks {
+    static func makeURL(string: String) -> URL? {
+        URL(string: string)
+    }
+    static func elementAction(name: String, attributes: [Attribute]) -> ElementAction {
+        .default
+    }
+}
+
+public struct DefaultCallbacks: AttributedStringCallbacks {
+}
+
+public struct AttributedStringConverterConfiguration {
+    #if os(iOS)
+    public var font: UIFont
+    public var monospaceFont: UIFont
+    public var color: UIColor
+    #elseif os(macOS)
+    public var font: NSFont
+    public var monospaceFont: NSFont
+    public var color: NSColor
+    #endif
+    public var paragraphStyle: NSParagraphStyle
+    
+    #if os(iOS)
+    public init(font: UIFont, monospaceFont: UIFont, color: UIColor, paragraphStyle: NSParagraphStyle) {
+        self.font = font
+        self.monospaceFont = monospaceFont
+        self.color = color
+        self.paragraphStyle = paragraphStyle
+    }
+    #elseif os(macOS)
+    public init(font: NSFont, monospaceFont: NSFont, color: NSColor, paragraphStyle: NSParagraphStyle) {
+        self.font = font
+        self.monospaceFont = monospaceFont
+        self.color = color
+        self.paragraphStyle = paragraphStyle
+    }
+    #endif
+}
+
+#if os(macOS)
+private extension NSFontDescriptor {
+    func withSymbolicTraits(_ traits: SymbolicTraits) -> NSFontDescriptor? {
+        let descriptor: NSFontDescriptor = self.withSymbolicTraits(traits)
+        return descriptor
+    }
+}
+private extension NSFontDescriptor.SymbolicTraits {
+    static var traitBold: Self { .bold }
+    static var traitItalic: Self { .italic }
+}
+#endif
+
+private struct FontTrait: OptionSet, Hashable {
+    static let bold = FontTrait(rawValue: 1 << 0)
+    static let italic = FontTrait(rawValue: 1 << 1)
+    static let monospace = FontTrait(rawValue: 1 << 2)
+    
+    let rawValue: Int
+    
+    init(rawValue: Int) {
+        self.rawValue = rawValue
+    }
+}
+
+private enum Style {
+    case bold
+    case italic
+    case monospace
+    case link(URL?)
+    case strikethrough
+    case blockquote
+    case orderedList(nextElementOrdinal: Int)
+    case unorderedList
+    
+    var type: StyleType {
+        switch self {
+        case .bold:
+            return .bold
+        case .italic:
+            return .italic
+        case .monospace:
+            return .monospace
+        case .link(_):
+            return .link
+        case .strikethrough:
+            return .strikethrough
+        case .blockquote:
+            return .blockquote
+        case .orderedList(nextElementOrdinal: _):
+            return .orderedList
+        case .unorderedList:
+            return .unorderedList
+        }
+    }
+    
+    enum StyleType: Equatable {
+        case bold
+        case italic
+        case monospace
+        case link
+        case strikethrough
+        case blockquote
+        case orderedList
+        case unorderedList
+    }
+}
+
+extension Collection where Element == Attribute {
+    public func attributeValue(for name: String) -> String? {
+        first(where: { $0.name == name })?.value
+    }
+}
+
+private let orderedTextList = OrderedNumberTextList(markerFormat: .decimal, options: 0)
+private let unorderedTextList = NSTextList(markerFormat: .disc, options: 0)
+
+private class OrderedNumberTextList: NSTextList {
+    override func marker(forItemNumber itemNumber: Int) -> String {
+        "\(super.marker(forItemNumber: itemNumber))."
+    }
+}

Sources/HTMLStreamer/InlineArray3.swift

@@ -1,220 +0,0 @@
-//
-//  InlineArray3.swift
-//  HTMLStreamer
-//
-//  Created by Shadowfacts on 11/19/23.
-//
-
-import Foundation
-
-/// An array with inline space for up to 3 elements.
-///
-/// If the array grows beyond 3 elements, it will be stored out-of-line.
-/// Once that happens, the array will never return to being stored inline,
-/// since the allocation cost has already been paid.
-struct InlineArray3<E> {
-    private var storage: Storage
-    
-    init() {
-        self.storage = .inline(nil, nil, nil)
-    }
-}
-
-extension InlineArray3 {
-    fileprivate enum Storage {
-        case inline(Element?, Element?, Element?)
-        case array(ContiguousArray<Element>)
-    }
-}
-
-extension InlineArray3: ExpressibleByArrayLiteral {
-    init(arrayLiteral elements: Element...) {
-        switch elements.count {
-        case 0:
-            self.storage = .inline(nil, nil, nil)
-        case 1:
-            self.storage = .inline(elements[0], nil, nil)
-        case 2:
-            self.storage = .inline(elements[0], elements[1], nil)
-        case 3:
-            self.storage = .inline(elements[0], elements[1], elements[2])
-        default:
-            self.storage = .array(.init(elements))
-        }
-    }
-}
-
-extension InlineArray3: MutableCollection {
-    typealias Element = E
-    typealias Index = Int
-    typealias Indices = Range<Int>
-    
-    subscript(position: Int) -> Element {
-        _read {
-            precondition(position < endIndex)
-            switch storage {
-            case .inline(let a, let b, let c):
-                switch position {
-                case 0:
-                    yield a.unsafelyUnwrapped
-                case 1:
-                    yield b.unsafelyUnwrapped
-                case 2:
-                    yield c.unsafelyUnwrapped
-                default:
-                    fatalError("unreachable")
-                }
-            case .array(let arr):
-                yield arr[position]
-            }
-        }
-        _modify {
-            precondition(position < endIndex)
-            switch storage {
-            case .inline(let a, let b, let c):
-                switch position {
-                case 0:
-                    var newValue = a.unsafelyUnwrapped
-                    yield &newValue
-                    storage = .inline(newValue, b, c)
-                case 1:
-                    var newValue = b.unsafelyUnwrapped
-                    yield &newValue
-                    storage = .inline(a, newValue, c)
-                case 2:
-                    var newValue = c.unsafelyUnwrapped
-                    yield &newValue
-                    storage = .inline(a, b, newValue)
-                default:
-                    fatalError("unreachable")
-                }
-            case .array(var arr):
-                yield &arr[position]
-            }
-        }
-    }
-    
-    var startIndex: Int {
-        0
-    }
-    
-    var endIndex: Int {
-        switch storage {
-        case .inline(let a, let b, let c):
-            a == nil ? 0 : b == nil ? 1 : c == nil ? 2 : 3
-        case .array(let arr):
-            arr.endIndex
-        }
-    }
-}
-
-extension InlineArray3: BidirectionalCollection {
-}
-
-extension InlineArray3: RandomAccessCollection {
-}
-
-extension InlineArray3: RangeReplaceableCollection {
-    mutating func replaceSubrange<C>(_ subrange: Range<Int>, with newElements: C) where C: Collection, Element == C.Element {
-        switch storage {
-        case .array(var arr):
-            arr.replaceSubrange(subrange, with: newElements)
-            storage = .array(arr)
-        case .inline(var a, var b, var c):
-            if count - subrange.count + newElements.count <= 3 {
-                // remove elements at subrange indices
-                if subrange.contains(2) {
-                    c = nil
-                }
-                if subrange.contains(1) {
-                    b = c
-                    c = nil
-                }
-                if subrange.contains(0) {
-                    a = b
-                    b = c
-                    c = nil
-                }
-                
-                // insert newElements starting at subrange.lowerBound
-                for (offset, el) in newElements.enumerated() {
-                    // assert that we have space to insert
-                    assert(c == nil)
-                    let newIndex = subrange.lowerBound + offset
-                    switch newIndex {
-                    case 2:
-                        c = el
-                    case 1:
-                        c = b
-                        b = el
-                    case 0:
-                        c = b
-                        b = a
-                        a = el
-                    default:
-                        fatalError("unreachable")
-                    }
-                }
-                
-                storage = .inline(a, b, c)
-            } else {
-                var arr: ContiguousArray = if let a {
-                    if let b {
-                        if let c {
-                            [a, b, c]
-                        } else {
-                            [a, b]
-                        }
-                    } else {
-                        [a]
-                    }
-                } else {
-                    []
-                }
-                arr.replaceSubrange(subrange, with: newElements)
-                storage = .array(arr)
-            }
-        }
-    }
-}
-
-private extension Collection {
-    func safeIndex(_ index: Index, offsetBy: Int) -> Index? {
-        var index = index
-        var offsetBy = offsetBy
-        while offsetBy > 0 {
-            if index < endIndex {
-                formIndex(after: &index)
-                offsetBy -= 1
-            } else {
-                return nil
-            }
-        }
-        return index >= endIndex ? nil : index
-    }
-}
-
-extension InlineArray3.Storage: Equatable where E: Equatable {
-}
-
-extension InlineArray3: Equatable where E: Equatable {
-}
-
-extension InlineArray3: CustomStringConvertible {
-    var description: String {
-        switch storage {
-        case .inline(nil, nil, nil):
-            return "[]"
-        case .inline(.some(let a), nil, nil):
-            return "[\(a)]"
-        case .inline(.some(let a), .some(let b), nil):
-            return "[\(a), \(b)]"
-        case .inline(.some(let a), .some(let b), .some(let c)):
-            return "[\(a), \(b), \(c)]"
-        case .inline(_, _, _):
-            fatalError("InlineArray3 invariant violated")
-        case .array(let arr):
-            return arr.description
-        }
-    }
-}

Tests/HTMLStreamerTests/AttributedStringConverterTests.swift

@@ -0,0 +1,217 @@
+//
+//  AttributedStringConverterTests.swift
+//  
+//
+//  Created by Shadowfacts on 11/24/23.
+//
+
+import XCTest
+@testable import HTMLStreamer
+
+final class AttributedStringConverterTests: XCTestCase {
+    
+    #if os(iOS)
+    private let font = UIFont.systemFont(ofSize: 13)
+    private let monospaceFont = UIFont.monospacedSystemFont(ofSize: 13, weight: .regular)
+    #elseif os(macOS)
+    private let font = NSFont.systemFont(ofSize: 13)
+    private lazy var italicFont = NSFont(descriptor: font.fontDescriptor.withSymbolicTraits(.italic), size: 13)!
+    private lazy var boldFont = NSFont(descriptor: font.fontDescriptor.withSymbolicTraits(.bold), size: 13)!
+    private lazy var boldItalicFont = NSFont(descriptor: font.fontDescriptor.withSymbolicTraits([.bold, .italic]), size: 13)!
+    private let monospaceFont = NSFont.monospacedSystemFont(ofSize: 13, weight: .regular)
+    #endif
+    private let blockquoteParagraphStyle: NSParagraphStyle = {
+        let style = NSParagraphStyle.default.mutableCopy() as! NSMutableParagraphStyle
+        style.headIndent = 32
+        style.firstLineHeadIndent = 32
+        return style
+    }()
+    private let listParagraphStyle: NSParagraphStyle = {
+        let style = NSParagraphStyle.default.mutableCopy() as! NSMutableParagraphStyle
+        style.headIndent = 32
+        style.firstLineHeadIndent = 0
+        style.tabStops = [NSTextTab(textAlignment: .right, location: 28), NSTextTab(textAlignment: .natural, location: 32)]
+        return style
+    }()
+    
+    private func convert(_ html: String) -> NSAttributedString {
+        convert(html, callbacks: DefaultCallbacks.self)
+    }
+    
+    private func convert<Callbacks: AttributedStringCallbacks>(_ html: String, callbacks _: Callbacks.Type = Callbacks.self) -> NSAttributedString {
+        let config = AttributedStringConverterConfiguration(
+            font: font,
+            monospaceFont: monospaceFont,
+            color: .black,
+            paragraphStyle: .default
+        )
+        var converter = AttributedStringConverter<Callbacks>(configuration: config)
+        return converter.convert(html: html)
+    }
+
+    func testConvertBR() {
+        XCTAssertEqual(convert("a<br>b"), NSAttributedString(string: "a\nb", attributes: [
+            .font: font,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testConvertA() {
+        XCTAssertEqual(convert("<a href='https://example.com'>link</a>"), NSAttributedString(string: "link", attributes: [
+            .font: font,
+            .paragraphStyle: NSParagraphStyle.default,
+            .link: URL(string: "https://example.com")!,
+        ]))
+        XCTAssertEqual(convert("<a>link</a>"), NSAttributedString(string: "link", attributes: [
+            .font: font,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testConvertP() {
+        XCTAssertEqual(convert("<p>a</p><p>b</p>"), NSAttributedString(string: "a\n\nb", attributes: [
+            .font: font,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testConvertEm() {
+        XCTAssertEqual(convert("<em>hello</em>"), NSAttributedString(string: "hello", attributes: [
+            .font: italicFont,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+        XCTAssertEqual(convert("<i>hello</i>"), NSAttributedString(string: "hello", attributes: [
+            .font: italicFont,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testConvertStrong() {
+        XCTAssertEqual(convert("<strong>hello</strong>"), NSAttributedString(string: "hello", attributes: [
+            .font: boldFont,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+        XCTAssertEqual(convert("<b>hello</b>"), NSAttributedString(string: "hello", attributes: [
+            .font: boldFont,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testConvertBoldItalic() {
+        XCTAssertEqual(convert("<strong><em>hello</em></strong>"), NSAttributedString(string: "hello", attributes: [
+            .font: boldItalicFont,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testIncorrectNesting() {
+        let result = NSMutableAttributedString()
+        result.append(NSAttributedString(string: "bold ", attributes: [
+            .font: boldFont,
+        ]))
+        result.append(NSAttributedString(string: "both", attributes: [
+            .font: boldItalicFont,
+        ]))
+        result.append(NSAttributedString(string: " italic", attributes: [
+            .font: italicFont,
+        ]))
+        result.addAttribute(.paragraphStyle, value: NSParagraphStyle.default, range: NSRange(location: 0, length: result.length))
+        XCTAssertEqual(convert("<strong>bold <em>both</strong> italic</em>"), result)
+    }
+    
+    func testDel() {
+        XCTAssertEqual(convert("<del>blah</del>"), NSAttributedString(string: "blah", attributes: [
+            .font: font,
+            .paragraphStyle: NSParagraphStyle.default,
+            .strikethroughStyle: NSUnderlineStyle.single.rawValue,
+        ]))
+    }
+    
+    func testCode() {
+        XCTAssertEqual(convert("<code>wee</code>"), NSAttributedString(string: "wee", attributes: [
+            .font: monospaceFont,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testPre() {
+        XCTAssertEqual(convert("<pre>wee</pre>"), NSAttributedString(string: "wee", attributes: [
+            .font: monospaceFont,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testBlockquote() {
+        XCTAssertEqual(convert("<blockquote>hello</blockquote>"), NSAttributedString(string: "hello", attributes: [
+            .font: italicFont,
+            .paragraphStyle: blockquoteParagraphStyle,
+        ]))
+        XCTAssertEqual(convert("<blockquote><b>hello</b></blockquote>"), NSAttributedString(string: "hello", attributes: [
+            .font: boldItalicFont,
+            .paragraphStyle: blockquoteParagraphStyle,
+        ]))
+    }
+    
+    func testSelfClosing() {
+        XCTAssertEqual(convert("<b />asdf"), NSAttributedString(string: "asdf", attributes: [
+            .font: font,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testMakeURLCallback() {
+        struct Callbacks: AttributedStringCallbacks {
+            static func makeURL(string: String) -> URL? {
+                URL(string: "https://apple.com")
+            }
+        }
+        let result = convert("<a href='https://example.com'>test</a>", callbacks: Callbacks.self)
+        XCTAssertEqual(result, NSAttributedString(string: "test", attributes: [
+            .font: font,
+            .paragraphStyle: NSParagraphStyle.default,
+            .link: URL(string: "https://apple.com")!,
+        ]))
+    }
+    
+    func testElementActionCallback() {
+        struct Callbacks: AttributedStringCallbacks {
+            static func elementAction(name: String, attributes: [Attribute]) -> ElementAction {
+                let clazz = attributes.attributeValue(for: "class")
+                if clazz == "invisible" {
+                    return .skip
+                } else if clazz == "ellipsis" {
+                    return .replace("…")
+                } else {
+                    return .default
+                }
+            }
+        }
+        let skipped = convert("<span class='invisible'>test</span>", callbacks: Callbacks.self)
+        XCTAssertEqual(skipped, NSAttributedString())
+        let skipNestped = convert("<span class='invisible'><b>test</b></span>", callbacks: Callbacks.self)
+        XCTAssertEqual(skipNestped, NSAttributedString())
+        let skipNestped2 = convert("<b><span class='invisible'>test</span></b>", callbacks: Callbacks.self)
+        XCTAssertEqual(skipNestped2, NSAttributedString())
+        let replaced = convert("<span class='ellipsis'>test</span>", callbacks: Callbacks.self)
+        XCTAssertEqual(replaced, NSAttributedString(string: "…", attributes: [
+            .font: font,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+    func testOrderedList() {
+        let result = convert("<ol><li>a</li><li>b</li></ol>")
+        XCTAssertEqual(result, NSAttributedString(string: "\t1.\ta\n\t2.\tb", attributes: [
+            .font: font,
+            .paragraphStyle: listParagraphStyle,
+        ]))
+    }
+    
+    func testMultiScalar() {
+        XCTAssertEqual(convert("🇺🇸"), NSAttributedString(string: "🇺🇸", attributes: [
+            .font: font,
+            .paragraphStyle: NSParagraphStyle.default,
+        ]))
+    }
+    
+}

Tests/HTMLStreamerTests/InlineArray3Tests.swift

@@ -1,38 +0,0 @@
-//
-//  InlineArray3Tests.swift
-//  
-//
-//  Created by Shadowfacts on 11/19/23.
-//
-
-import XCTest
-@testable import HTMLStreamer
-
-final class InlineArray3Tests: XCTestCase {
-    
-    func testReplaceSubrange() {
-        // same size
-        var a: InlineArray3 = [0, 1, 2]
-        a.replaceSubrange(0..<2, with: [3, 4])
-        XCTAssertEqual(a, [3, 4, 2])
-        
-        // grow
-        a = [0, 1]
-        a.replaceSubrange(1..<2, with: [2, 3])
-        XCTAssertEqual(a, [0, 2, 3])
-        
-        // shrink
-        a = [0, 1, 2]
-        a.replaceSubrange(0..<2, with: [])
-        XCTAssertEqual(a, [2])
-        a.removeFirst()
-        XCTAssertEqual(a, [])
-    }
-
-    func testRemoveLast() {
-        var a: InlineArray3 = [0, 1, 2]
-        a.removeLast(2)
-        XCTAssertEqual(a, [0])
-    }
-    
-}

Tests/HTMLStreamerTests/TokenizerTests.swift

@@ -11,7 +11,7 @@ import XCTest
 final class TokenizerTests: XCTestCase {
     
     private func tokenize(_ s: String) -> [Token] {
-        let iterator = Tokenizer(chars: s.makeIterator())
+        let iterator = Tokenizer(chars: s.unicodeScalars.makeIterator())
 //        let iterator = PrintIterator(inner: Tokenizer(chars: s.makeIterator()))
         return Array(AnySequence({ iterator }))
     }
@@ -19,10 +19,15 @@ final class TokenizerTests: XCTestCase {
     func testNamedCharacterReferences() {
         XCTAssertEqual(tokenize("&"), [.character("&")])
         // missing-semicolon-after-character-reference:
-        XCTAssertEqual(tokenize("¬in"), [.character("¬"), .character("i"), .character("n")])
+        XCTAssertEqual(tokenize("¬in"), [.character("¬"), .characterSequence("in")])
-        XCTAssertEqual(tokenize("&notin"), [.character("∉")])
+        XCTAssertEqual(tokenize("&notin"), [.character("¬"), .characterSequence("in")])
         // unknown-named-character-reference:
-        XCTAssertEqual(tokenize("&asdf"), "&asdf".map { .character($0) })
+        XCTAssertEqual(tokenize("&notit;"), [.character("¬"), .characterSequence("it;")])
+        XCTAssertEqual(tokenize("&asdf"), "&asdf".unicodeScalars.map { .character($0) })
+        XCTAssertEqual(tokenize("&a"), "&a".unicodeScalars.map { .character($0) })
+        
+        // attribute special case
+        XCTAssertEqual(tokenize("<a a='&nota' />"), [.startTag("a", selfClosing: true, attributes: [Attribute(name: "a", value: "&nota")])])
     }
     
     func testNumericCharacterReference() {
@@ -65,6 +70,10 @@ final class TokenizerTests: XCTestCase {
         XCTAssertEqual(tokenize(#"<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">"#), [.doctype("html", forceQuirks: false, publicIdentifier: "-//W3C//DTD HTML 4.01//EN", systemIdentifier: "http://www.w3.org/TR/html4/strict.dtd")])
     }
     
+    func testMultiScalar() {
+        XCTAssertEqual(tokenize("🇺🇸"), [.characterSequence("\u{1F1FA}\u{1F1F8}")])
+    }
+    
 }
 
 private struct PrintIterator<Inner: IteratorProtocol>: IteratorProtocol {