JavaScript Highligher: object literal parsing fixes

MongoView/Synax Highlighting/JavaScriptHighlighter.swift

@@ -78,7 +78,24 @@ class JavaScriptHighlighter {
         return attributed
     }
     
+    private func emit(token: TokenType, range: NSRange) {
+        let color: NSColor
+        switch token {
+        case .string:
+            color = .systemRed
+        case .number:
+            color = .systemBlue
+        case .punctuation:
+            color = .systemTeal
+        case .identifier:
+            return
+        }
+        attributed.addAttribute(.foregroundColor, value: color, range: range)
+    }
+
     private func consumeExpression() {
+        consumeWhitespace()
+        
         guard let char = peek() else { return }
         
         if identifierStarts.contains(char) {
@@ -102,6 +119,15 @@ class JavaScriptHighlighter {
         } else {
             consume()
         }
+        
+        consumeWhitespace()
+    }
+    
+    private func consumeWhitespace(newlines: Bool = true) {
+        let charSet = newlines ? CharacterSet.whitespacesAndNewlines : .whitespaces
+        while let char = peek(), charSet.contains(char) {
+            consume()
+        }
     }
 
     private func consumeIdentifier() {
@@ -124,7 +150,7 @@ class JavaScriptHighlighter {
             }
         }
         print("Number: \(text[numberStart..<currentIndex])")
-        attributed.addAttribute(.foregroundColor, value: NSColor.systemBlue, range: range(from: numberStart, to: currentIndex))
+        emit(token: .number, range: range(from: numberStart, to: currentIndex))
     }
     
     private func consumeString() {
@@ -137,7 +163,7 @@ class JavaScriptHighlighter {
             consume() // string closing quote
         }
         print("String: \(text[stringStart..<currentIndex])")
-        attributed.addAttribute(.foregroundColor, value: NSColor.systemRed, range: range(from: stringStart, to: currentIndex))
+        emit(token: .string, range: range(from: stringStart, to: currentIndex))
     }
     
     private func consumeTemplateString() {
@@ -148,7 +174,7 @@ class JavaScriptHighlighter {
                 consume() // $
                 consume() // {
                 print("Template string fragment: '\(text[stringFragmentStart!..<currentIndex])'")
-                attributed.addAttribute(.foregroundColor, value: NSColor.systemRed, range: range(from: stringFragmentStart!, to: currentIndex))
+                emit(token: .string, range: range(from: stringFragmentStart!, to: currentIndex))
                 consumeTemplateStringExpression()
                 stringFragmentStart = currentIndex
                 if currentIndex < text.endIndex && peek() == "}" {
@@ -158,7 +184,7 @@ class JavaScriptHighlighter {
                 stringFragmentStart = stringFragmentStart ?? currentIndex
                 consume() // `
                 print("Template string fragment: '\(text[stringFragmentStart!..<currentIndex])'")
-                attributed.addAttribute(.foregroundColor, value: NSColor.systemRed, range: range(from: stringFragmentStart!, to: currentIndex))
+                emit(token: .string, range: range(from: stringFragmentStart!, to: currentIndex))
                 stringFragmentStart = nil
                 break
             } else {
@@ -167,7 +193,7 @@ class JavaScriptHighlighter {
         }
         if let start = stringFragmentStart {
             print("Template string fragment: '\(text[start..<currentIndex])'")
-            attributed.addAttribute(.foregroundColor, value: NSColor.systemRed, range: range(from: start, to: currentIndex))
+            emit(token: .string, range: range(from: start, to: currentIndex))
         }
     }
     
@@ -187,7 +213,7 @@ class JavaScriptHighlighter {
     private func consumeFunctionCallOrGrouping() {
         consume() // (
         print("Opening (")
-        attributed.addAttribute(.foregroundColor, value: NSColor.systemTeal, range: prevCharRange())
+        emit(token: .punctuation, range: prevCharRange())
         indent += "  "
         while currentIndex < text.endIndex && peek() != ")" {
             consumeExpression()
@@ -196,27 +222,31 @@ class JavaScriptHighlighter {
         if currentIndex < text.endIndex {
             consume() // )
             print("Closing )")
-            attributed.addAttribute(.foregroundColor, value: NSColor.systemTeal, range: prevCharRange())
+            emit(token: .punctuation, range: prevCharRange())
         }
     }
     
     private func consumeObject() {
         consume() // {
         print("Opening {")
-        attributed.addAttribute(.foregroundColor, value: NSColor.systemTeal, range: prevCharRange())
+        emit(token: .punctuation, range: prevCharRange())
         indent += "  "
         object:
         while currentIndex < text.endIndex && peek() != "}" {
             consumeObjectKey()
             if peek() == ":" {
                 consume() // :
+                emit(token: .punctuation, range: prevCharRange())
+                consumeWhitespace()
                 while currentIndex < text.endIndex && peek() != "," && peek() != "}" {
                     consumeExpression()
                 }
-                if currentIndex < text.endIndex {
-                    break object
             }
-            } else if peek() == "," {
+            consumeWhitespace()
+
+            if peek() == "," {
+                consume() // ,
+                emit(token: .punctuation, range: prevCharRange())
                 continue
             } else {
                 break
@@ -226,11 +256,12 @@ class JavaScriptHighlighter {
         if currentIndex < text.endIndex {
             consume() // }
             print("Closing }")
-            attributed.addAttribute(.foregroundColor, value: NSColor.systemTeal, range: prevCharRange())
+            emit(token: .punctuation, range: prevCharRange())
         }
     }
     
     private func consumeObjectKey() {
+        consumeWhitespace()
         guard let char = peek() else { return }
         let keyStart = currentIndex!
         if identifierStarts.contains(char) {
@@ -239,18 +270,19 @@ class JavaScriptHighlighter {
             consumeString()
         }
         print("Object key: '\(text[keyStart..<currentIndex])'")
+        consumeWhitespace()
     }
     
     private func consumeDotLookup() {
         consume() // .
         print("Dot lookup")
-        attributed.addAttribute(.foregroundColor, value: NSColor.systemTeal, range: prevCharRange())
+        emit(token: .punctuation, range: prevCharRange())
     }
     
     private func consumeArray() {
         consume() // [
         print("Opening [")
-        attributed.addAttribute(.foregroundColor, value: NSColor.systemTeal, range: prevCharRange())
+        emit(token: .punctuation, range: prevCharRange())
         indent += "  "
         array:
         while currentIndex < text.endIndex && peek() != "]" {
@@ -258,6 +290,8 @@ class JavaScriptHighlighter {
             while currentIndex < text.endIndex {
                 if peek() == "," {
                     consume() // ,
+                    print("Array separator")
+                    emit(token: .punctuation, range: prevCharRange())
                     break
                 } else if peek() == "]" {
                     break array
@@ -272,8 +306,17 @@ class JavaScriptHighlighter {
         if currentIndex < text.endIndex {
             consume() // ]
             print("Closing ]")
-            attributed.addAttribute(.foregroundColor, value: NSColor.systemTeal, range: prevCharRange())
+            emit(token: .punctuation, range: prevCharRange())
         }
     }
     
 }
+
+extension JavaScriptHighlighter {
+    enum TokenType {
+        case identifier
+        case punctuation
+        case number
+        case string
+    }
+}

jstest/main.swift

@@ -8,6 +8,6 @@
 
 import Foundation
 
-let source = "`+`${{a:3}}`"
+let source = "{a: 1234 , 'b': 'blah', foo}"
 _ = JavaScriptHighlighter(text: source).highlight()