Assorted stuff I no longer recall

MastoSearchCore/Sources/MastoSearchCore/APIController.swift

@@ -53,7 +53,7 @@ public struct APIController {
             }
             let response = response as! HTTPURLResponse
             guard response.statusCode == 200 else {
-                completion(.failure(.unexpectedStatusCode(response.statusCode)))
+                completion(.failure(.unexpectedStatusCode(response.statusCode, response)))
                 return
             }
             guard let data = data else {
@@ -126,13 +126,17 @@ extension APIController {
     public enum RequestRange {
         case `default`
         case after(String)
+        case before(String)
         
         var queryParameters: [URLQueryItem] {
             switch self {
             case .default:
                 return []
             case .after(let id):
-                return [URLQueryItem(name: "min_id", value: id)]
+                // 40 is the most mastodon will return at once
+                return [URLQueryItem(name: "min_id", value: id), URLQueryItem(name: "count", value: "40")]
+            case .before(let id):
+                return [URLQueryItem(name: "max_id", value: id), URLQueryItem(name: "count", value: "40")]
             }
         }
     }
@@ -176,14 +180,14 @@ extension APIController {
 
 extension APIController {
     public enum Error: Swift.Error {
-        case unexpectedStatusCode(Int)
+        case unexpectedStatusCode(Int, HTTPURLResponse)
         case error(Swift.Error)
         case noData
         case decoding(Swift.Error)
         
         public var localizedDescription: String {
             switch self {
-            case .unexpectedStatusCode(let code):
+            case .unexpectedStatusCode(let code, _):
                 return "Unexpected status code \(code)"
             case .error(let inner):
                 return inner.localizedDescription

MastoSearchCore/Sources/MastoSearchCore/DatabaseController.swift

@@ -137,10 +137,17 @@ public class DatabaseController {
         }
     }
     
-    public func getNewestStatus(completion: @escaping (Status?) -> Void) {
+    public func getNewestAndOldestStatuses(completion: @escaping ((Status, Status)?) -> Void) {
         queue.inDatabase { db in
             let results = try! db.executeQuery("SELECT * FROM statuses ORDER BY published DESC LIMIT 1", values: nil)
-            completion(StatusSequence(results: results).makeIterator().next())
+            if let newest = StatusSequence(results: results).makeIterator().next() {
+                let results2 = try! db.executeQuery("SELECT * FROM statuses ORDER BY published ASC LIMIT 1", values: nil)
+                // if there was a newest, there must also be an oldest
+                let oldest = StatusSequence(results: results2).makeIterator().next()!
+                completion((newest, oldest))
+            } else {
+                completion(nil)
+            }
         }
     }
     

MastoSearchCore/Sources/MastoSearchCore/SyncController.swift

@@ -19,31 +19,50 @@ public class SyncController {
     private let logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "Sync")
     private var syncTotal = 0
     
-    public func syncStatuses(errorHandler: @escaping (APIController.Error) -> Void) {
+    private let dateFormatter = {
-        DatabaseController.shared.getNewestStatus { status in
+        let f = ISO8601DateFormatter()
-            guard let status else {
+        f.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
-                return
+        return f
-            }
+    }()
     
+    public func syncStatuses(errorHandler: @escaping (APIController.Error) -> Void) {
+        DatabaseController.shared.getNewestAndOldestStatuses { results in
+            if let results {
                 self.logger.log("Starting sync...")
                 self.syncTotal = 0
-            self.syncStatuses(range: .after(status.id), errorHandler: errorHandler)
+                self.syncStatuses(direction: .newer, range: .after(results.0.id), errorHandler: errorHandler)
+                self.syncStatuses(direction: .older, range: .before(results.1.id), errorHandler: errorHandler)
+            } else {
+                self.logger.log("No newest, starting backwards sync...")
+                self.syncTotal = 0
+                self.syncStatuses(direction: .older, range: .default, errorHandler: errorHandler)
+            }
         }
     }
     
-    private func syncStatuses(range: APIController.RequestRange, errorHandler: @escaping (APIController.Error) -> Void) {
+    private func syncStatuses(direction: Direction, range: APIController.RequestRange, errorHandler: @escaping (APIController.Error) -> Void) {
         APIController.shared.getStatuses(range: range) { response in
             switch response {
             case .failure(let error):
-                self.logger.error("Erorr syncing statuses: \(String(describing: error), privacy: .public)")
+                self.logger.error("Error syncing statuses: \(String(describing: error), privacy: .public)")
+                if case .unexpectedStatusCode(_, let resp) = error,
+                   resp.value(forHTTPHeaderField: "x-ratelimit-remaining") == "0",
+                   let reset = resp.value(forHTTPHeaderField: "x-ratelimit-reset"),
+                   let date = self.dateFormatter.date(from: reset) {
+                    self.logger.info("Rate limited, continuing at \(date)")
+                    DispatchQueue.main.asyncAfter(deadline: .now() + .seconds(Int(ceil(date.timeIntervalSinceNow)))) {
+                        self.syncStatuses(direction: direction, range: range, errorHandler: errorHandler)
+                    }
+                } else {
                     DispatchQueue.main.async {
                         errorHandler(error)
                     }
+                }
                 
             case .success(let statuses):
                 guard statuses.count > 0 else {
                     DispatchQueue.main.async {
-                        self.logger.log("Finished sync of \(self.syncTotal, privacy: .public) statuses")
+                        self.logger.log("Finished sync of \(self.syncTotal, privacy: .public) \(direction.name) statuses")
                         self.onSync.send()
                     }
                     return
@@ -62,7 +81,29 @@ public class SyncController {
                 
                 self.syncTotal += statuses.count
                 
-                self.syncStatuses(range: .after(statuses.first!.id), errorHandler: errorHandler)
+                self.syncStatuses(direction: direction, range: direction.nextRange(statuses: statuses), errorHandler: errorHandler)
+            }
+        }
+    }
+    
+    enum Direction {
+        case newer, older
+        
+        var name: String {
+            switch self {
+            case .newer:
+                return "newer"
+            case .older:
+                return "older"
+            }
+        }
+        
+        func nextRange(statuses: [APIController.Status]) -> APIController.RequestRange {
+            switch self {
+            case .newer:
+                return .after(statuses.first!.id)
+            case .older:
+                return .before(statuses.last!.id)
             }
         }
     }