diff --git a/MastoSearchCore/Sources/MastoSearchCore/APIController.swift b/MastoSearchCore/Sources/MastoSearchCore/APIController.swift index 90237fc..a2d5df0 100644 --- a/MastoSearchCore/Sources/MastoSearchCore/APIController.swift +++ b/MastoSearchCore/Sources/MastoSearchCore/APIController.swift @@ -53,7 +53,7 @@ public struct APIController { } let response = response as! HTTPURLResponse guard response.statusCode == 200 else { - completion(.failure(.unexpectedStatusCode(response.statusCode))) + completion(.failure(.unexpectedStatusCode(response.statusCode, response))) return } guard let data = data else { @@ -126,13 +126,17 @@ extension APIController { public enum RequestRange { case `default` case after(String) + case before(String) var queryParameters: [URLQueryItem] { switch self { case .default: return [] case .after(let id): - return [URLQueryItem(name: "min_id", value: id)] + // 40 is the most mastodon will return at once + return [URLQueryItem(name: "min_id", value: id), URLQueryItem(name: "count", value: "40")] + case .before(let id): + return [URLQueryItem(name: "max_id", value: id), URLQueryItem(name: "count", value: "40")] } } } @@ -176,14 +180,14 @@ extension APIController { extension APIController { public enum Error: Swift.Error { - case unexpectedStatusCode(Int) + case unexpectedStatusCode(Int, HTTPURLResponse) case error(Swift.Error) case noData case decoding(Swift.Error) public var localizedDescription: String { switch self { - case .unexpectedStatusCode(let code): + case .unexpectedStatusCode(let code, _): return "Unexpected status code \(code)" case .error(let inner): return inner.localizedDescription diff --git a/MastoSearchCore/Sources/MastoSearchCore/DatabaseController.swift b/MastoSearchCore/Sources/MastoSearchCore/DatabaseController.swift index 5674207..f3d99c4 100644 --- a/MastoSearchCore/Sources/MastoSearchCore/DatabaseController.swift +++ b/MastoSearchCore/Sources/MastoSearchCore/DatabaseController.swift @@ -137,10 +137,17 @@ public class DatabaseController { } } - public func getNewestStatus(completion: @escaping (Status?) -> Void) { + public func getNewestAndOldestStatuses(completion: @escaping ((Status, Status)?) -> Void) { queue.inDatabase { db in let results = try! db.executeQuery("SELECT * FROM statuses ORDER BY published DESC LIMIT 1", values: nil) - completion(StatusSequence(results: results).makeIterator().next()) + if let newest = StatusSequence(results: results).makeIterator().next() { + let results2 = try! db.executeQuery("SELECT * FROM statuses ORDER BY published ASC LIMIT 1", values: nil) + // if there was a newest, there must also be an oldest + let oldest = StatusSequence(results: results2).makeIterator().next()! + completion((newest, oldest)) + } else { + completion(nil) + } } } diff --git a/MastoSearchCore/Sources/MastoSearchCore/SyncController.swift b/MastoSearchCore/Sources/MastoSearchCore/SyncController.swift index 5454538..fa24c4f 100644 --- a/MastoSearchCore/Sources/MastoSearchCore/SyncController.swift +++ b/MastoSearchCore/Sources/MastoSearchCore/SyncController.swift @@ -19,31 +19,50 @@ public class SyncController { private let logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "Sync") private var syncTotal = 0 + private let dateFormatter = { + let f = ISO8601DateFormatter() + f.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return f + }() + public func syncStatuses(errorHandler: @escaping (APIController.Error) -> Void) { - DatabaseController.shared.getNewestStatus { status in - guard let status else { - return + DatabaseController.shared.getNewestAndOldestStatuses { results in + if let results { + self.logger.log("Starting sync...") + self.syncTotal = 0 + self.syncStatuses(direction: .newer, range: .after(results.0.id), errorHandler: errorHandler) + self.syncStatuses(direction: .older, range: .before(results.1.id), errorHandler: errorHandler) + } else { + self.logger.log("No newest, starting backwards sync...") + self.syncTotal = 0 + self.syncStatuses(direction: .older, range: .default, errorHandler: errorHandler) } - - self.logger.log("Starting sync...") - self.syncTotal = 0 - self.syncStatuses(range: .after(status.id), errorHandler: errorHandler) } } - private func syncStatuses(range: APIController.RequestRange, errorHandler: @escaping (APIController.Error) -> Void) { + private func syncStatuses(direction: Direction, range: APIController.RequestRange, errorHandler: @escaping (APIController.Error) -> Void) { APIController.shared.getStatuses(range: range) { response in switch response { case .failure(let error): - self.logger.error("Erorr syncing statuses: \(String(describing: error), privacy: .public)") - DispatchQueue.main.async { - errorHandler(error) + self.logger.error("Error syncing statuses: \(String(describing: error), privacy: .public)") + if case .unexpectedStatusCode(_, let resp) = error, + resp.value(forHTTPHeaderField: "x-ratelimit-remaining") == "0", + let reset = resp.value(forHTTPHeaderField: "x-ratelimit-reset"), + let date = self.dateFormatter.date(from: reset) { + self.logger.info("Rate limited, continuing at \(date)") + DispatchQueue.main.asyncAfter(deadline: .now() + .seconds(Int(ceil(date.timeIntervalSinceNow)))) { + self.syncStatuses(direction: direction, range: range, errorHandler: errorHandler) + } + } else { + DispatchQueue.main.async { + errorHandler(error) + } } case .success(let statuses): guard statuses.count > 0 else { DispatchQueue.main.async { - self.logger.log("Finished sync of \(self.syncTotal, privacy: .public) statuses") + self.logger.log("Finished sync of \(self.syncTotal, privacy: .public) \(direction.name) statuses") self.onSync.send() } return @@ -62,7 +81,29 @@ public class SyncController { self.syncTotal += statuses.count - self.syncStatuses(range: .after(statuses.first!.id), errorHandler: errorHandler) + self.syncStatuses(direction: direction, range: direction.nextRange(statuses: statuses), errorHandler: errorHandler) + } + } + } + + enum Direction { + case newer, older + + var name: String { + switch self { + case .newer: + return "newer" + case .older: + return "older" + } + } + + func nextRange(statuses: [APIController.Status]) -> APIController.RequestRange { + switch self { + case .newer: + return .after(statuses.first!.id) + case .older: + return .before(statuses.last!.id) } } }