shadowfacts
/
MastoSearch
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Assorted stuff I no longer recall

This commit is contained in:
Shadowfacts 2023-07-02 18:10:52 -07:00
parent 29377a31c8
commit ee77ee53f4
3 changed files with 71 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
MastoSearchCore/Sources/MastoSearchCore/APIController.swift
View File

@ -53,7 +53,7 @@ public struct APIController {
}
let response = response as! HTTPURLResponse
guard response.statusCode == 200 else {
completion(.failure(.unexpectedStatusCode(response.statusCode)))
completion(.failure(.unexpectedStatusCode(response.statusCode, response)))
return
}
guard let data = data else {
@ -126,13 +126,17 @@ extension APIController {
public enum RequestRange {
case `default`
case after(String)
case before(String)
var queryParameters: [URLQueryItem] {
switch self {
case .default:
return []
case .after(let id):
return [URLQueryItem(name: "min_id", value: id)]
// 40 is the most mastodon will return at once
return [URLQueryItem(name: "min_id", value: id), URLQueryItem(name: "count", value: "40")]
case .before(let id):
return [URLQueryItem(name: "max_id", value: id), URLQueryItem(name: "count", value: "40")]
}
}
}
@ -176,14 +180,14 @@ extension APIController {
extension APIController {
public enum Error: Swift.Error {
case unexpectedStatusCode(Int)
case unexpectedStatusCode(Int, HTTPURLResponse)
case error(Swift.Error)
case noData
case decoding(Swift.Error)
public var localizedDescription: String {
switch self {
case .unexpectedStatusCode(let code):
case .unexpectedStatusCode(let code, _):
return "Unexpected status code \(code)"
case .error(let inner):
return inner.localizedDescription

11
MastoSearchCore/Sources/MastoSearchCore/DatabaseController.swift
View File

@ -137,10 +137,17 @@ public class DatabaseController {
}
}
public func getNewestStatus(completion: @escaping (Status?) -> Void) {
public func getNewestAndOldestStatuses(completion: @escaping ((Status, Status)?) -> Void) {
queue.inDatabase { db in
let results = try! db.executeQuery("SELECT * FROM statuses ORDER BY published DESC LIMIT 1", values: nil)
completion(StatusSequence(results: results).makeIterator().next())
if let newest = StatusSequence(results: results).makeIterator().next() {
let results2 = try! db.executeQuery("SELECT * FROM statuses ORDER BY published ASC LIMIT 1", values: nil)
// if there was a newest, there must also be an oldest
let oldest = StatusSequence(results: results2).makeIterator().next()!
completion((newest, oldest))
} else {
completion(nil)
}
}
}

67
MastoSearchCore/Sources/MastoSearchCore/SyncController.swift
View File

@ -19,31 +19,50 @@ public class SyncController {
private let logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "Sync")
private var syncTotal = 0
private let dateFormatter = {
let f = ISO8601DateFormatter()
f.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
return f
}()
public func syncStatuses(errorHandler: @escaping (APIController.Error) -> Void) {
DatabaseController.shared.getNewestStatus { status in
guard let status else {
return
DatabaseController.shared.getNewestAndOldestStatuses { results in
if let results {
self.logger.log("Starting sync...")
self.syncTotal = 0
self.syncStatuses(direction: .newer, range: .after(results.0.id), errorHandler: errorHandler)
self.syncStatuses(direction: .older, range: .before(results.1.id), errorHandler: errorHandler)
} else {
self.logger.log("No newest, starting backwards sync...")
self.syncTotal = 0
self.syncStatuses(direction: .older, range: .default, errorHandler: errorHandler)
}
self.logger.log("Starting sync...")
self.syncTotal = 0
self.syncStatuses(range: .after(status.id), errorHandler: errorHandler)
}
}
private func syncStatuses(range: APIController.RequestRange, errorHandler: @escaping (APIController.Error) -> Void) {
private func syncStatuses(direction: Direction, range: APIController.RequestRange, errorHandler: @escaping (APIController.Error) -> Void) {
APIController.shared.getStatuses(range: range) { response in
switch response {
case .failure(let error):
self.logger.error("Erorr syncing statuses: \(String(describing: error), privacy: .public)")
DispatchQueue.main.async {
errorHandler(error)
self.logger.error("Error syncing statuses: \(String(describing: error), privacy: .public)")
if case .unexpectedStatusCode(_, let resp) = error,
resp.value(forHTTPHeaderField: "x-ratelimit-remaining") == "0",
let reset = resp.value(forHTTPHeaderField: "x-ratelimit-reset"),
let date = self.dateFormatter.date(from: reset) {
self.logger.info("Rate limited, continuing at \(date)")
DispatchQueue.main.asyncAfter(deadline: .now() + .seconds(Int(ceil(date.timeIntervalSinceNow)))) {
self.syncStatuses(direction: direction, range: range, errorHandler: errorHandler)
}
} else {
DispatchQueue.main.async {
errorHandler(error)
}
}
case .success(let statuses):
guard statuses.count > 0 else {
DispatchQueue.main.async {
self.logger.log("Finished sync of \(self.syncTotal, privacy: .public) statuses")
self.logger.log("Finished sync of \(self.syncTotal, privacy: .public) \(direction.name) statuses")
self.onSync.send()
}
return
@ -62,7 +81,29 @@ public class SyncController {
self.syncTotal += statuses.count
self.syncStatuses(range: .after(statuses.first!.id), errorHandler: errorHandler)
self.syncStatuses(direction: direction, range: direction.nextRange(statuses: statuses), errorHandler: errorHandler)
}
}
}
enum Direction {
case newer, older
var name: String {
switch self {
case .newer:
return "newer"
case .older:
return "older"
}
}
func nextRange(statuses: [APIController.Status]) -> APIController.RequestRange {
switch self {
case .newer:
return .after(statuses.first!.id)
case .older:
return .before(statuses.last!.id)
}
}
}