// // MastodonCachePersistentStore.swift // Tusker // // Created by Shadowfacts on 4/11/20. // Copyright © 2020 Shadowfacts. All rights reserved. // import Foundation import CoreData import Pachyderm import Combine import OSLog import Sentry import CloudKit import UserAccounts fileprivate let logger = Logger(subsystem: Bundle.main.bundleIdentifier!, category: "PersistentStore") class MastodonCachePersistentStore: NSPersistentCloudKitContainer { private let accountInfo: UserAccountInfo? private static let managedObjectModel: NSManagedObjectModel = { let url = Bundle.main.url(forResource: "Tusker", withExtension: "momd")! return NSManagedObjectModel(contentsOf: url)! }() private(set) lazy var backgroundContext: NSManagedObjectContext = { let context = NSManagedObjectContext(concurrencyType: .privateQueueConcurrencyType) context.persistentStoreCoordinator = self.persistentStoreCoordinator context.automaticallyMergesChangesFromParent = true context.mergePolicy = NSMergePolicy.mergeByPropertyObjectTrump return context }() private(set) lazy var prefetchBackgroundContext: NSManagedObjectContext = { let context = NSManagedObjectContext(concurrencyType: .privateQueueConcurrencyType) context.persistentStoreCoordinator = self.persistentStoreCoordinator context.automaticallyMergesChangesFromParent = true context.mergePolicy = NSMergePolicy.mergeByPropertyObjectTrump return context }() private var remoteChangeHandlerQueue = DispatchQueue(label: "PersistentStore remote changes") private var lastRemoteChangeToken: NSPersistentHistoryToken? // TODO: consider sending managed objects through this to avoid re-fetching things unnecessarily // would need to audit existing uses to make sure everything happens on the main thread // and when updating things on the background context would need to switch to main, refetch, and then publish let statusSubject = PassthroughSubject() let accountSubject = PassthroughSubject() let relationshipSubject = PassthroughSubject() init(for accountInfo: UserAccountInfo?, transient: Bool = false) { self.accountInfo = accountInfo let group = DispatchGroup() var instancesToMigrate: [URL]? = nil var hashtagsToMigrate: [Hashtag]? = nil if transient { super.init(name: "transient_cache", managedObjectModel: MastodonCachePersistentStore.managedObjectModel) let storeDescription = NSPersistentStoreDescription() storeDescription.type = NSInMemoryStoreType persistentStoreDescriptions = [storeDescription] } else { super.init(name: "\(accountInfo!.persistenceKey)_cache", managedObjectModel: MastodonCachePersistentStore.managedObjectModel) var localStoreLocation = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! localStoreLocation.appendPathComponent("\(accountInfo!.persistenceKey)_cache.sqlite", isDirectory: false) let localStoreDescription = NSPersistentStoreDescription(url: localStoreLocation) localStoreDescription.configuration = "Local" localStoreDescription.setOption(true as NSNumber, forKey: NSPersistentHistoryTrackingKey) localStoreDescription.setOption(true as NSNumber, forKey: NSPersistentStoreRemoteChangeNotificationPostOptionKey) var cloudStoreLocation = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! cloudStoreLocation.appendPathComponent("cloud.sqlite", isDirectory: false) let cloudStoreDescription = NSPersistentStoreDescription(url: cloudStoreLocation) cloudStoreDescription.configuration = "Cloud" let options = NSPersistentCloudKitContainerOptions(containerIdentifier: "iCloud.space.vaccor.Tusker") options.databaseScope = .private cloudStoreDescription.cloudKitContainerOptions = options cloudStoreDescription.setOption(true as NSNumber, forKey: NSPersistentHistoryTrackingKey) cloudStoreDescription.setOption(true as NSNumber, forKey: NSPersistentStoreRemoteChangeNotificationPostOptionKey) persistentStoreDescriptions = [ cloudStoreDescription, localStoreDescription, ] // workaround for migrating from using id in name to persistenceKey // can be removed after a sufficient time has passed if accountInfo!.id.contains("/") { for desc in persistentStoreDescriptions { guard let new = desc.url, !FileManager.default.fileExists(atPath: new.path) else { continue } do { for ext in ["sqlite", "sqlite-shm", "sqlite-wal"] { var old = new.deletingLastPathComponent() let components = accountInfo!.id.split(separator: "/") for dir in components.dropLast(1) { old.appendPathComponent(String(dir), isDirectory: true) } old.appendPathComponent("\(components.last!)_cache", isDirectory: false) old.appendPathExtension(ext) if FileManager.default.fileExists(atPath: old.path) { var expected = new.deletingLastPathComponent() expected.appendPathComponent("\(accountInfo!.persistenceKey)_cache", isDirectory: false) expected.appendPathExtension(ext) try FileManager.default.moveItem(at: old, to: expected) } } } catch {} } } // migrate saved data from local store to cloud store // this can be removed pre-app store release var defaultPath = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! defaultPath.appendPathComponent("\(accountInfo!.persistenceKey)_cache.sqlite", isDirectory: false) if FileManager.default.fileExists(atPath: defaultPath.path) { group.enter() let defaultDesc = NSPersistentStoreDescription(url: defaultPath) defaultDesc.configuration = "Default" let defaultPSC = NSPersistentContainer(name: "\(accountInfo!.persistenceKey)_cache", managedObjectModel: MastodonCachePersistentStore.managedObjectModel) defaultPSC.persistentStoreDescriptions = [defaultDesc] defaultPSC.loadPersistentStores { _, error in guard error == nil else { group.leave() return } defaultPSC.performBackgroundTask { context in if let instances = try? context.fetch(SavedInstance.fetchRequestWithoutAccountForMigrating()) { instancesToMigrate = instances.map(\.url) instances.forEach(context.delete(_:)) } if let hashtags = try? context.fetch(SavedHashtag.fetchRequestWithoutAccountForMigrating()) { hashtagsToMigrate = hashtags.map { Hashtag(name: $0.name, url: $0.url) } hashtags.forEach(context.delete(_:)) } if context.hasChanges { try? context.save() } group.leave() } } } } group.wait() loadPersistentStores { (description, error) in if let error = error { logger.error("Unable to load persistent store: \(String(describing: error), privacy: .public)") fatalError("Unable to load persistent store: \(String(describing: error))") } if description.configuration == "Cloud" { let context = self.backgroundContext context.perform { instancesToMigrate?.forEach({ url in if !context.objectExists(for: SavedInstance.fetchRequest(url: url, account: accountInfo!)) { _ = SavedInstance(url: url, account: accountInfo!, context: self.backgroundContext) } }) hashtagsToMigrate?.forEach({ hashtag in if !context.objectExists(for: SavedHashtag.fetchRequest(name: hashtag.name, account: accountInfo!)) { _ = SavedHashtag(hashtag: hashtag, account: accountInfo!, context: self.backgroundContext) } }) self.save(context: self.backgroundContext) } } } // changes to the Cloud CD model in development need this to be uncommented to update the CK schema // #if DEBUG // try! initializeCloudKitSchema(options: []) // #endif viewContext.automaticallyMergesChangesFromParent = true viewContext.mergePolicy = NSMergePolicy.mergeByPropertyObjectTrump NotificationCenter.default.addObserver(self, selector: #selector(managedObjectsDidChange), name: .NSManagedObjectContextObjectsDidChange, object: viewContext) NotificationCenter.default.addObserver(self, selector: #selector(remoteChanges), name: .NSPersistentStoreRemoteChange, object: persistentStoreCoordinator) } func save(context: NSManagedObjectContext) { guard context.hasChanges else { return } do { try context.save() } catch let error as NSError { logger.error("Unable to save managed object context: \(String(describing: error), privacy: .public)") let crumb = Breadcrumb(level: .fatal, category: "PersistentStore") // note: NSDetailedErrorsKey == "NSDetailedErrorsKey" != "NSDetailedErrors" if let detailed = error.userInfo["NSDetailedErrors"] as? [NSError] { crumb.data = [ "errors": detailed.compactMap { error -> [String: Any?]? in guard let object = error.userInfo[NSValidationObjectErrorKey] as? NSManagedObject else { return nil } return [ "entity": object.entity.name, "key": error.userInfo[NSValidationKeyErrorKey], "value": error.userInfo[NSValidationValueErrorKey], "message": error.localizedDescription, ] } ] } SentrySDK.addBreadcrumb(crumb) fatalError("Unable to save managed object context: \(String(describing: error))") } } func status(for id: String, in context: NSManagedObjectContext? = nil) -> StatusMO? { let context = context ?? viewContext let request: NSFetchRequest = StatusMO.fetchRequest() request.predicate = NSPredicate(format: "id = %@", id) request.fetchLimit = 1 if let result = try? context.fetch(request), let status = result.first { return status } else { return nil } } @discardableResult private func upsert(status: Status, context: NSManagedObjectContext) -> StatusMO { if let statusMO = self.status(for: status.id, in: context) { statusMO.updateFrom(apiStatus: status, container: self) return statusMO } else { return StatusMO(apiStatus: status, container: self, context: context) } } func addOrUpdate(status: Status, context: NSManagedObjectContext? = nil, completion: ((StatusMO) -> Void)? = nil) { let context = context ?? backgroundContext context.perform { let statusMO = self.upsert(status: status, context: context) self.save(context: context) completion?(statusMO) self.statusSubject.send(status.id) } } @MainActor func addOrUpdateOnViewContext(status: Status) -> StatusMO { let statusMO = self.upsert(status: status, context: viewContext) self.save(context: viewContext) statusSubject.send(status.id) return statusMO } func addAll(statuses: [Status], in context: NSManagedObjectContext? = nil, completion: (() -> Void)? = nil) { let context = context ?? backgroundContext context.perform { statuses.forEach { self.upsert(status: $0, context: context) } self.save(context: context) statuses.forEach { self.statusSubject.send($0.id) } completion?() } } func addAll(statuses: [Status], in context: NSManagedObjectContext? = nil) async { return await withCheckedContinuation { continuation in addAll(statuses: statuses, in: context) { continuation.resume() } } } func account(for id: String, in context: NSManagedObjectContext? = nil) -> AccountMO? { let context = context ?? viewContext let request: NSFetchRequest = AccountMO.fetchRequest() request.predicate = NSPredicate(format: "id = %@", id) request.fetchLimit = 1 if let result = try? context.fetch(request), let account = result.first { return account } else { return nil } } @discardableResult private func upsert(account: Account, in context: NSManagedObjectContext) -> AccountMO { if let accountMO = self.account(for: account.id, in: context) { accountMO.updateFrom(apiAccount: account, container: self) return accountMO } else { return AccountMO(apiAccount: account, container: self, context: context) } } func addOrUpdate(account: Account, in context: NSManagedObjectContext? = nil, completion: ((AccountMO) -> Void)? = nil) { let context = context ?? backgroundContext context.perform { let accountMO = self.upsert(account: account, in: context) self.save(context: context) completion?(accountMO) self.accountSubject.send(account.id) } } func relationship(forAccount id: String, in context: NSManagedObjectContext? = nil) -> RelationshipMO? { let context = context ?? viewContext let request: NSFetchRequest = RelationshipMO.fetchRequest() request.predicate = NSPredicate(format: "accountID = %@", id) request.fetchLimit = 1 if let result = try? context.fetch(request), let relationship = result.first { return relationship } else { return nil } } @discardableResult private func upsert(relationship: Relationship, in context: NSManagedObjectContext) -> RelationshipMO { if let relationshipMO = self.relationship(forAccount: relationship.accountID, in: context) { relationshipMO.updateFrom(apiRelationship: relationship, container: self) return relationshipMO } else { let relationshipMO = RelationshipMO(apiRelationship: relationship, container: self, context: context) return relationshipMO } } func addOrUpdate(relationship: Relationship, in context: NSManagedObjectContext? = nil, completion: ((RelationshipMO) -> Void)? = nil) { let context = context ?? backgroundContext context.perform { let relationshipMO = self.upsert(relationship: relationship, in: context) self.save(context: context) completion?(relationshipMO) self.relationshipSubject.send(relationship.accountID) } } func addAll(accounts: [Account], in context: NSManagedObjectContext? = nil, completion: (() -> Void)? = nil) { let context = context ?? backgroundContext context.perform { accounts.forEach { self.upsert(account: $0, in: context) } self.save(context: context) completion?() accounts.forEach { self.accountSubject.send($0.id) } } } func addAll(accounts: [Account], in context: NSManagedObjectContext? = nil) async { await withCheckedContinuation { continuation in addAll(accounts: accounts, in: context) { continuation.resume() } } } func addAll(notifications: [Pachyderm.Notification], completion: (() -> Void)? = nil) { backgroundContext.perform { let statuses = notifications.compactMap { $0.status } // filter out mentions, otherwise we would double increment the reference count of those accounts // since the status has the same account as the notification let accounts = notifications.filter { $0.kind != .mention }.map { $0.account } statuses.forEach { self.upsert(status: $0, context: self.backgroundContext) } accounts.forEach { self.upsert(account: $0, in: self.backgroundContext) } self.save(context: self.backgroundContext) completion?() statuses.forEach { self.statusSubject.send($0.id) } accounts.forEach { self.accountSubject.send($0.id) } } } func performBatchUpdates(_ block: @escaping (_ context: NSManagedObjectContext, _ addAccounts: ([Account]) -> Void, _ addStatuses: ([Status]) -> Void) -> Void, completion: (() -> Void)? = nil) { backgroundContext.perform { var updatedAccounts = [String]() var updatedStatuses = [String]() block(self.backgroundContext, { (accounts) in accounts.forEach { self.upsert(account: $0, in: self.backgroundContext) } updatedAccounts.append(contentsOf: accounts.map { $0.id }) }, { (statuses) in statuses.forEach { self.upsert(status: $0, context: self.backgroundContext) } updatedStatuses.append(contentsOf: statuses.map { $0.id }) }) updatedAccounts.forEach(self.accountSubject.send) updatedStatuses.forEach(self.statusSubject.send) self.save(context: self.backgroundContext) completion?() } } func updateFollowedHashtags(_ hashtags: [Hashtag], completion: @escaping (Result<[FollowedHashtag], Error>) -> Void) { viewContext.perform { do { var all = try self.viewContext.fetch(FollowedHashtag.fetchRequest()) let toDelete = all.filter { existing in !hashtags.contains(where: { $0.name == existing.name }) }.map(\.objectID) if !toDelete.isEmpty { try self.viewContext.execute(NSBatchDeleteRequest(objectIDs: toDelete)) } for hashtag in hashtags where !all.contains(where: { $0.name == hashtag.name}) { let mo = FollowedHashtag(hashtag: hashtag, context: self.viewContext) all.append(mo) } self.save(context: self.viewContext) completion(.success(all)) } catch { completion(.failure(error)) } } } func hasFollowedHashtag(_ hashtag: Hashtag) -> Bool { do { let req = FollowedHashtag.fetchRequest(name: name) return try viewContext.count(for: req) > 0 } catch { return false } } func updateFilters(_ filters: [AnyFilter], completion: @escaping (Result<[FilterMO], Error>) -> Void) { viewContext.perform { do { var all = try self.viewContext.fetch(FilterMO.fetchRequest()) let toDelete = all.filter { existing in !filters.contains(where: { $0.id == existing.id }) }.map(\.objectID) if !toDelete.isEmpty { try self.viewContext.execute(NSBatchDeleteRequest(objectIDs: toDelete)) } for filter in filters { if let existing = all.first(where: { $0.id == filter.id }) { existing.updateFrom(apiFilter: filter, context: self.viewContext) } else { let mo = FilterMO(context: self.viewContext) mo.updateFrom(apiFilter: filter, context: self.viewContext) all.append(mo) } } self.save(context: self.viewContext) completion(.success(all)) } catch { completion(.failure(error)) } } } func getTimelinePosition(timeline: Timeline) -> TimelinePosition? { guard let accountInfo else { return nil } do { let req = TimelinePosition.fetchRequest(timeline: timeline, account: accountInfo) return try viewContext.fetch(req).first } catch { return nil } } @objc private func managedObjectsDidChange(_ notification: Foundation.Notification) { let changes = hasChangedSavedHashtagsOrInstances(notification) if changes.hashtags { NotificationCenter.default.post(name: .savedHashtagsChanged, object: nil) } if changes.instances { NotificationCenter.default.post(name: .savedInstancesChanged, object: nil) } } private func hasChangedSavedHashtagsOrInstances(_ notification: Foundation.Notification) -> (hashtags: Bool, instances: Bool) { var changes: (hashtags: Bool, instances: Bool) = (false, false) if let inserted = notification.userInfo?[NSInsertedObjectsKey] as? Set { for object in inserted { if object is SavedHashtag { changes.hashtags = true } else if object is SavedInstance { changes.instances = true } if changes.hashtags && changes.instances { return changes } } } if let deleted = notification.userInfo?[NSDeletedObjectsKey] as? Set { for object in deleted { if object is SavedHashtag { changes.hashtags = true } else if object is SavedInstance { changes.instances = true } if changes.hashtags && changes.instances { return changes } } } return changes } @objc private func remoteChanges(_ notification: Foundation.Notification) { guard let token = notification.userInfo?[NSPersistentHistoryTokenKey] as? NSPersistentHistoryToken else { return } remoteChangeHandlerQueue.async { defer { self.lastRemoteChangeToken = token } let req = NSPersistentHistoryChangeRequest.fetchHistory(after: self.lastRemoteChangeToken) self.backgroundContext.performAndWait { if let result = try? self.backgroundContext.execute(req) as? NSPersistentHistoryResult, let transactions = result.result as? [NSPersistentHistoryTransaction], !transactions.isEmpty { var changedHashtags = false var changedInstances = false var changedTimelinePositions = Set() var changedAccountPrefs = false outer: for transaction in transactions { for change in transaction.changes ?? [] { if change.changedObjectID.entity.name == "SavedHashtag" { changedHashtags = true } else if change.changedObjectID.entity.name == "SavedInstance" { changedInstances = true } else if change.changedObjectID.entity.name == "TimelinePosition" { changedTimelinePositions.insert(change.changedObjectID) } else if change.changedObjectID.entity.name == "AccountPreferences" { changedAccountPrefs = true } } } DispatchQueue.main.async { if changedHashtags { NotificationCenter.default.post(name: .savedHashtagsChanged, object: nil) } if changedInstances { NotificationCenter.default.post(name: .savedInstancesChanged, object: nil) } for id in changedTimelinePositions { guard let timelinePosition = try? self.viewContext.existingObject(with: id) as? TimelinePosition else { continue } // the kvo observer that clears the LazilyDecoding cache doesn't always fire on remote changes, so do it manually timelinePosition.changedRemotely() NotificationCenter.default.post(name: .timelinePositionChanged, object: timelinePosition) } if changedAccountPrefs { NotificationCenter.default.post(name: .accountPreferencesChangedRemotely, object: nil) } } } } } } } extension Foundation.Notification.Name { static let timelinePositionChanged = Notification.Name("timelinePositionChanged") static let accountPreferencesChangedRemotely = Notification.Name("accountPreferencesChangedRemotely") }