Start adding CoreData-based "reference" counting for statuses

Prune old statuses that aren't likely to be shown again when scrolling
in timeline table view

Tusker/CoreData/MastodonCachePersistentStore.swift

@@ -37,17 +37,23 @@ class MastodonCachePersistentStore: NSPersistentContainer {
         }
     }
     
-    private func upsert(status: Status) {
+    private func upsert(status: Status, incrementReferenceCount: Bool) {
         if let statusMO = self.status(for: status.id, in: self.backgroundContext) {
             statusMO.updateFrom(apiStatus: status, container: self)
+            if incrementReferenceCount {
+                statusMO.incrementReferenceCount()
+            }
         } else {
-            _ = StatusMO(apiStatus: status, container: self, context: self.backgroundContext)
+            let statusMO = StatusMO(apiStatus: status, container: self, context: self.backgroundContext)
+            if incrementReferenceCount {
+                statusMO.incrementReferenceCount()
+            }
         }
     }
     
-    func addOrUpdate(status: Status, save: Bool = true) {
+    func addOrUpdate(status: Status, incrementReferenceCount: Bool, save: Bool = true) {
         backgroundContext.perform {
-            self.upsert(status: status)
+            self.upsert(status: status, incrementReferenceCount: incrementReferenceCount)
             if save, self.backgroundContext.hasChanges {
                 try! self.backgroundContext.save()
             }
@@ -56,7 +62,7 @@ class MastodonCachePersistentStore: NSPersistentContainer {
 
     func addAll(statuses: [Status], completion: (() -> Void)? = nil) {
         backgroundContext.perform {
-            statuses.forEach(self.upsert(status:))
+            statuses.forEach { self.upsert(status: $0, incrementReferenceCount: true) }
             if self.backgroundContext.hasChanges {
                 try! self.backgroundContext.save()
             }

Tusker/CoreData/StatusMO.swift

@@ -35,6 +35,7 @@ public final class StatusMO: NSManagedObject {
     @NSManaged public var pinned: Bool
     @NSManaged public var reblogged: Bool
     @NSManaged public var reblogsCount: Int
+    @NSManaged public var referenceCount: Int
     @NSManaged public var sensitive: Bool
     @NSManaged public var spoilerText: String
     @NSManaged public var uri: String // todo: are both uri and url necessary?
@@ -64,12 +65,30 @@ public final class StatusMO: NSManagedObject {
         }
     }
     
+    func incrementReferenceCount() {
+        referenceCount += 1
+    }
+    
+    func decrementReferenceCount() {
+        referenceCount -= 1
+        if referenceCount <= 0 {
+            managedObjectContext!.delete(self)
+        }
+    }
+    
+    public override func prepareForDeletion() {
+        super.prepareForDeletion()
+        reblog?.decrementReferenceCount()
+    }
+
 }
 
 extension StatusMO {
     convenience init(apiStatus status: Pachyderm.Status, container: MastodonCachePersistentStore, context: NSManagedObjectContext) {
         self.init(context: context)
         self.updateFrom(apiStatus: status, container: container)
+        
+        reblog?.incrementReferenceCount()
     }
     
     func updateFrom(apiStatus status: Pachyderm.Status, container: MastodonCachePersistentStore) {

Tusker/CoreData/Tusker.xcdatamodeld/Tusker.xcdatamodel/contents

@@ -43,6 +43,7 @@
         <attribute name="pinned" optional="YES" attributeType="Boolean" usesScalarValueType="YES"/>
         <attribute name="reblogged" optional="YES" attributeType="Boolean" usesScalarValueType="YES"/>
         <attribute name="reblogsCount" attributeType="Integer 64" defaultValueString="0" usesScalarValueType="YES"/>
+        <attribute name="referenceCount" attributeType="Integer 64" defaultValueString="0" usesScalarValueType="YES"/>
         <attribute name="sensitive" attributeType="Boolean" usesScalarValueType="YES"/>
         <attribute name="spoilerText" attributeType="String"/>
         <attribute name="uri" attributeType="String"/>
@@ -58,6 +59,6 @@
     </entity>
     <elements>
         <element name="Account" positionX="169.21875" positionY="78.9609375" width="128" height="313"/>
-        <element name="Status" positionX="-63" positionY="-18" width="128" height="403"/>
+        <element name="Status" positionX="-63" positionY="-18" width="128" height="418"/>
     </elements>
 </model>

Tusker/MastodonCache.swift

@@ -59,12 +59,10 @@ class MastodonCache {
     
     func add(status: Status) {
         set(status: status, for: status.id)
-        mastodonController?.persistentContainer.addOrUpdate(status: status)
     }
     
     func addAll(statuses: [Status]) {
         statuses.forEach(add)
-        mastodonController?.persistentContainer.addAll(statuses: statuses)
     }
     
     // MARK: - Accounts
@@ -94,12 +92,10 @@ class MastodonCache {
     
     func add(account: Account) {
         set(account: account, for: account.id)
-        mastodonController?.persistentContainer.addOrUpdate(account: account)
     }
     
     func addAll(accounts: [Account]) {
         accounts.forEach(add)
-        mastodonController?.persistentContainer.addAll(accounts: accounts)
     }
     
     // MARK: - Relationships

Tusker/SceneDelegate.swift

@@ -109,6 +109,8 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate {
         // Called as the scene transitions from the foreground to the background.
         // Use this method to save data, release shared resources, and store enough scene-specific state information
         // to restore the scene back to its current state.
+        
+        try! scene.session.mastodonController?.persistentContainer.viewContext.save()
     }
     
     func activateAccount(_ account: LocalData.UserAccountInfo) {

Tusker/Screens/Timeline/TimelineTableViewController.swift

@@ -19,6 +19,9 @@ class TimelineTableViewController: EnhancedTableViewController {
     var newer: RequestRange?
     var older: RequestRange?
     
+    private var prevScrollViewContentOffset: CGPoint?
+    private var scrollViewDirection: CGFloat = 0
+    
     init(for timeline: Timeline, mastodonController: MastodonController) {
         self.timeline = timeline
         self.mastodonController = mastodonController
@@ -38,6 +41,17 @@ class TimelineTableViewController: EnhancedTableViewController {
         fatalError("init(coder:) has not been implemented")
     }
     
+    deinit {
+        // decrement reference counts of any statuses we still have
+        // if the app is currently being quit, this will not affect the persisted data because
+        // the persistent container would already have been saved in SceneDelegate.sceneDidEnterBackground(_:)
+        for segment in timelineSegments {
+            for (id, _) in segment {
+                mastodonController.persistentContainer.status(for: id)?.decrementReferenceCount()
+            }
+        }
+    }
+    
     func statusID(for indexPath: IndexPath) -> String {
         return timelineSegments[indexPath.section][indexPath.row].id
     }
@@ -59,7 +73,6 @@ class TimelineTableViewController: EnhancedTableViewController {
         let request = Client.getStatuses(timeline: timeline)
         mastodonController.run(request) { response in
             guard case let .success(statuses, pagination) = response else { fatalError() }
-//            self.mastodonController.cache.addAll(statuses: statuses)
             // todo: possible race condition here? we update the underlying data before waiting to reload the table view
             self.timelineSegments.insert(statuses.map { ($0.id, .unknown) }, at: 0)
             self.newer = pagination?.newer
@@ -96,6 +109,59 @@ class TimelineTableViewController: EnhancedTableViewController {
     // MARK: - Table view delegate
     
     override func tableView(_ tableView: UITableView, willDisplay cell: UITableViewCell, forRowAt indexPath: IndexPath) {
+        // when scrolling upwards, decrement reference counts for old statuses, if necessary
+        if scrollViewDirection < 0 {
+            if indexPath.section <= timelineSegments.count - 2 {
+                // decrement ref counts for all sections below the section below the current section
+                // (e.g., there exist sections 0, 1, 2 and we're currently scrolling upwards in section 0, we want to remove section 2)
+                
+                // todo: this is in the hot path for scrolling, possibly move this to a background thread?
+                let sectionsToRemove = indexPath.section + 1..<timelineSegments.count
+                for section in sectionsToRemove {
+                    for (id, _) in timelineSegments.remove(at: section) {
+                        mastodonController.persistentContainer.status(for: id)?.decrementReferenceCount()
+                    }
+                }
+                // see below comment about DispatchQueue.main.async
+                DispatchQueue.main.async {
+                    UIView.performWithoutAnimation {
+                        tableView.deleteSections(IndexSet(sectionsToRemove), with: .none)
+                    }
+                }
+            } else {
+                // we are scrolling in the last or second to last section
+                // grab the last section and, if there is more than one page in the last section
+                // (we never want to remove the only page in the last section unless there is another section between it and the user),
+                // remove the last page and decrement reference counts
+                let pageSize = 20 // todo: this should come from somewhere in Pachyderm
+                let lastSection = timelineSegments.last!
+                if lastSection.count > 2 * pageSize,
+                    indexPath.row < lastSection.count - (2 * pageSize) {
+                    // todo: this is in the hot path for scrolling, possibly move this to a background thread?
+                    let statusesToRemove = lastSection[lastSection.count - pageSize..<lastSection.count]
+                    for (id, _) in statusesToRemove {
+                        mastodonController.persistentContainer.status(for: id)?.decrementReferenceCount()
+                    }
+                    timelineSegments[timelineSegments.count - 1].removeLast(20)
+                    
+                    let removedIndexPaths = (lastSection.count - 20..<lastSection.count).map { IndexPath(row: $0, section: timelineSegments.count - 1) }
+                    // Removing this DispatchQueue.main.async call causes things to break when scrolling
+                    // back down towards the removed rows. There would be a index out of bounds crash
+                    // because, while we've already removed the statuses from our model, the table view doesn't seem to know that.
+                    // It seems like tableView update calls made from inside tableView(_:willDisplay:forRowAt:) are silently ignored.
+                    // Calling tableView.numberOfRows(inSection: 0) when trapped in the debugger after the aforementioned IOOB
+                    // will produce an incorrect value (it will be some multiple of pageSize too high).
+                    // Deferring the tableView update until the next runloop iteration seems to solve that.
+                    DispatchQueue.main.async {
+                        UIView.performWithoutAnimation {
+                            tableView.deleteRows(at: removedIndexPaths, with: .none)
+                        }
+                    }
+                }
+            }
+        }
+        
+        // load older statuses, if necessary
         if indexPath.section == timelineSegments.count - 1,
             indexPath.row == timelineSegments[indexPath.section].count - 1 {
             guard let older = older else { return }
@@ -104,7 +170,6 @@ class TimelineTableViewController: EnhancedTableViewController {
             mastodonController.run(request) { response in
                 guard case let .success(newStatuses, pagination) = response else { fatalError() }
                 self.older = pagination?.older
-//                self.mastodonController.cache.addAll(statuses: newStatuses)
                 let newRows = self.timelineSegments.last!.count..<(self.timelineSegments.last!.count + newStatuses.count)
                 let newIndexPaths = newRows.map { IndexPath(row: $0, section: self.timelineSegments.count - 1) }
                 self.timelineSegments[self.timelineSegments.count - 1].append(contentsOf: newStatuses.map { ($0.id, .unknown) })
@@ -138,7 +203,6 @@ class TimelineTableViewController: EnhancedTableViewController {
         mastodonController.run(request) { response in
             guard case let .success(newStatuses, pagination) = response else { fatalError() }
             self.newer = pagination?.newer
-//            self.mastodonController.cache.addAll(statuses: newStatuses)
             self.timelineSegments[0].insert(contentsOf: newStatuses.map { ($0.id, .unknown) }, at: 0)
             
             if let newer = pagination?.newer {
@@ -163,6 +227,15 @@ class TimelineTableViewController: EnhancedTableViewController {
         }
     }
     
+    // Mark: Scroll View Delegate
+    override func scrollViewDidScroll(_ scrollView: UIScrollView) {
+        if let prev = prevScrollViewContentOffset {
+            scrollViewDirection = scrollView.contentOffset.y - prev.y
+        }
+        prevScrollViewContentOffset = scrollView.contentOffset
+    }
+    
+
     @objc func composePressed(_ sender: Any) {
         compose()
     }
@@ -192,7 +265,10 @@ extension TimelineTableViewController: UITableViewDataSourcePrefetching {
     
     func tableView(_ tableView: UITableView, cancelPrefetchingForRowsAt indexPaths: [IndexPath]) {
         for indexPath in indexPaths {
-            guard let status = mastodonController.persistentContainer.status(for: statusID(for: indexPath)) else { continue }
+            // todo: this means when removing cells, we can't cancel prefetching
+            // is this an issue?
+            guard indexPath.section < timelineSegments.count, indexPath.row < timelineSegments[indexPath.section].count,
+                let status = mastodonController.persistentContainer.status(for: statusID(for: indexPath)) else { continue }
             ImageCache.avatars.cancelWithoutCallback(status.account.avatar)
             for attachment in status.attachments {
                 ImageCache.attachments.cancelWithoutCallback(attachment.url)