Background widget refresh

Fervor/Sources/Fervor/FervorClient.swift

@@ -13,7 +13,7 @@ public actor FervorClient: Sendable {
     private let session: URLSession
     public private(set) var accessToken: String?
     
-    private let decoder: JSONDecoder = {
+    public static let decoder: JSONDecoder = {
         let d = JSONDecoder()
         let withFractionalSeconds = ISO8601DateFormatter()
         withFractionalSeconds.formatOptions = [.withInternetDateTime, .withFractionalSeconds]
@@ -47,16 +47,22 @@ public actor FervorClient: Sendable {
         return components.url!
     }
     
-    private func performRequest<T: Decodable>(_ request: URLRequest) async throws -> T {
+    private func configureRequest(_ request: URLRequest) -> URLRequest {
         var request = request
-        if let accessToken = accessToken {
+        if let accessToken = accessToken,
+           request.value(forHTTPHeaderField: "Authorization") == nil {
             request.addValue("Bearer \(accessToken)", forHTTPHeaderField: "Authorization")
         }
-        let (data, response) = try await session.data(for: request, delegate: nil)
+        return request
+    }
+    
+    private func performRequest<T: Decodable>(_ request: URLRequest) async throws -> T {
+        let request = configureRequest(request)
+        let (data, response) = try await session.data(for: request)
         if (response as! HTTPURLResponse).statusCode == 404 {
             throw Error.notFound
         }
-        let decoded = try decoder.decode(T.self, from: data)
+        let decoded = try FervorClient.decoder.decode(T.self, from: data)
         return decoded
     }
     
@@ -108,6 +114,12 @@ public actor FervorClient: Sendable {
         return try await performRequest(request)
     }
     
+    public func itemsRequest(limit: Int) -> URLRequest {
+        return configureRequest(URLRequest(url: buildURL(path: "/api/v1/items", queryItems: [
+            URLQueryItem(name: "limit", value: limit.formatted()),
+        ])))
+    }
+    
     public func item(id: FervorID) async throws -> Item? {
         let request = URLRequest(url: buildURL(path: "/api/v1/items/\(id)"))
         do {

Persistence/Sources/Persistence/PersistentContainer.swift

@@ -17,7 +17,7 @@ public class PersistentContainer: NSPersistentContainer, @unchecked Sendable {
         return NSManagedObjectModel(contentsOf: url)!
     }()
     
-    private(set) lazy var backgroundContext: NSManagedObjectContext = {
+    public private(set) lazy var backgroundContext: NSManagedObjectContext = {
         let context = NSManagedObjectContext(concurrencyType: .privateQueueConcurrencyType)
         // todo: should the background context really be parented to the view context, or should they both be direct children of the PSC?
         context.parent = self.viewContext
@@ -126,6 +126,29 @@ public class PersistentContainer: NSPersistentContainer, @unchecked Sendable {
         try await self.saveViewContext()
     }
     
+    public func upsertItems(_ items: [Fervor.Item]) async throws {
+        try await backgroundContext.perform {
+            try self.doUpsertItems(items, setProgress: nil)
+        }
+    }
+    
+    private func doUpsertItems(_ items: [Fervor.Item], setProgress: ((_ current: Int, _ total: Int) -> Void)?) throws {
+        let req = Item.fetchRequest()
+        req.predicate = NSPredicate(format: "id in %@", items.map(\.id))
+        let existing = try self.backgroundContext.fetch(req)
+        self.logger.debug("doUpsertItems: updating \(existing.count, privacy: .public) items, inserting \(items.count - existing.count, privacy: .public)")
+        // todo: this feels like it'll be slow when there are many items
+        for (index, item) in items.enumerated() {
+            setProgress?(index, items.count)
+            if let existing = existing.first(where: { $0.id == item.id }) {
+                existing.updateFromServer(item)
+            } else {
+                let mo = Item(context: self.backgroundContext)
+                mo.updateFromServer(item)
+            }
+        }
+    }
+    
     public func syncItems(_ syncUpdate: ItemsSyncUpdate, setProgress: @escaping (_ current: Int, _ total: Int) -> Void) async throws {
         try await backgroundContext.perform {
             self.logger.debug("syncItems: deleting \(syncUpdate.delete.count, privacy: .public) items")
@@ -141,20 +164,7 @@ public class PersistentContainer: NSPersistentContainer, @unchecked Sendable {
                 // todo: does the background/view contexts need to get saved then?
             }
             
-            let req = Item.fetchRequest()
+            try self.doUpsertItems(syncUpdate.upsert, setProgress: setProgress)
-            req.predicate = NSPredicate(format: "id in %@", syncUpdate.upsert.map(\.id))
-            let existing = try self.backgroundContext.fetch(req)
-            self.logger.debug("syncItems: updating \(existing.count, privacy: .public) items, inserting \(syncUpdate.upsert.count - existing.count, privacy: .public)")
-            // todo: this feels like it'll be slow when there are many items
-            for (index, item) in syncUpdate.upsert.enumerated() {
-                setProgress(index, syncUpdate.upsert.count)
-                if let existing = existing.first(where: { $0.id == item.id }) {
-                    existing.updateFromServer(item)
-                } else {
-                    let mo = Item(context: self.backgroundContext)
-                    mo.updateFromServer(item)
-                }
-            }
             
             if self.backgroundContext.hasChanges {
                 try self.backgroundContext.save()

Reader.xcodeproj/project.pbxproj

@@ -20,6 +20,7 @@
 		D68B303D2792204B00E8B3FA /* read.js in Resources */ = {isa = PBXBuildFile; fileRef = D68B303C2792204B00E8B3FA /* read.js */; };
 		D68B30402792729A00E8B3FA /* AppSplitViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = D68B303F2792729A00E8B3FA /* AppSplitViewController.swift */; };
 		D68B304227932ED500E8B3FA /* UserActivities.swift in Sources */ = {isa = PBXBuildFile; fileRef = D68B304127932ED500E8B3FA /* UserActivities.swift */; };
+		D6A8E878286126DB007C6A2B /* BackgroundManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = D6A8E877286126DB007C6A2B /* BackgroundManager.swift */; };
 		D6C687EC272CD27600874C10 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = D6C687EB272CD27600874C10 /* AppDelegate.swift */; };
 		D6C687EE272CD27600874C10 /* SceneDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = D6C687ED272CD27600874C10 /* SceneDelegate.swift */; };
 		D6C687F8272CD27700874C10 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = D6C687F7272CD27700874C10 /* Assets.xcassets */; };
@@ -126,6 +127,7 @@
 		D68B303E27923C0000E8B3FA /* Reader.entitlements */ = {isa = PBXFileReference; lastKnownFileType = text.plist.entitlements; path = Reader.entitlements; sourceTree = "<group>"; };
 		D68B303F2792729A00E8B3FA /* AppSplitViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppSplitViewController.swift; sourceTree = "<group>"; };
 		D68B304127932ED500E8B3FA /* UserActivities.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UserActivities.swift; sourceTree = "<group>"; };
+		D6A8E877286126DB007C6A2B /* BackgroundManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackgroundManager.swift; sourceTree = "<group>"; };
 		D6AB5E9A285F6FE100157F2F /* Fervor */ = {isa = PBXFileReference; lastKnownFileType = wrapper; path = Fervor; sourceTree = "<group>"; };
 		D6AB5E9B285F706F00157F2F /* Persistence */ = {isa = PBXFileReference; lastKnownFileType = wrapper; path = Persistence; sourceTree = "<group>"; };
 		D6C687E8272CD27600874C10 /* Reader.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Reader.app; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -307,6 +309,7 @@
 				D68408EE2794808E00E327D2 /* Preferences.swift */,
 				D608238C27DE729E00D7D5F9 /* ItemListType.swift */,
 				D6EEDE8A285FA7FD009F854E /* LocalData+Migration.swift */,
+				D6A8E877286126DB007C6A2B /* BackgroundManager.swift */,
 				D6D5FA25285FEA5B00BBF188 /* Widgets */,
 				D65B18AF2750468B004A9448 /* Screens */,
 				D6C687F7272CD27700874C10 /* Assets.xcassets */,
@@ -630,6 +633,7 @@
 				D6D5FA24285FE9DB00BBF188 /* WidgetHelper.swift in Sources */,
 				D6E24363278BA1410005E546 /* ItemCollectionViewCell.swift in Sources */,
 				D6E2436E278BD8160005E546 /* ReadViewController.swift in Sources */,
+				D6A8E878286126DB007C6A2B /* BackgroundManager.swift in Sources */,
 				D65B18C127505348004A9448 /* HomeViewController.swift in Sources */,
 				D6C687EE272CD27600874C10 /* SceneDelegate.swift in Sources */,
 			);

Reader/AppDelegate.swift

@@ -10,6 +10,7 @@ import WebKit
 import OSLog
 import Combine
 import Persistence
+import BackgroundTasks
 
 @main
 class AppDelegate: UIResponder, UIApplicationDelegate {
@@ -49,8 +50,22 @@ class AppDelegate: UIResponder, UIApplicationDelegate {
         NotificationCenter.default.addObserver(self, selector: #selector(updateAppearance), name: .appearanceChanged, object: nil)
         updateAppearance()
         
+        BGTaskScheduler.shared.register(forTaskWithIdentifier: BackgroundManager.refreshIdentifier, using: nil) { task in
+            Task {
+                BackgroundManager.shared.handleBackgroundRefresh(task: task as! BGAppRefreshTask)
+            }
+        }
+        
         return true
     }
+    
+    func application(_ application: UIApplication, handleEventsForBackgroundURLSession identifier: String, completionHandler: @escaping () -> Void) {
+        if identifier == BackgroundManager.refreshIdentifier {
+            BackgroundManager.shared.doRefresh()
+        } else {
+            fatalError("Unknown background url session identifier: \(identifier)")
+        }
+    }
 
     // MARK: UISceneSession Lifecycle
 

Reader/BackgroundManager.swift

@@ -0,0 +1,132 @@
+//
+//  BackgroundManager.swift
+//  Reader
+//
+//  Created by Shadowfacts on 6/20/22.
+//
+
+import Foundation
+import BackgroundTasks
+import OSLog
+import Persistence
+import Fervor
+
+private let logger = Logger(subsystem: "net.shadowfacts.Reader", category: "BackgroundManager")
+
+class BackgroundManager: NSObject {
+    static let shared = BackgroundManager()
+    
+    private override init() {}
+    
+    static let refreshIdentifier = "net.shadowfacts.Reader.refresh"
+    
+    private var refreshTask: BGAppRefreshTask?
+    private var completedRefreshRequests = 0
+    
+    private var receivedData: [Int: Data] = [:]
+    
+    func scheduleRefresh() {
+        // we schedule refreshes from sceneDidEnterBackground, but there may be multiple scenes, and we only want one refresh task
+        BGTaskScheduler.shared.cancel(taskRequestWithIdentifier: BackgroundManager.refreshIdentifier)
+        
+        logger.debug("Scheduling background refresh task")
+        
+        let request = BGAppRefreshTaskRequest(identifier: BackgroundManager.refreshIdentifier)
+        request.earliestBeginDate = Date(timeIntervalSinceNow: 2 * 60 * 60)
+//        request.earliestBeginDate = Date(timeIntervalSinceNow: 60)
+        
+        do {
+            try BGTaskScheduler.shared.submit(request)
+        } catch {
+            logger.error("Unable to schedule app refresh: \(String(describing: error), privacy: .public)")
+        }
+    }
+    
+    func handleBackgroundRefresh(task: BGAppRefreshTask) {
+        logger.debug("Handling background refresh task")
+        
+        scheduleRefresh()
+        
+        guard !LocalData.accounts.isEmpty else {
+            task.setTaskCompleted(success: true)
+            return
+        }
+        
+        self.refreshTask = task
+        self.completedRefreshRequests = 0
+        
+        doRefresh()
+        
+        task.expirationHandler = {
+            task.setTaskCompleted(success: self.completedRefreshRequests == LocalData.accounts.count)
+        }
+    }
+    
+    func doRefresh() {
+        let config = URLSessionConfiguration.background(withIdentifier: BackgroundManager.refreshIdentifier)
+        config.sessionSendsLaunchEvents = true
+        let backgroundSession = URLSession(configuration: config, delegate: self, delegateQueue: nil)
+        
+        Task {
+            for account in LocalData.accounts {
+                let fervorController = await FervorController(account: account, session: backgroundSession)
+                let itemsRequest = await fervorController.client.itemsRequest(limit: 32)
+                let task = backgroundSession.dataTask(with: itemsRequest)
+                task.taskDescription = account.id.base64EncodedString()
+                task.resume()
+            }
+        }
+    }
+    
+    private func receivedData(_ data: Data, for task: URLSessionDataTask) {
+        guard let taskDescription = task.taskDescription,
+              let id = Data(base64Encoded: taskDescription),
+              let account = LocalData.account(with: id) else {
+            logger.error("Could not find account to handle request")
+            return
+        }
+        Task { @MainActor in
+            defer {
+                if let refreshTask {
+                    refreshTask.setTaskCompleted(success: completedRefreshRequests == LocalData.accounts.count)
+                }
+            }
+            
+            let items: [Fervor.Item]
+            do {
+                items = try FervorClient.decoder.decode([Fervor.Item].self, from: data)
+            } catch {
+                logger.error("Unable to decode items: \(String(describing: error), privacy: .public)")
+                return
+            }
+            
+            let fervorController = await FervorController(account: account)
+            do {
+                try await fervorController.persistentContainer.upsertItems(items)
+                logger.info("Upserted \(items.count) items during background refresh")
+                
+                await WidgetHelper.updateWidgetData(fervorController: fervorController)
+                
+                completedRefreshRequests += 1
+            } catch {
+                logger.error("Unable to upsert items: \(String(describing: error), privacy: .public)")
+            }
+        }
+    }
+}
+
+extension BackgroundManager: URLSessionDataDelegate {
+    func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) {
+        if !receivedData.keys.contains(dataTask.taskIdentifier) {
+            receivedData[dataTask.taskIdentifier] = Data(capacity: Int(dataTask.countOfBytesExpectedToReceive))
+        }
+        receivedData[dataTask.taskIdentifier]!.append(data)
+        
+        logger.debug("Received chunk of \(data.count) bytes")
+        
+        if dataTask.countOfBytesReceived == dataTask.countOfBytesExpectedToReceive {
+            logger.info("Received all data for task, handling update")
+            receivedData(receivedData.removeValue(forKey: dataTask.taskIdentifier)!, for: dataTask)
+        }
+    }
+}

Reader/FervorController.swift

@@ -31,9 +31,9 @@ actor FervorController {
     private var lastSyncState = SyncState.done
     private var cancellables = Set<AnyCancellable>()
     
-    init(instanceURL: URL, account: LocalData.Account?) async {
+    init(instanceURL: URL, account: LocalData.Account?, session: URLSession = .shared) async {
         self.instanceURL = instanceURL
-        self.client = FervorClient(instanceURL: instanceURL, accessToken: account?.token.accessToken)
+        self.client = FervorClient(instanceURL: instanceURL, accessToken: account?.token.accessToken, session: session)
         self.account = account
         self.clientID = account?.clientID
         self.clientSecret = account?.clientSecret
@@ -45,8 +45,8 @@ actor FervorController {
         }
     }
     
-    convenience init(account: LocalData.Account) async {
+    convenience init(account: LocalData.Account, session: URLSession = .shared) async {
-        await self.init(instanceURL: account.instanceURL, account: account)
+        await self.init(instanceURL: account.instanceURL, account: account, session: session)
     }
     
     private func setSyncState(_ state: SyncState) {

Reader/Info.plist

@@ -37,5 +37,13 @@
 			</array>
 		</dict>
 	</dict>
+	<key>BGTaskSchedulerPermittedIdentifiers</key>
+	<array>
+		<string>net.shadowfacts.Reader.refresh</string>
+	</array>
+	<key>UIBackgroundModes</key>
+	<array>
+		<string>fetch</string>
+	</array>
 </dict>
 </plist>

Reader/SceneDelegate.swift

@@ -47,7 +47,9 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate {
                 fervorController = await FervorController(account: account)
                 syncFromServer()
                 createAppUI()
-                if let activity = activity {
+                if !connectionOptions.urlContexts.isEmpty {
+                    self.scene(scene, openURLContexts: connectionOptions.urlContexts)
+                } else if let activity = activity {
                     await setupUI(from: activity)
                 }
                 setupSceneActivationConditions()
@@ -90,15 +92,6 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate {
     func sceneWillResignActive(_ scene: UIScene) {
         // Called when the scene will move from an active state to an inactive state.
         // This may occur due to temporary interruptions (ex. an incoming phone call).
-        
-        if let fervorController = fervorController {
-            Task(priority: .userInitiated) {
-                await fervorController.syncReadToServer()
-            }
-            Task(priority: .userInitiated) {
-                await WidgetHelper.updateWidgetData(fervorController: fervorController)
-            }
-        }
     }
 
     func sceneWillEnterForeground(_ scene: UIScene) {
@@ -110,6 +103,17 @@ class SceneDelegate: UIResponder, UIWindowSceneDelegate {
         // Called as the scene transitions from the foreground to the background.
         // Use this method to save data, release shared resources, and store enough scene-specific state information
         // to restore the scene back to its current state.
+        
+        if let fervorController = fervorController {
+            Task(priority: .userInitiated) {
+                await fervorController.syncReadToServer()
+            }
+            Task(priority: .userInitiated) {
+                await WidgetHelper.updateWidgetData(fervorController: fervorController)
+            }
+        }
+        
+        BackgroundManager.shared.scheduleRefresh()
     }
     
     func stateRestorationActivity(for scene: UIScene) -> NSUserActivity? {

Reader/Widgets/WidgetHelper.swift

@@ -8,6 +8,9 @@
 import Foundation
 import WidgetKit
 import Persistence
+import OSLog
+
+private let logger = Logger(subsystem: "net.shadowfacts.Reader", category: "WidgetHelper")
 
 struct WidgetHelper {
     private init() {}
@@ -17,12 +20,13 @@ struct WidgetHelper {
     static func updateWidgetData(fervorController: FervorController) async {
         // Accessing CoreData from the widget extension puts us over the memory limit, so we pre-generate all the data it needs and save it to disk
         
-        let prioritizedItems: [WidgetData.Item] = await fervorController.persistentContainer.performBackgroundTask { ctx in
+        let context = fervorController.persistentContainer.backgroundContext
+        let prioritizedItems: [WidgetData.Item] = await context.perform {
             let req = Item.fetchRequest()
             req.sortDescriptors = [NSSortDescriptor(key: "published", ascending: false)]
             req.fetchLimit = 32
             req.predicate = NSPredicate(format: "read = NO")
-            var items = (try? ctx.fetch(req)) ?? []
+            var items = (try? context.fetch(req)) ?? []
             
             var prioritizedItems: [Item] = []