From 920f926b48ac847372cf0025444748dbe0971623 Mon Sep 17 00:00:00 2001 From: Shadowfacts Date: Fri, 19 Jun 2020 19:14:24 -0400 Subject: [PATCH] Add text recognition image description for image attachments --- .../ComposeAttachmentsViewController.swift | 13 ++++ .../ComposeAttachmentTableViewCell.swift | 72 +++++++++++++++++++ .../ComposeAttachmentTableViewCell.xib | 6 ++ 3 files changed, 91 insertions(+) diff --git a/Tusker/Screens/Compose/ComposeAttachmentsViewController.swift b/Tusker/Screens/Compose/ComposeAttachmentsViewController.swift index 84176aae..f6990866 100644 --- a/Tusker/Screens/Compose/ComposeAttachmentsViewController.swift +++ b/Tusker/Screens/Compose/ComposeAttachmentsViewController.swift @@ -315,6 +315,15 @@ class ComposeAttachmentsViewController: UITableViewController { actions.append(UIAction(title: "Edit Drawing", image: UIImage(systemName: "hand.draw"), identifier: nil, discoverabilityTitle: nil, attributes: [], state: .off, handler: { (_) in self.presentComposeDrawingViewController(editingAttachmentAt: indexPath.row) })) + case .asset(_), .image(_): + if attachment.data.type == .image, + let cell = tableView.cellForRow(at: indexPath) as? ComposeAttachmentTableViewCell { + + let title = NSLocalizedString("Recognize Text", comment: "recognize image attachment text menu item title") + actions.append(UIAction(title: title, image: UIImage(systemName: "doc.text.viewfinder"), identifier: nil, discoverabilityTitle: nil, attributes: [], state: .off, handler: { (_) in + cell.recognizeTextFromImage() + })) + } default: break } @@ -503,6 +512,10 @@ extension ComposeAttachmentsViewController: AssetPickerViewControllerDelegate { } extension ComposeAttachmentsViewController: ComposeAttachmentTableViewCellDelegate { + func composeAttachment(_ cell: ComposeAttachmentTableViewCell, present viewController: UIViewController, animated: Bool) { + self.present(viewController, animated: animated) + } + func removeAttachment(_ cell: ComposeAttachmentTableViewCell) { guard let indexPath = tableView.indexPath(for: cell) else { return } attachments.remove(at: indexPath.row) diff --git a/Tusker/Views/Attachment Cells/ComposeAttachmentTableViewCell.swift b/Tusker/Views/Attachment Cells/ComposeAttachmentTableViewCell.swift index 03667017..95d797df 100644 --- a/Tusker/Views/Attachment Cells/ComposeAttachmentTableViewCell.swift +++ b/Tusker/Views/Attachment Cells/ComposeAttachmentTableViewCell.swift @@ -9,8 +9,10 @@ import UIKit import Photos import AVFoundation +import Vision protocol ComposeAttachmentTableViewCellDelegate: class { + func composeAttachment(_ cell: ComposeAttachmentTableViewCell, present viewController: UIViewController, animated: Bool) func removeAttachment(_ cell: ComposeAttachmentTableViewCell) func attachmentDescriptionChanged(_ cell: ComposeAttachmentTableViewCell) } @@ -23,9 +25,27 @@ class ComposeAttachmentTableViewCell: UITableViewCell { @IBOutlet weak var descriptionTextView: UITextView! @IBOutlet weak var descriptionPlaceholderLabel: UILabel! @IBOutlet weak var removeButton: UIButton! + @IBOutlet weak var activityIndicator: UIActivityIndicatorView! var attachment: CompositionAttachment! + var state: State = .allowEntry { + didSet { + switch state { + case .allowEntry: + descriptionTextView.isEditable = true + updateDescriptionPlaceholderLabel() + activityIndicator.stopAnimating() + case .recognizingText: + descriptionTextView.isEditable = false + descriptionPlaceholderLabel.isHidden = true + activityIndicator.startAnimating() + } + } + } + + private var textRecognitionRequest: VNRecognizeTextRequest? + override func awakeFromNib() { super.awakeFromNib() @@ -74,17 +94,69 @@ class ComposeAttachmentTableViewCell: UITableViewCell { removeButton.isEnabled = enabled } + func recognizeTextFromImage() { + precondition(attachment.data.type == .image) + state = .recognizingText + + DispatchQueue.global(qos: .userInitiated).async { + self.attachment.data.getData { (data, mimeType) in + let handler = VNImageRequestHandler(data: data, options: [:]) + let request = VNRecognizeTextRequest { (request, error) in + DispatchQueue.main.async { + self.state = .allowEntry + + if let results = request.results as? [VNRecognizedTextObservation] { + var text = "" + for observation in results { + let result = observation.topCandidates(1).first! + text.append(result.string) + text.append("\n") + } + self.descriptionTextView.text = text + self.textViewDidChange(self.descriptionTextView) + } + } + } + request.recognitionLevel = .accurate + request.usesLanguageCorrection = true + self.textRecognitionRequest = request + DispatchQueue.global(qos: .userInitiated).async { + do { + try handler.perform([request]) + } catch { + // The perform call throws an error with code 1 if the request is cancelled, which we don't want to show an alert for. + guard (error as NSError).code != 1 else { return } + DispatchQueue.main.async { + self.state = .allowEntry + let title = NSLocalizedString("Text Recognition Failed", comment: "text recognition error alert title") + let message = error.localizedDescription + let alert = UIAlertController(title: title, message: message, preferredStyle: .alert) + self.delegate?.composeAttachment(self, present: alert, animated: true) + } + } + } + } + } + } + override func prepareForReuse() { super.prepareForReuse() assetImageView.image = nil } @IBAction func removeButtonPressed(_ sender: Any) { + textRecognitionRequest?.cancel() delegate?.removeAttachment(self) } } +extension ComposeAttachmentTableViewCell { + enum State { + case allowEntry, recognizingText + } +} + extension ComposeAttachmentTableViewCell: UITextViewDelegate { func textViewDidChange(_ textView: UITextView) { attachment.attachmentDescription = textView.text diff --git a/Tusker/Views/Attachment Cells/ComposeAttachmentTableViewCell.xib b/Tusker/Views/Attachment Cells/ComposeAttachmentTableViewCell.xib index 784a482f..7f6264f1 100644 --- a/Tusker/Views/Attachment Cells/ComposeAttachmentTableViewCell.xib +++ b/Tusker/Views/Attachment Cells/ComposeAttachmentTableViewCell.xib @@ -57,6 +57,9 @@ + @@ -64,12 +67,15 @@ + + +