Add text recognition image description for image attachments

This commit is contained in:
Shadowfacts 2020-06-19 19:14:24 -04:00
parent 6e27399e10
commit 920f926b48
Signed by untrusted user: shadowfacts
GPG Key ID: 94A5AB95422746E5
3 changed files with 91 additions and 0 deletions

View File

@ -315,6 +315,15 @@ class ComposeAttachmentsViewController: UITableViewController {
actions.append(UIAction(title: "Edit Drawing", image: UIImage(systemName: "hand.draw"), identifier: nil, discoverabilityTitle: nil, attributes: [], state: .off, handler: { (_) in actions.append(UIAction(title: "Edit Drawing", image: UIImage(systemName: "hand.draw"), identifier: nil, discoverabilityTitle: nil, attributes: [], state: .off, handler: { (_) in
self.presentComposeDrawingViewController(editingAttachmentAt: indexPath.row) self.presentComposeDrawingViewController(editingAttachmentAt: indexPath.row)
})) }))
case .asset(_), .image(_):
if attachment.data.type == .image,
let cell = tableView.cellForRow(at: indexPath) as? ComposeAttachmentTableViewCell {
let title = NSLocalizedString("Recognize Text", comment: "recognize image attachment text menu item title")
actions.append(UIAction(title: title, image: UIImage(systemName: "doc.text.viewfinder"), identifier: nil, discoverabilityTitle: nil, attributes: [], state: .off, handler: { (_) in
cell.recognizeTextFromImage()
}))
}
default: default:
break break
} }
@ -503,6 +512,10 @@ extension ComposeAttachmentsViewController: AssetPickerViewControllerDelegate {
} }
extension ComposeAttachmentsViewController: ComposeAttachmentTableViewCellDelegate { extension ComposeAttachmentsViewController: ComposeAttachmentTableViewCellDelegate {
func composeAttachment(_ cell: ComposeAttachmentTableViewCell, present viewController: UIViewController, animated: Bool) {
self.present(viewController, animated: animated)
}
func removeAttachment(_ cell: ComposeAttachmentTableViewCell) { func removeAttachment(_ cell: ComposeAttachmentTableViewCell) {
guard let indexPath = tableView.indexPath(for: cell) else { return } guard let indexPath = tableView.indexPath(for: cell) else { return }
attachments.remove(at: indexPath.row) attachments.remove(at: indexPath.row)

View File

@ -9,8 +9,10 @@
import UIKit import UIKit
import Photos import Photos
import AVFoundation import AVFoundation
import Vision
protocol ComposeAttachmentTableViewCellDelegate: class { protocol ComposeAttachmentTableViewCellDelegate: class {
func composeAttachment(_ cell: ComposeAttachmentTableViewCell, present viewController: UIViewController, animated: Bool)
func removeAttachment(_ cell: ComposeAttachmentTableViewCell) func removeAttachment(_ cell: ComposeAttachmentTableViewCell)
func attachmentDescriptionChanged(_ cell: ComposeAttachmentTableViewCell) func attachmentDescriptionChanged(_ cell: ComposeAttachmentTableViewCell)
} }
@ -23,9 +25,27 @@ class ComposeAttachmentTableViewCell: UITableViewCell {
@IBOutlet weak var descriptionTextView: UITextView! @IBOutlet weak var descriptionTextView: UITextView!
@IBOutlet weak var descriptionPlaceholderLabel: UILabel! @IBOutlet weak var descriptionPlaceholderLabel: UILabel!
@IBOutlet weak var removeButton: UIButton! @IBOutlet weak var removeButton: UIButton!
@IBOutlet weak var activityIndicator: UIActivityIndicatorView!
var attachment: CompositionAttachment! var attachment: CompositionAttachment!
var state: State = .allowEntry {
didSet {
switch state {
case .allowEntry:
descriptionTextView.isEditable = true
updateDescriptionPlaceholderLabel()
activityIndicator.stopAnimating()
case .recognizingText:
descriptionTextView.isEditable = false
descriptionPlaceholderLabel.isHidden = true
activityIndicator.startAnimating()
}
}
}
private var textRecognitionRequest: VNRecognizeTextRequest?
override func awakeFromNib() { override func awakeFromNib() {
super.awakeFromNib() super.awakeFromNib()
@ -74,17 +94,69 @@ class ComposeAttachmentTableViewCell: UITableViewCell {
removeButton.isEnabled = enabled removeButton.isEnabled = enabled
} }
func recognizeTextFromImage() {
precondition(attachment.data.type == .image)
state = .recognizingText
DispatchQueue.global(qos: .userInitiated).async {
self.attachment.data.getData { (data, mimeType) in
let handler = VNImageRequestHandler(data: data, options: [:])
let request = VNRecognizeTextRequest { (request, error) in
DispatchQueue.main.async {
self.state = .allowEntry
if let results = request.results as? [VNRecognizedTextObservation] {
var text = ""
for observation in results {
let result = observation.topCandidates(1).first!
text.append(result.string)
text.append("\n")
}
self.descriptionTextView.text = text
self.textViewDidChange(self.descriptionTextView)
}
}
}
request.recognitionLevel = .accurate
request.usesLanguageCorrection = true
self.textRecognitionRequest = request
DispatchQueue.global(qos: .userInitiated).async {
do {
try handler.perform([request])
} catch {
// The perform call throws an error with code 1 if the request is cancelled, which we don't want to show an alert for.
guard (error as NSError).code != 1 else { return }
DispatchQueue.main.async {
self.state = .allowEntry
let title = NSLocalizedString("Text Recognition Failed", comment: "text recognition error alert title")
let message = error.localizedDescription
let alert = UIAlertController(title: title, message: message, preferredStyle: .alert)
self.delegate?.composeAttachment(self, present: alert, animated: true)
}
}
}
}
}
}
override func prepareForReuse() { override func prepareForReuse() {
super.prepareForReuse() super.prepareForReuse()
assetImageView.image = nil assetImageView.image = nil
} }
@IBAction func removeButtonPressed(_ sender: Any) { @IBAction func removeButtonPressed(_ sender: Any) {
textRecognitionRequest?.cancel()
delegate?.removeAttachment(self) delegate?.removeAttachment(self)
} }
} }
extension ComposeAttachmentTableViewCell {
enum State {
case allowEntry, recognizingText
}
}
extension ComposeAttachmentTableViewCell: UITextViewDelegate { extension ComposeAttachmentTableViewCell: UITextViewDelegate {
func textViewDidChange(_ textView: UITextView) { func textViewDidChange(_ textView: UITextView) {
attachment.attachmentDescription = textView.text attachment.attachmentDescription = textView.text

View File

@ -57,6 +57,9 @@
<constraint firstAttribute="height" relation="greaterThanOrEqual" constant="80" id="jWo-An-3h6"/> <constraint firstAttribute="height" relation="greaterThanOrEqual" constant="80" id="jWo-An-3h6"/>
</constraints> </constraints>
</stackView> </stackView>
<activityIndicatorView hidden="YES" opaque="NO" contentMode="scaleToFill" horizontalHuggingPriority="750" verticalHuggingPriority="750" hidesWhenStopped="YES" style="medium" translatesAutoresizingMaskIntoConstraints="NO" id="Kzy-5r-UW8">
<rect key="frame" x="179" y="38" width="20" height="20"/>
</activityIndicatorView>
</subviews> </subviews>
<constraints> <constraints>
<constraint firstAttribute="bottom" secondItem="xRe-ec-Coh" secondAttribute="bottom" constant="8" id="DOS-Wv-G3s"/> <constraint firstAttribute="bottom" secondItem="xRe-ec-Coh" secondAttribute="bottom" constant="8" id="DOS-Wv-G3s"/>
@ -64,12 +67,15 @@
<constraint firstItem="h6T-x4-yzl" firstAttribute="trailing" secondItem="cwP-Eh-5dJ" secondAttribute="trailing" constant="4" id="KN2-Ve-3B2"/> <constraint firstItem="h6T-x4-yzl" firstAttribute="trailing" secondItem="cwP-Eh-5dJ" secondAttribute="trailing" constant="4" id="KN2-Ve-3B2"/>
<constraint firstItem="h6T-x4-yzl" firstAttribute="top" secondItem="cwP-Eh-5dJ" secondAttribute="top" constant="8" id="P3B-Jo-XMs"/> <constraint firstItem="h6T-x4-yzl" firstAttribute="top" secondItem="cwP-Eh-5dJ" secondAttribute="top" constant="8" id="P3B-Jo-XMs"/>
<constraint firstItem="h6T-x4-yzl" firstAttribute="leading" secondItem="cwP-Eh-5dJ" secondAttribute="leading" constant="4" id="UjP-Gs-ZjO"/> <constraint firstItem="h6T-x4-yzl" firstAttribute="leading" secondItem="cwP-Eh-5dJ" secondAttribute="leading" constant="4" id="UjP-Gs-ZjO"/>
<constraint firstItem="Kzy-5r-UW8" firstAttribute="centerX" secondItem="cwP-Eh-5dJ" secondAttribute="centerX" id="czP-Ia-Ddc"/>
<constraint firstItem="Kzy-5r-UW8" firstAttribute="centerY" secondItem="cwP-Eh-5dJ" secondAttribute="centerY" id="eel-xx-aFq"/>
<constraint firstItem="xRe-ec-Coh" firstAttribute="leading" secondItem="H2p-sc-9uM" secondAttribute="leading" constant="8" id="gRN-PV-gm6"/> <constraint firstItem="xRe-ec-Coh" firstAttribute="leading" secondItem="H2p-sc-9uM" secondAttribute="leading" constant="8" id="gRN-PV-gm6"/>
<constraint firstAttribute="trailing" secondItem="xRe-ec-Coh" secondAttribute="trailing" constant="8" id="tyE-HK-4qb"/> <constraint firstAttribute="trailing" secondItem="xRe-ec-Coh" secondAttribute="trailing" constant="8" id="tyE-HK-4qb"/>
</constraints> </constraints>
</tableViewCellContentView> </tableViewCellContentView>
<viewLayoutGuide key="safeArea" id="njF-e1-oar"/> <viewLayoutGuide key="safeArea" id="njF-e1-oar"/>
<connections> <connections>
<outlet property="activityIndicator" destination="Kzy-5r-UW8" id="lmy-NY-Owu"/>
<outlet property="assetImageView" destination="GLY-o8-47z" id="hZH-ur-m4z"/> <outlet property="assetImageView" destination="GLY-o8-47z" id="hZH-ur-m4z"/>
<outlet property="descriptionPlaceholderLabel" destination="h6T-x4-yzl" id="jBe-R0-Sfn"/> <outlet property="descriptionPlaceholderLabel" destination="h6T-x4-yzl" id="jBe-R0-Sfn"/>
<outlet property="descriptionTextView" destination="cwP-Eh-5dJ" id="pxJ-zF-GKC"/> <outlet property="descriptionTextView" destination="cwP-Eh-5dJ" id="pxJ-zF-GKC"/>