crossmate

A collaborative crossword app for iOS
Log | Files | Refs | LICENSE

NYTOverlayLetterRecognizer.swift (4051B)

      1 import CoreGraphics
      2 import Foundation
      3 import ImageIO
      4 import Vision
      5 
      6 /// Conservatively recognizes NYT after-solve tiles that contain one uppercase
      7 /// ASCII letter.
      8 ///
      9 /// This is intentionally not a general image classifier. The converter calls
     10 /// it only after proving that every inked tile belongs to an empty NYT block
     11 /// cell. Recognition is all-or-nothing: a missing, lowercase, multi-character,
     12 /// low-confidence, or rendering-dependent result makes the caller preserve the
     13 /// complete overlay as image data.
     14 enum NYTOverlayLetterRecognizer {
     15     private static let canvasSize = 192
     16     private static let inset = 16
     17     private static let minimumConfidence: VNConfidence = 0.5
     18     private static let backgrounds: [CGFloat] = [0, 0.25]
     19 
     20     /// Returns one letter for every tile, or `nil` if any tile is ambiguous.
     21     static func letters(in tiles: [GridPosition: Data]) -> [GridPosition: Character]? {
     22         guard !tiles.isEmpty else { return nil }
     23 
     24         var result: [GridPosition: Character] = [:]
     25         result.reserveCapacity(tiles.count)
     26         for (position, data) in tiles {
     27             guard let source = image(from: data) else { return nil }
     28 
     29             var agreed: Character?
     30             for gray in backgrounds {
     31                 guard let rendered = render(source, backgroundGray: gray),
     32                       let letter = recognize(rendered) else {
     33                     return nil
     34                 }
     35                 if let agreed, agreed != letter { return nil }
     36                 agreed = letter
     37             }
     38             guard let agreed else { return nil }
     39             result[position] = agreed
     40         }
     41         return result
     42     }
     43 
     44     /// The acceptance boundary is kept separate so its deliberately strict
     45     /// ASCII/case policy can be tested without invoking the OS OCR engine.
     46     static func acceptedLetter(_ text: String, confidence: VNConfidence) -> Character? {
     47         guard confidence >= minimumConfidence,
     48               text.utf8.count == 1,
     49               let byte = text.utf8.first,
     50               (UInt8(ascii: "A")...UInt8(ascii: "Z")).contains(byte) else {
     51             return nil
     52         }
     53         return Character(text)
     54     }
     55 
     56     private static func image(from data: Data) -> CGImage? {
     57         guard let source = CGImageSourceCreateWithData(data as CFData, nil) else {
     58             return nil
     59         }
     60         return CGImageSourceCreateImageAtIndex(source, 0, nil)
     61     }
     62 
     63     private static func render(_ source: CGImage, backgroundGray: CGFloat) -> CGImage? {
     64         guard let context = CGContext(
     65             data: nil,
     66             width: canvasSize,
     67             height: canvasSize,
     68             bitsPerComponent: 8,
     69             bytesPerRow: 0,
     70             space: CGColorSpaceCreateDeviceRGB(),
     71             bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue
     72         ) else {
     73             return nil
     74         }
     75         context.setFillColor(CGColor(gray: backgroundGray, alpha: 1))
     76         context.fill(CGRect(x: 0, y: 0, width: canvasSize, height: canvasSize))
     77         context.interpolationQuality = .high
     78         context.draw(
     79             source,
     80             in: CGRect(
     81                 x: inset,
     82                 y: inset,
     83                 width: canvasSize - inset * 2,
     84                 height: canvasSize - inset * 2
     85             )
     86         )
     87         return context.makeImage()
     88     }
     89 
     90     private static func recognize(_ image: CGImage) -> Character? {
     91         let request = VNRecognizeTextRequest()
     92         request.recognitionLevel = .fast
     93         request.recognitionLanguages = ["en-US"]
     94         request.usesLanguageCorrection = false
     95         request.minimumTextHeight = 0
     96 
     97         do {
     98             try VNImageRequestHandler(cgImage: image, options: [:]).perform([request])
     99         } catch {
    100             return nil
    101         }
    102         guard let results = request.results,
    103               results.count == 1,
    104               let candidate = results[0].topCandidates(1).first else {
    105             return nil
    106         }
    107         return acceptedLetter(candidate.string, confidence: candidate.confidence)
    108     }
    109 }