commit df420d47fa51fe4e592941f620a002d31d834091
parent 694b174307386ff11659ebb45bdf5ee32a1c5da8
Author: Michael Camilleri <[email protected]>
Date: Wed, 29 Jul 2026 07:57:16 +0900
Recognise block-square overlay letters
External puzzles can reveal otherwise-unrepresented letters in black
squares through an after-solve overlay. Keeping those tiles only as PNG
data means XD cannot distinguish letter art that Crossmate could render
as native text.
This commit recognises an overlay as text only when every inked tile
lands on an empty block from certain puzzles and the Vision framework
agrees on one uppercase ASCII letter across two backdrops. Any open or
rebus target, lowercase result or ambiguous reading keeps the entire
overlay as PNG data, so styled puzzle content remain lossless. ConVer
rises to 12 so stored exteranl sources are regenerated.
Co-Authored-By: Codex GPT 5.6 Sol <[email protected]>
Diffstat:
7 files changed, 249 insertions(+), 2 deletions(-)
diff --git a/Crossmate.xcodeproj/project.pbxproj b/Crossmate.xcodeproj/project.pbxproj
@@ -42,6 +42,7 @@
1A1A8A9AB36D02E2A5A9ED28 /* GameViewedStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = B9AE0F26E602A9246F5C6ABF /* GameViewedStore.swift */; };
1AAFF86B40CBBFF1EC9ADF9F /* GridThumbnailView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6B1F07B5DDE2A8B49B28392A /* GridThumbnailView.swift */; };
1D08DDEDEF5433912CC6D4DB /* GameViewedStoreTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9406C27662147CD3C0783644 /* GameViewedStoreTests.swift */; };
+ 1E404AD99B04912167B83176 /* NYTOverlayLetterRecognizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 570F8FE85CC46CE3CB5CFEC1 /* NYTOverlayLetterRecognizer.swift */; };
1F4E5473F78A5CEDBA9719CE /* NYTAuthService.swift in Sources */ = {isa = PBXBuildFile; fileRef = A253416F4FEA271A80B22A73 /* NYTAuthService.swift */; };
205894D447BFE2CCEB07575D /* GameStoreRevocationLockTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DD8E8D1BC7DF9EBB08F8CABD /* GameStoreRevocationLockTests.swift */; };
24F7ED458A1C09F8CF309B35 /* PuzzleNotificationText+GameEntity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0DF9C2FEF0D3584864DFC967 /* PuzzleNotificationText+GameEntity.swift */; };
@@ -51,6 +52,7 @@
2641299DE1F2E84E8C21E037 /* LogScrubberTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C06E2CC3A77CB306BD2DF867 /* LogScrubberTests.swift */; };
267ED5B329F05A30430B73A0 /* EngagementHost.swift in Sources */ = {isa = PBXBuildFile; fileRef = 18C701DAE36000DE19F7CC95 /* EngagementHost.swift */; };
26DC22F88FA10C47BC06975E /* PersistenceRecoveryTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4A467BC00116EEC8500BE6A1 /* PersistenceRecoveryTests.swift */; };
+ 2765E30202EF955B425CB1A3 /* NYTOverlayLetterRecognizerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DD8B50452D3AFE25D98B497A /* NYTOverlayLetterRecognizerTests.swift */; };
2A273C98FE3AC5E4C9BE1D88 /* PushPayloadCipherTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6C793B5B9684D49328E42129 /* PushPayloadCipherTests.swift */; };
2AF2550B08CE79F8615B3076 /* FriendZone.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7A4AFF292381C9B33C0F2CD6 /* FriendZone.swift */; };
2C0DFC182240A2519ED1FA6A /* GameMutatorTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = BFC1C59A30FB2571598273E4 /* GameMutatorTests.swift */; };
@@ -377,6 +379,7 @@
52B50A841D92D1F2B173E7DF /* ShareLinkShortener.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShareLinkShortener.swift; sourceTree = "<group>"; };
54A448620E71515AAA7BA3DD /* PersistenceControllerHealTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PersistenceControllerHealTests.swift; sourceTree = "<group>"; };
56BC76178319D0D669CD50FF /* CloudService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudService.swift; sourceTree = "<group>"; };
+ 570F8FE85CC46CE3CB5CFEC1 /* NYTOverlayLetterRecognizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NYTOverlayLetterRecognizer.swift; sourceTree = "<group>"; };
57B1734CF731C2E405A39159 /* PuzzleView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PuzzleView.swift; sourceTree = "<group>"; };
5990D989AD745211A18848E4 /* ShareLinkRouteTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShareLinkRouteTests.swift; sourceTree = "<group>"; };
5C74683332956B0D1CA37589 /* ShareController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ShareController.swift; sourceTree = "<group>"; };
@@ -515,6 +518,7 @@
DB55FC337CF72C650373210A /* PlayerColor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PlayerColor.swift; sourceTree = "<group>"; };
DB851649DE78AAAC5A928C52 /* Square.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Square.swift; sourceTree = "<group>"; };
DBD2570A5A3573D66B3C4A52 /* GameRowView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GameRowView.swift; sourceTree = "<group>"; };
+ DD8B50452D3AFE25D98B497A /* NYTOverlayLetterRecognizerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NYTOverlayLetterRecognizerTests.swift; sourceTree = "<group>"; };
DD8E8D1BC7DF9EBB08F8CABD /* GameStoreRevocationLockTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = GameStoreRevocationLockTests.swift; sourceTree = "<group>"; };
E18FF14E0D73B0D2DB427F08 /* JoiningPuzzleView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = JoiningPuzzleView.swift; sourceTree = "<group>"; };
E25A040EA4DC9672C895A7AC /* InviteEntity+DisplayName.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "InviteEntity+DisplayName.swift"; sourceTree = "<group>"; };
@@ -658,6 +662,7 @@
FEDD63AD5E33E2B0399780EF /* NotificationNavigationBrokerTests.swift */,
47532AED239AEF476D8E9206 /* NotificationStateTests.swift */,
ED8154F949E1D94252F70765 /* NYTAuthServiceTests.swift */,
+ DD8B50452D3AFE25D98B497A /* NYTOverlayLetterRecognizerTests.swift */,
7845E863A179DBB0B932C234 /* NYTOverlaySlicerTests.swift */,
A0CCED43B365363662C45A30 /* NYTPuzzleFetcherTests.swift */,
B34489D0864DF76AF436E391 /* NYTPuzzleUpgraderTests.swift */,
@@ -941,6 +946,7 @@
28AB8C4D560CA5D4E4C61750 /* InviteDeliveryStore.swift */,
33878A29B09A6154C7A63C82 /* KeychainHelper.swift */,
A253416F4FEA271A80B22A73 /* NYTAuthService.swift */,
+ 570F8FE85CC46CE3CB5CFEC1 /* NYTOverlayLetterRecognizer.swift */,
A55219DFC3EEC971E3A8BA6D /* NYTOverlaySlicer.swift */,
B0938B0ACB40772EE522D77C /* NYTPuzzleFetcher.swift */,
CE54EF557E8D808BAA20EA54 /* NYTPuzzleUpgrader.swift */,
@@ -1134,6 +1140,7 @@
F5F333B36654AEAF69A3C220 /* MovesJournalTests.swift in Sources */,
C1930083671621AC79CF95DD /* MovesUpdaterTests.swift in Sources */,
C1D97A4CD02BC9C22C4208BB /* NYTAuthServiceTests.swift in Sources */,
+ 2765E30202EF955B425CB1A3 /* NYTOverlayLetterRecognizerTests.swift in Sources */,
EA9DA088AF131B0E6A874E70 /* NYTOverlaySlicerTests.swift in Sources */,
DCAA1C4A4E4943675EC6632A /* NYTPuzzleFetcherTests.swift in Sources */,
50C02D37A41D55CFA5D307E2 /* NYTPuzzleUpgraderTests.swift in Sources */,
@@ -1281,6 +1288,7 @@
1F4E5473F78A5CEDBA9719CE /* NYTAuthService.swift in Sources */,
C843CADAA263CED503528A4E /* NYTBrowseView.swift in Sources */,
6D2AF361587E43D807BA212F /* NYTLoginView.swift in Sources */,
+ 1E404AD99B04912167B83176 /* NYTOverlayLetterRecognizer.swift in Sources */,
A4BF96473CD6D937402239C1 /* NYTOverlaySlicer.swift in Sources */,
0241DC498C645FE1BDA00FB0 /* NYTPuzzleFetcher.swift in Sources */,
85B9BAC5ED404FE4496250CB /* NYTPuzzleUpgrader.swift in Sources */,
diff --git a/Crossmate/Models/XD.swift b/Crossmate/Models/XD.swift
@@ -10,7 +10,7 @@ struct XD: Sendable {
/// owned NYT game should be re-fetched and re-converted. Only bumped when a
/// converter changes; puzzles generated in-house (Crossmake, bundled) carry
/// no converter version at all.
- static let currentConverterVersion = 11
+ static let currentConverterVersion = 12
/// Version of the XD→Puzzle/Core Data processing (`XD.parse`, `Puzzle(xd:)`,
/// and the cached summary fields). Held on the game entity, never
diff --git a/Crossmate/Services/NYTOverlayLetterRecognizer.swift b/Crossmate/Services/NYTOverlayLetterRecognizer.swift
@@ -0,0 +1,109 @@
+import CoreGraphics
+import Foundation
+import ImageIO
+import Vision
+
+/// Conservatively recognizes NYT after-solve tiles that contain one uppercase
+/// ASCII letter.
+///
+/// This is intentionally not a general image classifier. The converter calls
+/// it only after proving that every inked tile belongs to an empty NYT block
+/// cell. Recognition is all-or-nothing: a missing, lowercase, multi-character,
+/// low-confidence, or rendering-dependent result makes the caller preserve the
+/// complete overlay as image data.
+enum NYTOverlayLetterRecognizer {
+ private static let canvasSize = 192
+ private static let inset = 16
+ private static let minimumConfidence: VNConfidence = 0.5
+ private static let backgrounds: [CGFloat] = [0, 0.25]
+
+ /// Returns one letter for every tile, or `nil` if any tile is ambiguous.
+ static func letters(in tiles: [GridPosition: Data]) -> [GridPosition: Character]? {
+ guard !tiles.isEmpty else { return nil }
+
+ var result: [GridPosition: Character] = [:]
+ result.reserveCapacity(tiles.count)
+ for (position, data) in tiles {
+ guard let source = image(from: data) else { return nil }
+
+ var agreed: Character?
+ for gray in backgrounds {
+ guard let rendered = render(source, backgroundGray: gray),
+ let letter = recognize(rendered) else {
+ return nil
+ }
+ if let agreed, agreed != letter { return nil }
+ agreed = letter
+ }
+ guard let agreed else { return nil }
+ result[position] = agreed
+ }
+ return result
+ }
+
+ /// The acceptance boundary is kept separate so its deliberately strict
+ /// ASCII/case policy can be tested without invoking the OS OCR engine.
+ static func acceptedLetter(_ text: String, confidence: VNConfidence) -> Character? {
+ guard confidence >= minimumConfidence,
+ text.utf8.count == 1,
+ let byte = text.utf8.first,
+ (UInt8(ascii: "A")...UInt8(ascii: "Z")).contains(byte) else {
+ return nil
+ }
+ return Character(text)
+ }
+
+ private static func image(from data: Data) -> CGImage? {
+ guard let source = CGImageSourceCreateWithData(data as CFData, nil) else {
+ return nil
+ }
+ return CGImageSourceCreateImageAtIndex(source, 0, nil)
+ }
+
+ private static func render(_ source: CGImage, backgroundGray: CGFloat) -> CGImage? {
+ guard let context = CGContext(
+ data: nil,
+ width: canvasSize,
+ height: canvasSize,
+ bitsPerComponent: 8,
+ bytesPerRow: 0,
+ space: CGColorSpaceCreateDeviceRGB(),
+ bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue
+ ) else {
+ return nil
+ }
+ context.setFillColor(CGColor(gray: backgroundGray, alpha: 1))
+ context.fill(CGRect(x: 0, y: 0, width: canvasSize, height: canvasSize))
+ context.interpolationQuality = .high
+ context.draw(
+ source,
+ in: CGRect(
+ x: inset,
+ y: inset,
+ width: canvasSize - inset * 2,
+ height: canvasSize - inset * 2
+ )
+ )
+ return context.makeImage()
+ }
+
+ private static func recognize(_ image: CGImage) -> Character? {
+ let request = VNRecognizeTextRequest()
+ request.recognitionLevel = .fast
+ request.recognitionLanguages = ["en-US"]
+ request.usesLanguageCorrection = false
+ request.minimumTextHeight = 0
+
+ do {
+ try VNImageRequestHandler(cgImage: image, options: [:]).perform([request])
+ } catch {
+ return nil
+ }
+ guard let results = request.results,
+ results.count == 1,
+ let candidate = results[0].topCandidates(1).first else {
+ return nil
+ }
+ return acceptedLetter(candidate.string, confidence: candidate.confidence)
+ }
+}
diff --git a/Crossmate/Services/NYTToXDConverter.swift b/Crossmate/Services/NYTToXDConverter.swift
@@ -704,7 +704,61 @@ enum NYTToXDConverter {
) else {
return [:]
}
- return tiles.mapValues { png in
+
+ // NYT sometimes uses overlays to style information already carried by
+ // playable cells (for example the REDR/REDU rebuses in 2021's "Ruby
+ // Lips" puzzle). OCR is only useful for otherwise-empty block cells,
+ // such as the ENERGY letters hidden in the 2026-07-23 grid. Keeping
+ // this gate in terms of the raw NYT cells also prevents PUZ imports or
+ // arbitrary decoration images from ever reaching the recognizer.
+ if overlayTargetsOnlyEmptyBlocks(
+ Set(tiles.keys),
+ body: body,
+ width: width,
+ height: height
+ ), let letters = NYTOverlayLetterRecognizer.letters(in: tiles) {
+ return letters.mapValues { letter in
+ Puzzle.Decoration(content: .text(String(letter)), phase: .after)
+ }
+ }
+
+ return dataDecorations(for: tiles)
+ }
+
+ /// Whether every inked overlay tile lands on a NYT block represented by an
+ /// empty cell dictionary. One ineligible tile rejects text conversion for
+ /// the whole asset, so a mixed overlay is never partially guessed.
+ static func overlayTargetsOnlyEmptyBlocks(
+ _ positions: Set<GridPosition>,
+ body: [String: Any],
+ width: Int,
+ height: Int
+ ) -> Bool {
+ guard !positions.isEmpty,
+ width > 0,
+ height > 0,
+ let cells = body["cells"] as? [Any],
+ case let (cellCount, false) = width.multipliedReportingOverflow(by: height),
+ cells.count == cellCount else {
+ return false
+ }
+ for position in positions {
+ guard position.row >= 0,
+ position.row < height,
+ position.col >= 0,
+ position.col < width,
+ let cell = cells[position.row * width + position.col] as? [String: Any],
+ cell.isEmpty else {
+ return false
+ }
+ }
+ return true
+ }
+
+ private static func dataDecorations(
+ for tiles: [GridPosition: Data]
+ ) -> [GridPosition: Puzzle.Decoration] {
+ tiles.mapValues { png in
Puzzle.Decoration(
content: .data(
mimeType: "image/png",
diff --git a/Scripts/nyt-to-xd.sh b/Scripts/nyt-to-xd.sh
@@ -27,6 +27,7 @@ sources=(
"${repo_root}/Crossmate/Models/PlayerSelection.swift"
"${repo_root}/Crossmate/Models/GridPosition.swift"
"${repo_root}/Crossmate/Services/NYTOverlaySlicer.swift"
+ "${repo_root}/Crossmate/Services/NYTOverlayLetterRecognizer.swift"
"${repo_root}/Crossmate/Services/NYTToXDConverter.swift"
)
for source in "${sources[@]}"; do
diff --git a/Tests/Unit/NYTOverlayLetterRecognizerTests.swift b/Tests/Unit/NYTOverlayLetterRecognizerTests.swift
@@ -0,0 +1,20 @@
+import Testing
+import Vision
+
+@testable import Crossmate
+
+@Suite("NYT overlay letter recognizer")
+struct NYTOverlayLetterRecognizerTests {
+ @Test("Only one uppercase ASCII letter at sufficient confidence is accepted")
+ func acceptsStrictUppercaseASCII() {
+ #expect(NYTOverlayLetterRecognizer.acceptedLetter("A", confidence: 0.5) == "A")
+ #expect(NYTOverlayLetterRecognizer.acceptedLetter("Z", confidence: 1) == "Z")
+
+ #expect(NYTOverlayLetterRecognizer.acceptedLetter("a", confidence: 1) == nil)
+ #expect(NYTOverlayLetterRecognizer.acceptedLetter("É", confidence: 1) == nil)
+ #expect(NYTOverlayLetterRecognizer.acceptedLetter("AB", confidence: 1) == nil)
+ #expect(NYTOverlayLetterRecognizer.acceptedLetter("1", confidence: 1) == nil)
+ #expect(NYTOverlayLetterRecognizer.acceptedLetter("A", confidence: 0.49) == nil)
+ #expect(NYTOverlayLetterRecognizer.acceptedLetter("", confidence: 1) == nil)
+ }
+}
diff --git a/Tests/Unit/NYTToXDConverterTests.swift b/Tests/Unit/NYTToXDConverterTests.swift
@@ -893,4 +893,59 @@ struct NYTToXDConverterTests {
let data = try overlayJSON(afterSolve: 1, assetURIs: [""])
#expect(try NYTToXDConverter.afterSolveImageURL(jsonData: data) == nil)
}
+
+ @Test("Only overlays wholly on empty block dictionaries are eligible for letter OCR")
+ func onlyEmptyBlockOverlaysAreLetterEligible() {
+ let body: [String: Any] = [
+ "cells": [
+ [String: Any](),
+ ["answer": "A"],
+ [String: Any](),
+ ["answer": "REDR", "type": 2],
+ ]
+ ]
+
+ #expect(NYTToXDConverter.overlayTargetsOnlyEmptyBlocks(
+ [GridPosition(row: 0, col: 0), GridPosition(row: 1, col: 0)],
+ body: body,
+ width: 2,
+ height: 2
+ ))
+ #expect(!NYTToXDConverter.overlayTargetsOnlyEmptyBlocks(
+ [GridPosition(row: 0, col: 0), GridPosition(row: 0, col: 1)],
+ body: body,
+ width: 2,
+ height: 2
+ ))
+ #expect(!NYTToXDConverter.overlayTargetsOnlyEmptyBlocks(
+ [GridPosition(row: 1, col: 1)],
+ body: body,
+ width: 2,
+ height: 2
+ ))
+ }
+
+ @Test("An empty, malformed, or out-of-bounds overlay is not letter eligible")
+ func malformedOverlayIsNotLetterEligible() {
+ let body: [String: Any] = ["cells": [[String: Any](), [String: Any]()]]
+
+ #expect(!NYTToXDConverter.overlayTargetsOnlyEmptyBlocks(
+ [],
+ body: body,
+ width: 2,
+ height: 1
+ ))
+ #expect(!NYTToXDConverter.overlayTargetsOnlyEmptyBlocks(
+ [GridPosition(row: 1, col: 0)],
+ body: body,
+ width: 2,
+ height: 1
+ ))
+ #expect(!NYTToXDConverter.overlayTargetsOnlyEmptyBlocks(
+ [GridPosition(row: 0, col: 0)],
+ body: ["cells": [[String: Any]()]],
+ width: 2,
+ height: 1
+ ))
+ }
}
