Crossmake/Sources/Wordmake/main.swift (35384B)
1import Foundation
2
3let weekdayNames = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
4let weekdayLookup = Dictionary(uniqueKeysWithValues: weekdayNames.enumerated().map { ($0.element.lowercased(), $0.offset) })
5
6let veryWeakShortWords: Set<String> = [
7 "aah", "aal", "aar", "aba", "abe", "abo", "abu", "ado", "adz", "aer",
8 "ane", "ani", "ara", "ase", "ato", "ava", "een", "eme", "ene", "eon",
9 "ere", "ese", "ess", "est", "eta", "ete", "eth", "ier", "iii", "ile",
10 "ita", "ite", "lai", "lar", "mee", "nee", "oer", "ona", "onea",
11 "ort", "ose", "oto", "ree", "rei", "ria", "ser", "tae", "tba", "tbs",
12 "tko", "tnt", "tpe", "tsp", "ute", "wye", "xed", "xii",
13 "ance", "elee", "orle", "eder", "ssts"
14]
15
16let commonShortWords: Set<String> = [
17 "able", "acre", "act", "acts", "age", "aged", "ages", "air", "airs",
18 "ale", "ales", "all", "ally", "also", "and", "ant", "ante", "anti",
19 "ants", "ape", "apes", "area", "are", "ark", "arks", "arm", "arms",
20 "art", "arts", "ash", "ate", "aunt", "awe", "awed", "awes", "bad",
21 "bag", "bags", "bar", "bare", "bars", "base", "bash", "bat", "bats",
22 "bed", "beds", "bee", "beer", "bees", "bell", "belt", "bent", "best",
23 "bet", "bets", "bite", "blue", "bus", "but", "can", "cap", "car",
24 "care", "case", "cat", "cats", "clay", "cod", "coil", "cold", "cord",
25 "cow", "cue", "curl", "cut", "cuts", "deer", "dog", "dogs", "dry",
26 "ear", "ears", "eat", "eats", "eel", "eels", "era", "eras", "eye",
27 "eyes", "far", "farm", "fast", "fat", "feet", "few", "fire", "fish",
28 "fit", "fits", "free", "fun", "game", "gas", "gate", "get", "gets",
29 "goat", "good", "hair", "hand", "hard", "hat", "hate", "heat", "hen",
30 "her", "here", "hit", "hits", "ice", "idea", "ink", "inn", "iron",
31 "its", "kid", "kids", "knee", "land", "last", "late", "law", "lead",
32 "left", "let", "lets", "lie", "life", "line", "long", "lose", "lost",
33 "man", "many", "map", "men", "mile", "mind", "more", "name", "near",
34 "new", "note", "oar", "oars", "oil", "old", "one", "open", "ore",
35 "ores", "our", "out", "over", "own", "pan", "part", "past", "pen",
36 "pet", "pets", "pie", "pin", "pins", "play", "pot", "red", "rest",
37 "rice", "ring", "road", "root", "rose", "run", "runs", "sad", "salt",
38 "sat", "save", "sea", "seat", "see", "seed", "sees", "set", "sets",
39 "shoe", "side", "sit", "site", "sky", "snow", "son", "song", "star",
40 "step", "stop", "sun", "take", "tea", "team", "ten", "tens", "test",
41 "the", "tie", "ties", "time", "tree", "try", "use", "used", "uses",
42 "war", "way", "west", "wet", "win", "wine", "word", "work", "yard",
43 "year", "yes"
44]
45
46let shortExactFragments: Set<String> = [
47 "ance", "ence", "enne", "ette", "ible", "iest", "ism", "isms", "itis"
48]
49
50let crosswordeseWords: Set<String> = [
51 "adue", "anent", "anet", "antae", "atee", "atit", "atle", "bassi",
52 "brocatelle", "ecce", "ente", "esne", "etes", "gnar", "imset",
53 "labile", "laic", "laram", "mtida", "neer", "onor", "relee", "rete",
54 "terete", "togae", "ulee",
55 "aereo", "alai", "aretes", "aril", "atri", "aussi", "eaude", "ees",
56 "emeer", "enote", "erian", "ers", "esnes", "evoe", "ilia", "leisterer",
57 "mesne", "olea", "ooo", "oss", "ranee", "rorem", "seral",
58 "snee", "soras", "sri", "tse", "yeses",
59 "abas", "adano", "agena", "agin", "agorae", "ams", "ans", "aper",
60 "arear", "bretharte", "donees", "eloi", "endat", "etnas", "etui",
61 "hammshams", "nin", "oas", "ogees", "olan", "ont", "ossa", "otra",
62 "poetaster", "rainintheface", "rea", "retia", "roone", "rpi", "seta",
63 "sisi", "soli", "tet", "tieto", "totoe", "tra", "yser"
64]
65
66let easyAllowedWords: Set<String> = [
67 "n", "s", "e", "w", "ne", "nw", "se", "sw",
68 "nne", "nnw", "ene", "ese", "sse", "ssw", "wsw", "wnw"
69]
70
71struct Options {
72 var inputPath = "Data"
73 var outputPath = "Generated/word_list.json"
74 var countsOutputPath: String?
75 var minLength = 2
76 var maxLength: Int?
77 var minCount = 1
78 var lengthThresholds = false
79 var qualityFilter = false
80 var easyFillFilter = false
81 var badWordsPath: String?
82 var rejectionsOutputPath: String?
83 var qualityOutputPath: String?
84 var weekdays: Set<Int>?
85 var pretty = false
86}
87
88struct AnswerEvidence {
89 var count = 0
90 var badClueCount = 0
91 var obscureNameClueCount = 0
92 var fillBlankCount = 0
93 var foreignLanguageClueCount = 0
94 var cluePartCount = 0
95 var continuationClueCount = 0
96 var themeClueCount = 0
97 var latestSeen: String?
98 var sampleClues: [String] = []
99}
100
101struct WordmakeError: Error, CustomStringConvertible {
102 let description: String
103}
104
105let badClueRegex = try! NSRegularExpression(
106 pattern: #"\b(abbr|var|prefix|suffix|comb\.? form|archaic|poetic|dial|slang|obs|old-style|old fashioned|scot|irish|brit|fr|french|ger|german|lat|latin|span|spanish|ital|italian|port|portuguese|hebr|hebrew|yiddish|jap|japanese|inits|initials|letters|compass point|bearing|direction|shoe width|monogram|degs|degree|of yore|old|olden|heraldry|genus|legal|in law|anatomical|serf|serfs|slave|slaves|vassal|pilaster|opera voices|persian fairy|mideast|eastern v\.?i\.?p|arab prince|arab chieftain|hindu title|indian princess|rajah|dirk|knife of old|seed covering|seed casing|seed envelope|hip bones|pelvic bones|bitter vetch|marsh birds)\b"#,
107 options: [.caseInsensitive]
108)
109let obscureNameClueRegex = try! NSRegularExpression(
110 pattern: #"\b(actor|actress|author|composer|singer|poet|novelist|painter|artist|playwright|violinist|pianist|conductor|golfer|pitcher|baseballer|songwriter|politician|senator|governor|pope)\b"#,
111 options: [.caseInsensitive]
112)
113let fillBlankRegex = try! NSRegularExpression(pattern: #"_{2,}|\.{3,}"#)
114let foreignLanguageClueRegex = try! NSRegularExpression(
115 pattern: #":\s*(?:Sp|Span|Fr|Ger|Germ|It|Ital|Lat|Port|Heb|Yid|Jap|Rus|Gr|Gk|Swed|Norw|Dan|Arab|Chin|Hind|Skt|Turk|Finn)\.|,\s+in\s+[A-Z][a-z]"#
116)
117let cluePartRegex = try! NSRegularExpression(
118 pattern: #"\b(?:part|section|segment|installment)\s+\d+\b|\b\d+(?:st|nd|rd|th)\s+(?:part|section|segment|installment)\b"#,
119 options: [.caseInsensitive]
120)
121let continuationClueRegex = try! NSRegularExpression(
122 pattern: #"\b(?:end|start|middle|continuation)\s+of\s+(?:the\s+)?(?:quote|quip|remark|saying|joke)\b|^see\s+\d+"#,
123 options: [.caseInsensitive]
124)
125let themeClueRegex = try! NSRegularExpression(pattern: #"^\s*\*"#)
126let clueNumberRegex = try! NSRegularExpression(pattern: #"^\s*\d+\.\s*"#)
127let datePathRegex = try! NSRegularExpression(pattern: #"(\d{4})/(\d{2})/(\d{2})\.json$"#)
128
129func printUsage() {
130 print("""
131 Usage: Wordmake [options]
132
133 Options:
134 --input PATH Directory containing crossword JSON files. Default: Data
135 --output PATH Path for the generated JSON word list. Default: Generated/word_list.json
136 --counts-output PATH Optional path for a JSON object of normalized answer frequencies.
137 --min-length N Minimum normalized answer length to include. Default: 2
138 --max-length N Maximum normalized answer length to include.
139 --min-count N Minimum number of appearances required to include an answer. Default: 1
140 --length-thresholds Use length-based appearance thresholds.
141 --quality-filter Use stricter count thresholds intended for generated fill.
142 --easy-fill-filter Use clue-aware filters for easier generated fill.
143 --bad-words PATH Optional JSON word list to exclude from the generated dictionary.
144 --rejections-output PATH Optional path for rejected word metadata from --easy-fill-filter.
145 --quality-output PATH Optional path for per-answer quality metadata from --easy-fill-filter.
146 --weekdays LIST Optional comma-separated weekday filter, e.g. Monday,Tuesday.
147 --pretty Pretty-print generated JSON.
148 -h, --help Show this help.
149 """)
150}
151
152func parseOptions(_ arguments: [String]) throws -> Options {
153 var options = Options()
154 var index = 1
155
156 func requireValue(_ name: String) throws -> String {
157 guard index + 1 < arguments.count else {
158 throw WordmakeError(description: "Missing value for \(name)")
159 }
160 index += 1
161 return arguments[index]
162 }
163
164 while index < arguments.count {
165 let argument = arguments[index]
166 switch argument {
167 case "--input":
168 options.inputPath = try requireValue(argument)
169 case "--output":
170 options.outputPath = try requireValue(argument)
171 case "--counts-output":
172 options.countsOutputPath = try requireValue(argument)
173 case "--min-length":
174 guard let value = Int(try requireValue(argument)) else {
175 throw WordmakeError(description: "--min-length must be an integer")
176 }
177 options.minLength = value
178 case "--max-length":
179 guard let value = Int(try requireValue(argument)) else {
180 throw WordmakeError(description: "--max-length must be an integer")
181 }
182 options.maxLength = value
183 case "--min-count":
184 guard let value = Int(try requireValue(argument)) else {
185 throw WordmakeError(description: "--min-count must be an integer")
186 }
187 options.minCount = value
188 case "--length-thresholds":
189 options.lengthThresholds = true
190 case "--quality-filter":
191 options.qualityFilter = true
192 case "--easy-fill-filter":
193 options.easyFillFilter = true
194 case "--bad-words":
195 options.badWordsPath = try requireValue(argument)
196 case "--rejections-output":
197 options.rejectionsOutputPath = try requireValue(argument)
198 case "--quality-output":
199 options.qualityOutputPath = try requireValue(argument)
200 case "--weekdays":
201 options.weekdays = try parseWeekdays(try requireValue(argument))
202 case "--pretty":
203 options.pretty = true
204 case "--help", "-h":
205 printUsage()
206 exit(0)
207 default:
208 throw WordmakeError(description: "Unknown argument: \(argument)")
209 }
210 index += 1
211 }
212
213 return options
214}
215
216func parseWeekdays(_ value: String) throws -> Set<Int> {
217 var weekdays: Set<Int> = []
218 for rawName in value.split(separator: ",") {
219 let name = rawName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()
220 guard !name.isEmpty else {
221 continue
222 }
223 guard let weekday = weekdayLookup[name] else {
224 throw WordmakeError(description: "Unknown weekday '\(rawName)'; expected one of: \(weekdayNames.joined(separator: ", "))")
225 }
226 weekdays.insert(weekday)
227 }
228 guard !weekdays.isEmpty else {
229 throw WordmakeError(description: "--weekdays must include at least one weekday name")
230 }
231 return weekdays
232}
233
234func normalizeAnswer(_ text: String) -> String {
235 String(text.unicodeScalars.compactMap { scalar in
236 switch scalar.value {
237 case 65...90:
238 return Character(UnicodeScalar(scalar.value + 32)!)
239 case 97...122:
240 return Character(scalar)
241 default:
242 return nil
243 }
244 })
245}
246
247func stripClueNumber(_ clue: String) -> String {
248 let range = NSRange(clue.startIndex..<clue.endIndex, in: clue)
249 return clueNumberRegex.stringByReplacingMatches(in: clue, range: range, withTemplate: "")
250 .trimmingCharacters(in: .whitespacesAndNewlines)
251}
252
253func regexMatches(_ regex: NSRegularExpression, _ text: String) -> Bool {
254 let range = NSRange(text.startIndex..<text.endIndex, in: text)
255 return regex.firstMatch(in: text, range: range) != nil
256}
257
258func jsonFiles(in directory: URL) throws -> [URL] {
259 guard let enumerator = FileManager.default.enumerator(
260 at: directory,
261 includingPropertiesForKeys: [.isRegularFileKey],
262 options: [.skipsHiddenFiles]
263 ) else {
264 return []
265 }
266 return try enumerator.compactMap { item -> URL? in
267 guard let url = item as? URL, url.pathExtension == "json" else {
268 return nil
269 }
270 let values = try url.resourceValues(forKeys: [.isRegularFileKey])
271 return values.isRegularFile == true ? url : nil
272 }
273 .sorted { $0.path < $1.path }
274}
275
276func loadJSONObject(from url: URL) throws -> Any {
277 let data = try Foundation.Data(contentsOf: url)
278 return try JSONSerialization.jsonObject(with: data)
279}
280
281func normalizedAnswers(from puzzle: Any) -> [String] {
282 guard let puzzle = puzzle as? [String: Any],
283 let answers = puzzle["answers"] as? [String: Any] else {
284 return []
285 }
286 return ["across", "down"].flatMap { direction -> [String] in
287 guard let entries = answers[direction] as? [Any] else {
288 return []
289 }
290 return entries.compactMap { entry in
291 guard let answer = entry as? String else {
292 return nil
293 }
294 let normalized = normalizeAnswer(answer)
295 return normalized.isEmpty ? nil : normalized
296 }
297 }
298}
299
300func normalizedEntries(from puzzle: Any) -> [(answer: String, clue: String)] {
301 guard let puzzle = puzzle as? [String: Any],
302 let answers = puzzle["answers"] as? [String: Any],
303 let clues = puzzle["clues"] as? [String: Any] else {
304 return []
305 }
306 return ["across", "down"].flatMap { direction -> [(answer: String, clue: String)] in
307 guard let answerEntries = answers[direction] as? [Any],
308 let clueEntries = clues[direction] as? [Any] else {
309 return []
310 }
311 return zip(answerEntries, clueEntries).compactMap { answerEntry, clueEntry in
312 guard let answer = answerEntry as? String else {
313 return nil
314 }
315 let normalized = normalizeAnswer(answer)
316 guard !normalized.isEmpty else {
317 return nil
318 }
319 let clue = stripClueNumber(clueEntry as? String ?? "")
320 return (normalized, clue)
321 }
322 }
323}
324
325func shouldInclude(_ word: String, minLength: Int, maxLength: Int?) -> Bool {
326 if word.count < minLength {
327 return false
328 }
329 if let maxLength, word.count > maxLength {
330 return false
331 }
332 return true
333}
334
335func weekdayForPath(_ url: URL) -> Int? {
336 let path = url.path.replacingOccurrences(of: "\\", with: "/")
337 let range = NSRange(path.startIndex..<path.endIndex, in: path)
338 guard let match = datePathRegex.firstMatch(in: path, range: range),
339 match.numberOfRanges == 4,
340 let yearRange = Range(match.range(at: 1), in: path),
341 let monthRange = Range(match.range(at: 2), in: path),
342 let dayRange = Range(match.range(at: 3), in: path),
343 let year = Int(path[yearRange]),
344 let month = Int(path[monthRange]),
345 let day = Int(path[dayRange]) else {
346 return nil
347 }
348
349 var components = DateComponents()
350 components.calendar = Calendar(identifier: .gregorian)
351 components.year = year
352 components.month = month
353 components.day = day
354 guard let date = components.date,
355 let calendarWeekday = components.calendar?.component(.weekday, from: date) else {
356 return nil
357 }
358 return (calendarWeekday + 5) % 7
359}
360
361func isoDateForPath(_ url: URL) -> String? {
362 let path = url.path.replacingOccurrences(of: "\\", with: "/")
363 let range = NSRange(path.startIndex..<path.endIndex, in: path)
364 guard let match = datePathRegex.firstMatch(in: path, range: range),
365 match.numberOfRanges == 4,
366 let yearRange = Range(match.range(at: 1), in: path),
367 let monthRange = Range(match.range(at: 2), in: path),
368 let dayRange = Range(match.range(at: 3), in: path) else {
369 return nil
370 }
371
372 return "\(path[yearRange])-\(path[monthRange])-\(path[dayRange])"
373}
374
375func puzzleWeekday(_ puzzle: Any, path: URL) -> Int? {
376 if let puzzle = puzzle as? [String: Any],
377 let dow = puzzle["dow"] as? String,
378 let weekday = weekdayLookup[dow.trimmingCharacters(in: .whitespacesAndNewlines).lowercased()] {
379 return weekday
380 }
381 return weekdayForPath(path)
382}
383
384func shouldIncludePuzzle(_ puzzle: Any, path: URL, weekdays: Set<Int>?) -> Bool {
385 guard let weekdays else {
386 return true
387 }
388 guard let weekday = puzzleWeekday(puzzle, path: path) else {
389 return false
390 }
391 return weekdays.contains(weekday)
392}
393
394func requiredCount(word: String, flatMinCount: Int, lengthThresholds: Bool, qualityFilter: Bool) -> Int {
395 if qualityFilter {
396 if word.count < 5 {
397 return 15
398 }
399 if word.count > 7 {
400 return 1
401 }
402 return 3
403 }
404 guard lengthThresholds else {
405 return flatMinCount
406 }
407 if word.count < 5 {
408 return 10
409 }
410 if word.count > 7 {
411 return 1
412 }
413 return 2
414}
415
416func easyRequiredCount(_ word: String) -> Int {
417 if word.count <= 4 {
418 return 15
419 }
420 if word.count <= 7 {
421 return 3
422 }
423 return 1
424}
425
426func hasVowel(_ word: String) -> Bool {
427 word.contains { "aeiouy".contains($0) }
428}
429
430func clueRatio(_ part: Int, _ whole: Int) -> Double {
431 Double(part) / Double(max(whole, 1))
432}
433
434func hasSequentialLetterRun(_ word: String, minimumLength: Int = 4) -> Bool {
435 let scalars = word.unicodeScalars.map(\.value)
436 guard scalars.count >= minimumLength else {
437 return false
438 }
439
440 var runLength = 1
441 for index in 1..<scalars.count {
442 if scalars[index] == scalars[index - 1] + 1 {
443 runLength += 1
444 if runLength >= minimumLength {
445 return true
446 }
447 } else {
448 runLength = 1
449 }
450 }
451 return false
452}
453
454func isInitialSurnameArtifact(_ word: String) -> Bool {
455 let fusedNames: Set<String> = [
456 "ed", "ek", "el", "em", "et",
457 "jk", "jl", "jr",
458 "kd", "kt",
459 "mj",
460 "oj",
461 "tj",
462 "wc"
463 ]
464
465 return fusedNames.contains { word.hasPrefix($0) && word.count >= 6 }
466}
467
468func isPhraseFragmentArtifact(_ word: String, fillBlankRatio: Double) -> Bool {
469 guard word.count >= 5 else {
470 return false
471 }
472 let prefixes = [
473 "aan", "aand", "anold", "asof", "ated", "cani",
474 "cular",
475 "igota", "igot", "imeta", "imet", "ihave", "itisa", "itwas",
476 "weare", "youre"
477 ]
478 if prefixes.contains(where: { word.hasPrefix($0) }) {
479 return true
480 }
481 if word.count >= 10 {
482 return fillBlankRatio >= 0.8
483 }
484 return fillBlankRatio >= 0.8 && word.count <= 7
485}
486
487func isVerySuspiciousOneOff(_ word: String, evidence: AnswerEvidence) -> Bool {
488 guard word.count >= 8, evidence.count == 1, evidence.latestSeen != nil else {
489 return false
490 }
491 if evidence.obscureNameClueCount > 0 {
492 return true
493 }
494 if word.hasSuffix("c") || word.hasSuffix("v") || word.hasSuffix("x") {
495 return true
496 }
497 return false
498}
499
500func isStaleOneOffLongFill(_ word: String, evidence: AnswerEvidence) -> Bool {
501 guard word.count >= 8,
502 evidence.count == 1,
503 let latestSeen = evidence.latestSeen else {
504 return false
505 }
506 return latestSeen < "2000-01-01"
507}
508
509func isPunnyOneOff(_ word: String, evidence: AnswerEvidence) -> Bool {
510 guard word.count >= 8, evidence.count == 1 else {
511 return false
512 }
513 return evidence.sampleClues.contains { $0.contains("?") }
514}
515
516func stricterShortRequiredCount(_ word: String) -> Int? {
517 switch word.count {
518 case ...2:
519 return nil
520 case 3:
521 return 25
522 case 4:
523 return 12
524 case 5:
525 return 6
526 default:
527 return nil
528 }
529}
530
531func easyRejectionReason(word: String, evidence: AnswerEvidence) -> String? {
532 let length = word.count
533 let count = evidence.count
534
535 if easyAllowedWords.contains(word) {
536 return nil
537 }
538
539 if crosswordeseWords.contains(word) {
540 return "known crosswordese/glue fill"
541 }
542
543 let badRatio = clueRatio(evidence.badClueCount, count)
544 let obscureNameRatio = clueRatio(evidence.obscureNameClueCount, count)
545 let fillBlankRatio = clueRatio(evidence.fillBlankCount, count)
546 let foreignLanguageRatio = clueRatio(evidence.foreignLanguageClueCount, count)
547 let cluePartRatio = clueRatio(evidence.cluePartCount, count)
548
549 if count < easyRequiredCount(word) {
550 return "count \(count) below easy threshold \(easyRequiredCount(word))"
551 }
552
553 if let stricterCount = stricterShortRequiredCount(word),
554 !commonShortWords.contains(word),
555 count < stricterCount {
556 return "short entry count \(count) below generated-fill threshold \(stricterCount)"
557 }
558
559 if length <= 5 && count < 80 && evidence.badClueCount >= 2 && badRatio >= 0.35 {
560 return "short entry mostly clued as abbreviation/variant/form"
561 }
562
563 if length <= 5 &&
564 !commonShortWords.contains(word),
565 evidence.latestSeen.map({ $0 < "2000-01-01" }) == true {
566 return "stale short fill"
567 }
568
569 if hasSequentialLetterRun(word) {
570 return "alphabet-run artifact"
571 }
572
573 if length >= 5 && !hasVowel(word) {
574 return "long entry with no vowel"
575 }
576
577 if isInitialSurnameArtifact(word) {
578 return "initial-plus-name artifact"
579 }
580
581 if isPhraseFragmentArtifact(word, fillBlankRatio: fillBlankRatio) {
582 return "phrase fragment artifact"
583 }
584
585 if evidence.themeClueCount > 0 && (count <= 2 || clueRatio(evidence.themeClueCount, count) >= 0.5) {
586 return "one-off theme clue artifact"
587 }
588
589 if evidence.continuationClueCount > 0 && (count <= 3 || clueRatio(evidence.continuationClueCount, count) >= 0.5) {
590 return "cross-reference or quote-continuation artifact"
591 }
592
593 if isStaleOneOffLongFill(word, evidence: evidence) {
594 return "stale one-off long fill"
595 }
596
597 if isVerySuspiciousOneOff(word, evidence: evidence) {
598 return "one-off long artifact"
599 }
600
601 if isPunnyOneOff(word, evidence: evidence) {
602 return "one-off pun/theme artifact"
603 }
604
605 if length <= 5 && evidence.badClueCount >= 2 && badRatio >= 0.45 {
606 return "entry mostly clued as abbreviation/variant/form"
607 }
608
609 if length <= 5 && evidence.obscureNameClueCount >= 2 && obscureNameRatio >= 0.5 {
610 return "short entry mostly clued as obscure name/title"
611 }
612
613 if length <= 4 && commonShortWords.contains(word) {
614 return nil
615 }
616
617 if length <= 4 && veryWeakShortWords.contains(word) {
618 return "known weak short fill"
619 }
620
621 if length <= 4 && !hasVowel(word) {
622 return "short entry with no vowel"
623 }
624
625 if length <= 5 && shortExactFragments.contains(word) {
626 return "short suffix-like fragment"
627 }
628
629 if evidence.cluePartCount > 0 && (count <= 3 || cluePartRatio >= 0.5) {
630 return "entry mostly clued as one part of a longer answer"
631 }
632
633 if length <= 5 &&
634 evidence.foreignLanguageClueCount >= 2 &&
635 (count <= 10 || foreignLanguageRatio >= 0.5) {
636 return "entry mostly clued as foreign-language translation"
637 }
638
639 if length <= 4 && evidence.badClueCount >= 2 && badRatio >= 0.18 {
640 return "short entry mostly clued as abbreviation/variant/foreign/form"
641 }
642
643 if length <= 4 && evidence.obscureNameClueCount >= 4 && obscureNameRatio >= 0.35 {
644 return "short entry mostly clued as obscure name"
645 }
646
647 if length <= 4 && count < 40 && evidence.fillBlankCount >= 8 && fillBlankRatio >= 0.55 {
648 return "short entry mostly dependent on fill-in-the-blank clues"
649 }
650
651 if length <= 3 && count < 60 && (badRatio + obscureNameRatio + fillBlankRatio) >= 0.45 {
652 return "rare short entry with weak clue evidence"
653 }
654
655 return nil
656}
657
658func loadAnswers(inputURL: URL, options: Options) throws -> (counts: [String: Int], filesRead: Int, filesUsed: Int) {
659 var counts: [String: Int] = [:]
660 var filesRead = 0
661 var filesUsed = 0
662
663 for path in try jsonFiles(in: inputURL) {
664 do {
665 let puzzle = try loadJSONObject(from: path)
666 filesRead += 1
667 guard shouldIncludePuzzle(puzzle, path: path, weekdays: options.weekdays) else {
668 continue
669 }
670 filesUsed += 1
671 for answer in normalizedAnswers(from: puzzle) where shouldInclude(answer, minLength: options.minLength, maxLength: options.maxLength) {
672 counts[answer, default: 0] += 1
673 }
674 } catch {
675 fputs("Skipping \(path.path): \(error)\n", stderr)
676 }
677 }
678
679 return (counts, filesRead, filesUsed)
680}
681
682func loadAnswerEvidence(inputURL: URL, options: Options) throws -> (evidence: [String: AnswerEvidence], filesRead: Int, filesUsed: Int) {
683 var evidence: [String: AnswerEvidence] = [:]
684 var filesRead = 0
685 var filesUsed = 0
686
687 for path in try jsonFiles(in: inputURL) {
688 do {
689 let puzzle = try loadJSONObject(from: path)
690 filesRead += 1
691 guard shouldIncludePuzzle(puzzle, path: path, weekdays: options.weekdays) else {
692 continue
693 }
694 let latestSeen = isoDateForPath(path)
695 filesUsed += 1
696 for (answer, clue) in normalizedEntries(from: puzzle) where shouldInclude(answer, minLength: options.minLength, maxLength: options.maxLength) {
697 var entry = evidence[answer] ?? AnswerEvidence()
698 entry.count += 1
699 if let latestSeen {
700 if entry.latestSeen.map({ latestSeen > $0 }) ?? true {
701 entry.latestSeen = latestSeen
702 }
703 }
704 if entry.sampleClues.count < 5 && !clue.isEmpty {
705 entry.sampleClues.append(clue)
706 }
707 if regexMatches(badClueRegex, clue) {
708 entry.badClueCount += 1
709 }
710 if regexMatches(obscureNameClueRegex, clue) {
711 entry.obscureNameClueCount += 1
712 }
713 if regexMatches(fillBlankRegex, clue) {
714 entry.fillBlankCount += 1
715 }
716 if regexMatches(foreignLanguageClueRegex, clue) {
717 entry.foreignLanguageClueCount += 1
718 }
719 if regexMatches(cluePartRegex, clue) {
720 entry.cluePartCount += 1
721 }
722 if regexMatches(continuationClueRegex, clue) {
723 entry.continuationClueCount += 1
724 }
725 if regexMatches(themeClueRegex, clue) {
726 entry.themeClueCount += 1
727 }
728 evidence[answer] = entry
729 }
730 } catch {
731 fputs("Skipping \(path.path): \(error)\n", stderr)
732 }
733 }
734
735 return (evidence, filesRead, filesUsed)
736}
737
738func loadBadWords(path: String) throws -> Set<String> {
739 let value = try loadJSONObject(from: URL(fileURLWithPath: path))
740 guard let words = value as? [Any] else {
741 throw WordmakeError(description: "expected JSON array")
742 }
743 return Set(words.compactMap { word in
744 guard let word = word as? String else {
745 return nil
746 }
747 let normalized = normalizeAnswer(word)
748 return normalized.isEmpty ? nil : normalized
749 })
750}
751
752func sortedDictionary(_ dictionary: [String: Any]) -> [String: Any] {
753 Dictionary(uniqueKeysWithValues: dictionary.keys.sorted().map { ($0, dictionary[$0]!) })
754}
755
756func evidenceObject(_ entry: AnswerEvidence, rejectionReason: String?) -> [String: Any] {
757 [
758 "count": entry.count,
759 "bad_clue_count": entry.badClueCount,
760 "obscure_name_clue_count": entry.obscureNameClueCount,
761 "fill_blank_count": entry.fillBlankCount,
762 "foreign_language_clue_count": entry.foreignLanguageClueCount,
763 "clue_part_count": entry.cluePartCount,
764 "continuation_clue_count": entry.continuationClueCount,
765 "theme_clue_count": entry.themeClueCount,
766 "latest_seen": entry.latestSeen as Any? ?? NSNull(),
767 "sample_clues": entry.sampleClues,
768 "rejection_reason": rejectionReason as Any? ?? NSNull()
769 ]
770}
771
772func rejectionObject(reason: String, entry: AnswerEvidence) -> [String: Any] {
773 [
774 "reason": reason,
775 "count": entry.count,
776 "bad_clue_count": entry.badClueCount,
777 "obscure_name_clue_count": entry.obscureNameClueCount,
778 "fill_blank_count": entry.fillBlankCount,
779 "foreign_language_clue_count": entry.foreignLanguageClueCount,
780 "clue_part_count": entry.cluePartCount,
781 "continuation_clue_count": entry.continuationClueCount,
782 "theme_clue_count": entry.themeClueCount,
783 "latest_seen": entry.latestSeen as Any? ?? NSNull(),
784 "sample_clues": entry.sampleClues
785 ]
786}
787
788func writeJSON(_ value: Any, to path: String, pretty: Bool) throws {
789 let url = URL(fileURLWithPath: path)
790 let directory = url.deletingLastPathComponent()
791 if directory.path != "." {
792 try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
793 }
794 var options: JSONSerialization.WritingOptions = [.sortedKeys]
795 if pretty {
796 options.insert(.prettyPrinted)
797 }
798 let data = try JSONSerialization.data(withJSONObject: value, options: options)
799 var output = data
800 output.append(0x0A)
801 try output.write(to: url)
802}
803
804func validate(_ options: Options, inputURL: URL) throws {
805 var isDirectory: ObjCBool = false
806 guard FileManager.default.fileExists(atPath: inputURL.path, isDirectory: &isDirectory), isDirectory.boolValue else {
807 throw WordmakeError(description: "Input directory not found: \(options.inputPath)")
808 }
809 guard options.minLength >= 1 else {
810 throw WordmakeError(description: "--min-length must be at least 1")
811 }
812 if let maxLength = options.maxLength, maxLength < options.minLength {
813 throw WordmakeError(description: "--max-length must be greater than or equal to --min-length")
814 }
815 guard options.minCount >= 1 else {
816 throw WordmakeError(description: "--min-count must be at least 1")
817 }
818 let filters = [options.lengthThresholds, options.qualityFilter, options.easyFillFilter].filter { $0 }.count
819 guard filters <= 1 else {
820 throw WordmakeError(description: "Choose only one of --length-thresholds, --quality-filter, or --easy-fill-filter")
821 }
822 if (options.rejectionsOutputPath != nil || options.qualityOutputPath != nil) && !options.easyFillFilter {
823 throw WordmakeError(description: "--rejections-output and --quality-output require --easy-fill-filter")
824 }
825}
826
827func run() throws {
828 let options = try parseOptions(CommandLine.arguments)
829 let inputURL = URL(fileURLWithPath: options.inputPath)
830 try validate(options, inputURL: inputURL)
831
832 let badWords: Set<String>
833 if let badWordsPath = options.badWordsPath {
834 do {
835 badWords = try loadBadWords(path: badWordsPath)
836 } catch {
837 throw WordmakeError(description: "Could not read --bad-words \(badWordsPath): \(error)")
838 }
839 } else {
840 badWords = []
841 }
842
843 var counts: [String: Int]
844 let filesRead: Int
845 let filesUsed: Int
846 var words: [String]
847 var easyRejections: [String: Any] = [:]
848 var wordQuality: [String: Any] = [:]
849
850 if options.easyFillFilter {
851 let result = try loadAnswerEvidence(inputURL: inputURL, options: options)
852 filesRead = result.filesRead
853 filesUsed = result.filesUsed
854 counts = Dictionary(uniqueKeysWithValues: result.evidence.map { ($0.key, $0.value.count) })
855 words = []
856
857 for word in result.evidence.keys.sorted() {
858 guard let entry = result.evidence[word] else {
859 continue
860 }
861 let reason = easyRejectionReason(word: word, evidence: entry)
862 let finalReason = badWords.contains(word) ? "bad word" : reason
863 wordQuality[word] = evidenceObject(entry, rejectionReason: finalReason)
864 if badWords.contains(word) {
865 continue
866 }
867 if let reason {
868 easyRejections[word] = rejectionObject(reason: reason, entry: entry)
869 } else {
870 words.append(word)
871 }
872 }
873 } else {
874 let result = try loadAnswers(inputURL: inputURL, options: options)
875 counts = result.counts
876 filesRead = result.filesRead
877 filesUsed = result.filesUsed
878 words = counts.keys
879 .filter { word in
880 !badWords.contains(word) &&
881 (counts[word] ?? 0) >= requiredCount(
882 word: word,
883 flatMinCount: options.minCount,
884 lengthThresholds: options.lengthThresholds,
885 qualityFilter: options.qualityFilter
886 )
887 }
888 .sorted()
889 }
890
891 try writeJSON(words, to: options.outputPath, pretty: options.pretty)
892 if let countsOutputPath = options.countsOutputPath {
893 try writeJSON(Dictionary(uniqueKeysWithValues: counts.keys.sorted().map { ($0, counts[$0]!) }), to: countsOutputPath, pretty: options.pretty)
894 }
895 if let rejectionsOutputPath = options.rejectionsOutputPath {
896 try writeJSON(sortedDictionary(easyRejections), to: rejectionsOutputPath, pretty: options.pretty)
897 }
898 if let qualityOutputPath = options.qualityOutputPath {
899 try writeJSON(sortedDictionary(wordQuality), to: qualityOutputPath, pretty: options.pretty)
900 }
901
902 print("Read \(filesRead) puzzle files")
903 if let weekdays = options.weekdays {
904 let selected = weekdays.sorted().map { weekdayNames[$0] }.joined(separator: ", ")
905 print("Used \(filesUsed) \(selected) puzzle files")
906 }
907 print("Wrote \(words.count) unique answers to \(options.outputPath)")
908 if options.lengthThresholds {
909 print("Included answers using length thresholds: <5 letters >=10, 5-7 letters >=2, >7 letters >=1")
910 } else if options.qualityFilter {
911 print("Included answers using quality filter: <5 letters >=15, 5-7 letters >=3, >7 letters >=1")
912 } else if options.easyFillFilter {
913 print("Included answers using clue-aware easy fill filter")
914 print("Rejected \(easyRejections.count) answers with easy fill filter")
915 } else if options.minCount > 1 {
916 print("Included answers appearing at least \(options.minCount) times")
917 }
918 if let badWordsPath = options.badWordsPath {
919 print("Excluded \(badWords.count) bad words from \(badWordsPath)")
920 }
921 if let countsOutputPath = options.countsOutputPath {
922 print("Wrote answer frequencies to \(countsOutputPath)")
923 }
924 if let rejectionsOutputPath = options.rejectionsOutputPath {
925 print("Wrote easy-filter rejections to \(rejectionsOutputPath)")
926 }
927 if let qualityOutputPath = options.qualityOutputPath {
928 print("Wrote word quality metadata to \(qualityOutputPath)")
929 }
930}
931
932do {
933 try run()
934} catch {
935 fputs("Wordmake: \(error)\n", stderr)
936 exit(1)
937}