Crossmate/Sync/CloudDiagnostics.swift (26626B)
1import CloudKit
2import Foundation
3
4extension SyncEngine {
5 private struct StorageAuditAsset {
6 let containerLabel: String
7 let recordType: String
8 let field: String
9 let recordName: String
10 let zoneName: String
11 let bytes: Int64
12 }
13
14 private struct StorageAuditFieldTotal {
15 var count = 0
16 var bytes: Int64 = 0
17 var unavailableCount = 0
18 }
19
20 private struct StorageAuditTotals {
21 var zoneCount = 0
22 var recordCount = 0
23 var title: String?
24 var recordCounts: [String: Int] = [:]
25 var assetFields: [String: StorageAuditFieldTotal] = [:]
26 var inlineBytes: Int64 = 0
27 var unavailableAssetCount = 0
28 var recordErrorCount = 0
29 var assets: [StorageAuditAsset] = []
30
31 var assetBytes: Int64 {
32 assetFields.values.reduce(0) { $0 + $1.bytes }
33 }
34
35 var assetCount: Int {
36 assetFields.values.reduce(0) { $0 + $1.count }
37 }
38
39 mutating func merge(_ other: StorageAuditTotals) {
40 zoneCount += other.zoneCount
41 recordCount += other.recordCount
42 inlineBytes += other.inlineBytes
43 unavailableAssetCount += other.unavailableAssetCount
44 recordErrorCount += other.recordErrorCount
45 assets.append(contentsOf: other.assets)
46 for (type, count) in other.recordCounts {
47 recordCounts[type, default: 0] += count
48 }
49 for (field, value) in other.assetFields {
50 assetFields[field, default: StorageAuditFieldTotal()].count += value.count
51 assetFields[field, default: StorageAuditFieldTotal()].bytes += value.bytes
52 assetFields[field, default: StorageAuditFieldTotal()].unavailableCount +=
53 value.unavailableCount
54 }
55 }
56 }
57
58 private struct DocumentsAuditFile: Sendable {
59 let path: String
60 let bytes: Int64
61 }
62
63 private struct DocumentsAuditTotals: Sendable {
64 var fileCount = 0
65 var bytes: Int64 = 0
66 var unavailableCount = 0
67 var files: [DocumentsAuditFile] = []
68 }
69
70 struct DiagnosticSnapshot: Sendable {
71 let accountStatus: CKAccountStatus
72 let engineRunning: Bool
73 let pendingChangesCount: Int
74 let privatePendingCount: Int
75 let sharedPendingCount: Int
76 let pendingInvitationCount: Int
77 }
78
79
80
81
82 func pendingSaveRecordNames(scope: CKDatabase.Scope) -> [String] {
83 let engine = scope == .shared ? sharedEngine : privateEngine
84 guard let engine else { return [] }
85 return engine.state.pendingRecordZoneChanges.compactMap {
86 if case .saveRecord(let id) = $0 { return id.recordName }
87 return nil
88 }
89 }
90
91
92
93 func pendingDeletedZoneNames(scope: CKDatabase.Scope) -> [String] {
94 let engine = scope == .shared ? sharedEngine : privateEngine
95 guard let engine else { return [] }
96 return engine.state.pendingDatabaseChanges.compactMap {
97 if case .deleteZone(let id) = $0 { return id.zoneName }
98 return nil
99 }
100 }
101
102 func diagnosticSnapshot() async -> DiagnosticSnapshot {
103 let status: CKAccountStatus
104 do { status = try await container.accountStatus() }
105 catch { status = .couldNotDetermine }
106 let running = privateEngine != nil
107 let privateCount = privateEngine.map { $0.state.pendingRecordZoneChanges.count } ?? 0
108 let sharedCount = sharedEngine.map { $0.state.pendingRecordZoneChanges.count } ?? 0
109 return DiagnosticSnapshot(
110 accountStatus: status,
111 engineRunning: running,
112 pendingChangesCount: privateCount + sharedCount,
113 privatePendingCount: privateCount,
114 sharedPendingCount: sharedCount,
115 pendingInvitationCount: pendingInvitationPingCount()
116 )
117 }
118
119
120
121 func probeContainer() async -> [(name: String, result: String)] {
122 var results: [(String, String)] = []
123 results.append(("containerIdentifier", container.containerIdentifier ?? "nil"))
124 do {
125 let s = try await container.accountStatus()
126 results.append(("accountStatus", describeStatus(s)))
127 } catch {
128 results.append(("accountStatus", describe(error)))
129 }
130 do {
131 let id = try await container.userRecordID()
132 results.append(("userRecordID", id.recordName))
133 } catch {
134 results.append(("userRecordID", describe(error)))
135 }
136 do {
137 let zones = try await container.privateCloudDatabase.allRecordZones()
138 let names = zones.map(\.zoneID.zoneName).joined(separator: ", ")
139 results.append(("privateZones", "\(zones.count) zone(s): [\(names)]"))
140 } catch {
141 results.append(("privateZones", describe(error)))
142 }
143 do {
144 let zones = try await container.sharedCloudDatabase.allRecordZones()
145 let names = zones.map(\.zoneID.zoneName).joined(separator: ", ")
146 results.append(("sharedZones", "\(zones.count) zone(s): [\(names)]"))
147 } catch {
148 results.append(("sharedZones", describe(error)))
149 }
150
151
152
153
154
155 results.append(await probeSubscriptions(database: container.privateCloudDatabase, label: "privateSubs"))
156 results.append(await probeSubscriptions(database: container.sharedCloudDatabase, label: "sharedSubs"))
157 return results
158 }
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178 func auditPrivateCloudStorage(
179 progress: @MainActor @Sendable (String) -> Void
180 ) async {
181 let containers: [(label: String, container: CKContainer)] = [
182 ("v4", container),
183 ("v3", CloudContainer.v3Container),
184 ("v2", CloudContainer.v2Container),
185 ("v1", CloudContainer.v1Container),
186 ]
187 let labels = containers.map(\.label).joined(separator: ", ")
188 await progress(
189 "storage audit: starting entitled containers [\(labels)] in the " +
190 "\(cloudEnvironment()) CloudKit environment"
191 )
192
193 var grandTotals = StorageAuditTotals()
194 var unreadable: [String] = []
195 for source in containers {
196 if let totals = await auditPrivateCloudStorage(
197 label: source.label,
198 container: source.container,
199 progress: progress
200 ) {
201 grandTotals.merge(totals)
202 } else {
203 unreadable.append(source.label)
204 }
205 }
206
207 await logStorageTotals(
208 grandTotals,
209 prefix: "storage audit CloudKit grand totals",
210 includeLargest: true,
211 progress: progress
212 )
213
214
215
216 if unreadable.isEmpty {
217 await progress("storage audit coverage: every entitled container was read")
218 } else {
219 await progress(
220 "storage audit coverage INCOMPLETE: grand totals omit " +
221 "[\(unreadable.joined(separator: ", "))] — those containers failed to " +
222 "list zones, so any bytes they hold are unmeasured"
223 )
224 }
225 await auditICloudDocuments(progress: progress)
226 await progress(
227 "storage audit complete: measured bytes exclude CloudKit metadata, shares, " +
228 "zone overhead, encryption overhead, and server-retained state"
229 )
230 }
231
232
233
234
235
236
237 private nonisolated func cloudEnvironment() -> String {
238 let value = Bundle.main.object(forInfoDictionaryKey: "CrossmateICloudEnvironment")
239 guard let environment = value as? String, !environment.isEmpty else {
240 return "unreported"
241 }
242 return environment
243 }
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261 @discardableResult
262 func deleteLegacyCloudData(
263 progress: @MainActor @Sendable (String) -> Void
264 ) async -> Bool {
265 let container = CloudContainer.v3Container
266 let identifier = container.containerIdentifier ?? "unknown"
267 let database = container.privateCloudDatabase
268 await progress(
269 "legacy delete: starting v3 [\(identifier)] in the " +
270 "\(cloudEnvironment()) CloudKit environment"
271 )
272
273 let zones: [CKRecordZone]
274 do {
275 zones = try await database.allRecordZones()
276 } catch {
277 await progress("legacy delete: couldn't list zones — \(describe(error)) — nothing deleted")
278 return false
279 }
280
281 let defaultZoneID = CKRecordZone.default().zoneID
282 let customZoneIDs = zones
283 .map(\.zoneID)
284 .filter { $0 != defaultZoneID }
285 .sorted { $0.zoneName < $1.zoneName }
286 await progress("legacy delete: \(customZoneIDs.count) custom zone(s) to remove")
287
288 var deleted = 0
289 var failed = 0
290
291
292 for start in stride(from: 0, to: customZoneIDs.count, by: 20) {
293 let batch = Array(customZoneIDs[start..<min(start + 20, customZoneIDs.count)])
294 do {
295 let results = try await database.modifyRecordZones(
296 saving: [],
297 deleting: batch
298 )
299 for (zoneID, result) in results.deleteResults {
300 switch result {
301 case .success:
302 deleted += 1
303 case .failure(let error):
304 failed += 1
305 await progress(
306 "legacy delete: zone [\(zoneID.zoneName)] FAILED — \(describe(error))"
307 )
308 }
309 }
310 } catch {
311 failed += batch.count
312 await progress("legacy delete: batch of \(batch.count) FAILED — \(describe(error))")
313 }
314 await progress(
315 "legacy delete: \(deleted) deleted, \(failed) failed of \(customZoneIDs.count)"
316 )
317 }
318
319 let swept = failed == 0
320 await progress(
321 "legacy delete complete: zonesDeleted=\(deleted) zonesFailed=\(failed) " +
322 "swept=\(swept)"
323 )
324 return swept
325 }
326
327 private func auditPrivateCloudStorage(
328 label: String,
329 container: CKContainer,
330 progress: @MainActor @Sendable (String) -> Void
331 ) async -> StorageAuditTotals? {
332 let identifier = container.containerIdentifier ?? "unknown"
333 let database = container.privateCloudDatabase
334 await progress("storage audit container \(label) [\(identifier)]: starting private database")
335
336 let zones: [CKRecordZone]
337 do {
338 zones = try await database.allRecordZones()
339 } catch {
340 await progress(
341 "storage audit container \(label): couldn't list private zones — \(describe(error))"
342 )
343 return nil
344 }
345
346 let defaultZoneID = CKRecordZone.default().zoneID
347 let customZones = zones
348 .map(\.zoneID)
349 .filter { $0 != defaultZoneID }
350 .sorted { $0.zoneName < $1.zoneName }
351 await progress(
352 "storage audit container \(label): found \(customZones.count) custom zone(s); " +
353 "asset contents will be downloaded"
354 )
355
356 var totals = StorageAuditTotals()
357 for (index, zoneID) in customZones.enumerated() {
358 do {
359 let zone = try await auditStorageZone(zoneID, containerLabel: label, in: database)
360 var completedZone = zone
361 completedZone.zoneCount = 1
362 totals.merge(completedZone)
363
364 let types = formattedCounts(zone.recordCounts)
365 let title = zone.title.map { " title=[\(singleLine($0))]" } ?? ""
366 await progress(
367 "storage audit container \(label) zone \(index + 1)/\(customZones.count) " +
368 "[\(zoneID.zoneName)]:\(title) records=\(zone.recordCount) " +
369 "assets=\(zone.assetCount) assetBytes=\(formatBytes(zone.assetBytes)) " +
370 "inline=\(formatBytes(zone.inlineBytes)) " +
371 "types=[\(types)]"
372 )
373 } catch {
374 await progress(
375 "storage audit container \(label) zone \(index + 1)/\(customZones.count) " +
376 "[\(zoneID.zoneName)] FAILED — \(describe(error))"
377 )
378 }
379 }
380
381 await logStorageTotals(
382 totals,
383 prefix: "storage audit container \(label) totals",
384 includeLargest: false,
385 progress: progress
386 )
387 return totals
388 }
389
390 private func logStorageTotals(
391 _ totals: StorageAuditTotals,
392 prefix: String,
393 includeLargest: Bool,
394 progress: @MainActor @Sendable (String) -> Void
395 ) async {
396 await progress(
397 "\(prefix): zones=\(totals.zoneCount) " +
398 "records=\(totals.recordCount) " +
399 "assets=\(totals.assetCount) assetBytes=\(formatBytes(totals.assetBytes)) " +
400 "inline=\(formatBytes(totals.inlineBytes)) " +
401 "measured=\(formatBytes(totals.assetBytes + totals.inlineBytes))"
402 )
403 await progress("\(prefix) record types: [\(formattedCounts(totals.recordCounts))]")
404
405 for field in totals.assetFields.keys.sorted() {
406 guard let value = totals.assetFields[field] else { continue }
407 await progress(
408 "\(prefix) asset field \(field): count=\(value.count) " +
409 "bytes=\(formatBytes(value.bytes)) unavailable=\(value.unavailableCount)"
410 )
411 }
412
413 guard includeLargest else {
414 if totals.unavailableAssetCount > 0 || totals.recordErrorCount > 0 {
415 await logStorageWarnings(totals, prefix: prefix, progress: progress)
416 }
417 return
418 }
419
420 let largest = totals.assets
421 .sorted { $0.bytes > $1.bytes }
422 .prefix(20)
423 for (index, asset) in largest.enumerated() {
424 await progress(
425 "\(prefix) largest #\(index + 1): \(asset.recordType).\(asset.field) " +
426 "bytes=\(formatBytes(asset.bytes)) container=\(asset.containerLabel) " +
427 "zone=\(asset.zoneName) " +
428 "record=\(asset.recordName)"
429 )
430 }
431
432 if totals.unavailableAssetCount > 0 || totals.recordErrorCount > 0 {
433 await logStorageWarnings(totals, prefix: prefix, progress: progress)
434 }
435 }
436
437 private func logStorageWarnings(
438 _ totals: StorageAuditTotals,
439 prefix: String,
440 progress: @MainActor @Sendable (String) -> Void
441 ) async {
442 await progress(
443 "\(prefix) warnings: unavailableAssets=\(totals.unavailableAssetCount) " +
444 "recordErrors=\(totals.recordErrorCount)"
445 )
446 }
447
448 private func auditICloudDocuments(
449 progress: @MainActor @Sendable (String) -> Void
450 ) async {
451 guard let root = FileManager.default.url(
452 forUbiquityContainerIdentifier: CloudContainer.v1Identifier
453 ) else {
454 await progress("storage audit iCloud Documents: container unavailable")
455 return
456 }
457 let documents = root.appendingPathComponent("Documents", isDirectory: true)
458 let totals = Self.scanICloudDocuments(at: documents)
459
460 await progress(
461 "storage audit iCloud Documents totals: files=\(totals.fileCount) " +
462 "bytes=\(formatBytes(totals.bytes)) unavailable=\(totals.unavailableCount)"
463 )
464 for (index, file) in totals.files.sorted(by: { $0.bytes > $1.bytes }).prefix(20).enumerated() {
465 await progress(
466 "storage audit iCloud Documents largest #\(index + 1): " +
467 "bytes=\(formatBytes(file.bytes)) path=[\(singleLine(file.path))]"
468 )
469 }
470 }
471
472 private nonisolated static func scanICloudDocuments(
473 at documents: URL
474 ) -> DocumentsAuditTotals {
475 let keys: [URLResourceKey] = [.isRegularFileKey, .fileSizeKey]
476 guard let enumerator = FileManager.default.enumerator(
477 at: documents,
478 includingPropertiesForKeys: keys,
479 options: [.skipsHiddenFiles, .skipsPackageDescendants]
480 ) else {
481 return DocumentsAuditTotals(unavailableCount: 1)
482 }
483
484 var totals = DocumentsAuditTotals()
485 for case let url as URL in enumerator {
486 guard let values = try? url.resourceValues(forKeys: Set(keys)),
487 values.isRegularFile == true
488 else { continue }
489 totals.fileCount += 1
490 if let size = values.fileSize {
491 let bytes = Int64(size)
492 totals.bytes += bytes
493 totals.files.append(
494 DocumentsAuditFile(
495 path: url.path.replacingOccurrences(of: documents.path + "/", with: ""),
496 bytes: bytes
497 )
498 )
499 } else {
500 totals.unavailableCount += 1
501 }
502 }
503
504 return totals
505 }
506
507 private func auditStorageZone(
508 _ zoneID: CKRecordZone.ID,
509 containerLabel: String,
510 in database: CKDatabase
511 ) async throws -> StorageAuditTotals {
512 var totals = StorageAuditTotals()
513 var token: CKServerChangeToken?
514 var moreComing = true
515
516 while moreComing {
517 let page = try await database.recordZoneChanges(
518 inZoneWith: zoneID,
519 since: token
520 )
521 token = page.changeToken
522 moreComing = page.moreComing
523
524 for result in page.modificationResultsByID.values {
525 do {
526 let record = try result.get().record
527 measure(record, containerLabel: containerLabel, into: &totals)
528 } catch {
529 totals.recordErrorCount += 1
530 }
531 }
532 }
533 return totals
534 }
535
536 private func measure(
537 _ record: CKRecord,
538 containerLabel: String,
539 into totals: inout StorageAuditTotals
540 ) {
541 totals.recordCount += 1
542 totals.recordCounts[record.recordType, default: 0] += 1
543 if totals.title == nil,
544 record.recordType == "Game"
545 || record.recordType == Archive.recordType
546 || record.recordType == Archive.legacyRecordType {
547 totals.title = record["title"] as? String
548 }
549
550 for field in record.allKeys() {
551 guard let value = record[field] else { continue }
552 if let asset = value as? CKAsset {
553 let key = "\(record.recordType).\(field)"
554 var fieldTotal = totals.assetFields[key, default: StorageAuditFieldTotal()]
555 fieldTotal.count += 1
556 if let bytes = asset.fileURL.flatMap(assetFileSize) {
557 fieldTotal.bytes += bytes
558 totals.assets.append(
559 StorageAuditAsset(
560 containerLabel: containerLabel,
561 recordType: record.recordType,
562 field: field,
563 recordName: record.recordID.recordName,
564 zoneName: record.recordID.zoneID.zoneName,
565 bytes: bytes
566 )
567 )
568 } else {
569 fieldTotal.unavailableCount += 1
570 totals.unavailableAssetCount += 1
571 }
572 totals.assetFields[key] = fieldTotal
573 } else {
574 totals.inlineBytes += approximateInlineSize(value)
575 }
576 }
577 }
578
579 private func assetFileSize(at url: URL) -> Int64? {
580 guard let attributes = try? FileManager.default.attributesOfItem(atPath: url.path),
581 let size = attributes[.size] as? NSNumber
582 else { return nil }
583 return size.int64Value
584 }
585
586 private func approximateInlineSize(_ value: Any) -> Int64 {
587 switch value {
588 case let data as Data:
589 return Int64(data.count)
590 case let string as String:
591 return Int64(string.utf8.count)
592 case let values as [Any]:
593 return values.reduce(0) { $0 + approximateInlineSize($1) }
594 case is NSNumber, is Date:
595 return 8
596 case let reference as CKRecord.Reference:
597 return Int64(reference.recordID.recordName.utf8.count)
598 default:
599 return 0
600 }
601 }
602
603 private func formattedCounts(_ counts: [String: Int]) -> String {
604 counts.keys.sorted().compactMap { key in
605 counts[key].map { "\(key)=\($0)" }
606 }.joined(separator: ", ")
607 }
608
609 private func formatBytes(_ bytes: Int64) -> String {
610 "\(ByteCountFormatter.string(fromByteCount: bytes, countStyle: .file)) (\(bytes) B)"
611 }
612
613 private func singleLine(_ value: String) -> String {
614 String(value.replacing(/\s+/, with: " ").prefix(80))
615 }
616
617 private func probeSubscriptions(
618 database: CKDatabase,
619 label: String
620 ) async -> (String, String) {
621 do {
622 let subs = try await database.allSubscriptions()
623 if subs.isEmpty {
624 return (label, "0 subscriptions — pushes will not fire")
625 }
626 let descriptions = subs.map { sub -> String in
627 let kind: String
628 switch sub {
629 case is CKDatabaseSubscription: kind = "database"
630 case is CKQuerySubscription: kind = "query"
631 case is CKRecordZoneSubscription: kind = "zone"
632 default: kind = "other(\(type(of: sub)))"
633 }
634 let silent = sub.notificationInfo?.shouldSendContentAvailable == true ? "silent" : "alert-only"
635 return "\(kind):\(sub.subscriptionID)[\(silent)]"
636 }
637 return (label, "\(subs.count): [\(descriptions.joined(separator: ", "))]")
638 } catch {
639 return (label, describe(error))
640 }
641 }
642
643 nonisolated func describe(_ error: Error) -> String {
644 let nsError = error as NSError
645 return "ERROR domain=\(nsError.domain) code=\(nsError.code) \(nsError.localizedDescription)"
646 }
647
648 private nonisolated func describeStatus(_ status: CKAccountStatus) -> String {
649 switch status {
650 case .available: return "available"
651 case .noAccount: return "noAccount"
652 case .restricted: return "restricted"
653 case .couldNotDetermine: return "couldNotDetermine"
654 case .temporarilyUnavailable: return "temporarilyUnavailable"
655 @unknown default: return "unknown(\(status.rawValue))"
656 }
657 }
658}