fix: infer actual system audio clock rate

Meetingnotes.xcodeproj/project.pbxproj

@@ -276,7 +276,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
 				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 41;
+				CURRENT_PROJECT_VERSION = 43;
 				DEVELOPMENT_ASSET_PATHS = "\"meetingnotes/Preview Content\"";
 				DEVELOPMENT_TEAM = G9LVHZAJNX;
 				ENABLE_HARDENED_RUNTIME = YES;
@@ -290,7 +290,7 @@
 					"@executable_path/../Frameworks",
 				);
 				MACOSX_DEPLOYMENT_TARGET = 15.0;
-				MARKETING_VERSION = 1.1.29;
+				MARKETING_VERSION = 1.1.31;
 				ONLY_ACTIVE_ARCH = NO;
 				OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI";
 				PRODUCT_BUNDLE_IDENTIFIER = net.jamesbone.meetingnotes;
@@ -312,7 +312,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
 				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 41;
+				CURRENT_PROJECT_VERSION = 43;
 				DEVELOPMENT_ASSET_PATHS = "\"meetingnotes/Preview Content\"";
 				DEVELOPMENT_TEAM = G9LVHZAJNX;
 				ENABLE_HARDENED_RUNTIME = YES;
@@ -326,7 +326,7 @@
 					"@executable_path/../Frameworks",
 				);
 				MACOSX_DEPLOYMENT_TARGET = 15.0;
-				MARKETING_VERSION = 1.1.29;
+				MARKETING_VERSION = 1.1.31;
 				ONLY_ACTIVE_ARCH = YES;
 				OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI";
 				PRODUCT_BUNDLE_IDENTIFIER = net.jamesbone.meetingnotes;

meetingnotes/Managers/AudioManager.swift

@@ -27,6 +27,7 @@ private struct SystemCaptureDiagnostics {
     var targetFormat = "unavailable"
     var selectedInputSampleRate: Double?
     var targetSampleRate: Double?
+    var inputRateInference = "unavailable"
     var firstBufferLayout: String?
     var callbackCount: UInt64 = 0
     var inputFrameCount: UInt64 = 0
@@ -443,18 +444,67 @@ final class AudioManager: NSObject, ObservableObject {
         systemDiagnostics.targetSampleRate = targetFormat.sampleRate
         var inputFormat: AVAudioFormat?
         var converter: AVAudioConverter?
+        var pendingBuffer: AVAudioPCMBuffer?
+        var pendingInputTime: AudioTimeStamp?
+        var pendingNow: AudioTimeStamp?
         try tap.run(on: tapQueue) { [weak self] inNow, inputData, inInputTime, _, _ in
             guard let self else { return }
             if inputFormat == nil {
+                guard let callbackFormat = self.inputFormat(
+                    for: inputData,
+                    sampleRate: advertisedInputFormat.sampleRate
+                ),
+                let currentBuffer = self.copyAudioBuffer(from: inputData, format: callbackFormat) else {
+                    self.systemDiagnostics.discardedCallbackCount += 1
+                    return
+                }
+
+                guard let previousBuffer = pendingBuffer,
+                      let previousInputTime = pendingInputTime else {
+                    pendingBuffer = currentBuffer
+                    pendingInputTime = inInputTime.pointee
+                    pendingNow = inNow.pointee
+                    return
+                }
+
+                let selectedSampleRate = self.effectiveInputSampleRate(
+                    advertisedSampleRate: advertisedInputFormat.sampleRate,
+                    previousFrameLength: previousBuffer.frameLength,
+                    previousTimestamp: previousInputTime,
+                    currentTimestamp: inInputTime.pointee
+                )
                 inputFormat = self.inputFormat(
                     for: inputData,
-                    advertisedFormat: advertisedInputFormat
+                    sampleRate: selectedSampleRate
                 )
                 if let inputFormat {
                     self.systemDiagnostics.selectedInputFormat = self.audioFormatSummary(inputFormat)
                     self.systemDiagnostics.selectedInputSampleRate = inputFormat.sampleRate
                     converter = AVAudioConverter(from: inputFormat, to: targetFormat)
                 }
+
+                guard let inputFormat, let converter else { return }
+                self.processAudioBuffer(
+                    { self.copyAudioBuffer(from: previousBuffer.audioBufferList, format: inputFormat) },
+                    converter: converter,
+                    targetFormat: targetFormat,
+                    source: .system,
+                    callbackTimestamp: previousInputTime,
+                    ioTimestamp: pendingNow
+                )
+                pendingBuffer = nil
+                pendingInputTime = nil
+                pendingNow = nil
+
+                self.processAudioBuffer(
+                    { self.copyAudioBuffer(from: currentBuffer.audioBufferList, format: inputFormat) },
+                    converter: converter,
+                    targetFormat: targetFormat,
+                    source: .system,
+                    callbackTimestamp: inInputTime.pointee,
+                    ioTimestamp: inNow.pointee
+                )
+                return
             }
             guard let inputFormat, let converter else { return }
             // The tap queue is serial. Reusing the converter preserves its
@@ -475,7 +525,7 @@ final class AudioManager: NSObject, ObservableObject {
 
     private func inputFormat(
         for inputData: UnsafePointer<AudioBufferList>,
-        advertisedFormat: AVAudioFormat
+        sampleRate: Double
     ) -> AVAudioFormat? {
         let buffers = UnsafeMutableAudioBufferListPointer(
             UnsafeMutablePointer(mutating: inputData)
@@ -492,12 +542,58 @@ final class AudioManager: NSObject, ObservableObject {
         let isInterleaved = buffers.count == 1 && channelCount > 1
         return AVAudioFormat(
             commonFormat: .pcmFormatFloat32,
-            sampleRate: advertisedFormat.sampleRate,
+            sampleRate: sampleRate,
             channels: AVAudioChannelCount(channelCount),
             interleaved: isInterleaved
         )
     }
 
+    private func effectiveInputSampleRate(
+        advertisedSampleRate: Double,
+        previousFrameLength: AVAudioFrameCount,
+        previousTimestamp: AudioTimeStamp,
+        currentTimestamp: AudioTimeStamp
+    ) -> Double {
+        let timestampsAreValid = previousTimestamp.mFlags.contains(.sampleTimeValid)
+            && currentTimestamp.mFlags.contains(.sampleTimeValid)
+        let sampleTimeDelta = currentTimestamp.mSampleTime - previousTimestamp.mSampleTime
+        guard timestampsAreValid,
+              advertisedSampleRate > 0,
+              previousFrameLength > 0,
+              sampleTimeDelta.isFinite,
+              sampleTimeDelta > 0 else {
+            systemDiagnostics.inputRateInference = "advertised (sample timestamps unavailable)"
+            return advertisedSampleRate
+        }
+
+        let inferredSampleRate = advertisedSampleRate * Double(previousFrameLength) / sampleTimeDelta
+        let plausibleRange = (advertisedSampleRate * 0.25)...(advertisedSampleRate * 1.25)
+        guard inferredSampleRate.isFinite, plausibleRange.contains(inferredSampleRate) else {
+            systemDiagnostics.inputRateInference = String(
+                format: "advertised (invalid inference %.3f from %u frames / %.3f sample-time units)",
+                inferredSampleRate,
+                previousFrameLength,
+                sampleTimeDelta
+            )
+            return advertisedSampleRate
+        }
+
+        let commonSampleRates: [Double] = [8_000, 11_025, 12_000, 16_000, 22_050, 24_000, 32_000, 44_100, 48_000, 88_200, 96_000]
+        let selectedSampleRate = commonSampleRates
+            .min(by: { abs($0 - inferredSampleRate) < abs($1 - inferredSampleRate) })
+            .flatMap { abs($0 - inferredSampleRate) / $0 <= 0.01 ? $0 : nil }
+            ?? inferredSampleRate
+        systemDiagnostics.inputRateInference = String(
+            format: "advertised=%.3f,inferred=%.3f,selected=%.3f,previousFrames=%u,sampleTimeDelta=%.3f",
+            advertisedSampleRate,
+            inferredSampleRate,
+            selectedSampleRate,
+            previousFrameLength,
+            sampleTimeDelta
+        )
+        return selectedSampleRate
+    }
+
     private func copyAudioBuffer(
         from inputData: UnsafePointer<AudioBufferList>,
         format: AVAudioFormat
@@ -863,6 +959,7 @@ final class AudioManager: NSObject, ObservableObject {
             "tapAdvertisedFormat=\(systemDiagnostics.tapAdvertisedFormat)",
             "aggregateInputFormat=\(systemDiagnostics.aggregateInputFormat)",
             "selectedInputFormat=\(systemDiagnostics.selectedInputFormat)",
+            "inputRateInference=\(systemDiagnostics.inputRateInference)",
             "targetFormat=\(systemDiagnostics.targetFormat)",
             "firstBufferLayout=\(systemDiagnostics.firstBufferLayout ?? "unavailable")",
             "callbackCount=\(systemDiagnostics.callbackCount)",

meetingnotes/Managers/LocalStorageManager.swift

@@ -170,11 +170,13 @@ class LocalStorageManager {
                 }
             }
 
-            for audioFile in recoveryAudioFiles(in: continuationFolder) {
+            let recoveryFiles = recoveryAudioFiles(in: continuationFolder).map(\.url)
-                let destination = previousFolder.appendingPathComponent(audioFile.url.lastPathComponent)
+                + recoveryDiagnosticFiles(in: continuationFolder)
+            for recoveryFile in recoveryFiles {
+                let destination = previousFolder.appendingPathComponent(recoveryFile.lastPathComponent)
                 if FileManager.default.fileExists(atPath: destination.path) {
                     guard FileManager.default.contentsEqual(
-                        atPath: audioFile.url.path,
+                        atPath: recoveryFile.path,
                         andPath: destination.path
                     ) else {
                         rollbackMergedAudio(copiedAudioURLs, removeFolder: createdPreviousFolder ? previousFolder : nil)
@@ -184,7 +186,7 @@ class LocalStorageManager {
                 }
 
                 do {
-                    try FileManager.default.copyItem(at: audioFile.url, to: destination)
+                    try FileManager.default.copyItem(at: recoveryFile, to: destination)
                     copiedAudioURLs.append(destination)
                 } catch {
                     rollbackMergedAudio(copiedAudioURLs, removeFolder: createdPreviousFolder ? previousFolder : nil)
@@ -206,6 +208,7 @@ class LocalStorageManager {
         merged.userNotes = mergedText(previous.userNotes, continuation.userNotes)
         merged.generatedNotes = mergedText(previous.generatedNotes, continuation.generatedNotes)
         merged.templateId = previous.templateId ?? continuation.templateId
+        merged.transcriptionError = previous.transcriptionError ?? continuation.transcriptionError
         if !recoveryAudioFiles(in: previousFolder).isEmpty {
             merged.recoveryAudioFolderName = previous.id.uuidString
         }
@@ -347,6 +350,23 @@ class LocalStorageManager {
         }
     }
 
+    private func recoveryDiagnosticFiles(in folder: URL) -> [URL] {
+        guard let files = try? FileManager.default.contentsOfDirectory(
+            at: folder,
+            includingPropertiesForKeys: [.isRegularFileKey],
+            options: [.skipsHiddenFiles]
+        ) else {
+            return []
+        }
+
+        return files.filter { url in
+            let values = try? url.resourceValues(forKeys: [.isRegularFileKey])
+            return values?.isRegularFile == true
+                && url.pathExtension.caseInsensitiveCompare("txt") == .orderedSame
+                && url.lastPathComponent.lowercased().hasPrefix("audio-diagnostics-")
+        }
+    }
+
     func findRecoveryAudioFolder(for meeting: Meeting) -> URL? {
         let canonicalName = meeting.id.uuidString
         guard meeting.recoveryAudioFolderName == nil

meetingnotes/Services/CoderAPIClient.swift

@@ -30,6 +30,7 @@ enum CoderAPIError: LocalizedError {
     case missingModel(String)
     case invalidResponse
     case serviceError(Int, String)
+    case audioPreparationFailed(String, String)
 
     var errorDescription: String? {
         switch self {
@@ -43,6 +44,8 @@ enum CoderAPIError: LocalizedError {
             return "Coder returned an invalid response."
         case .serviceError(let status, let message):
             return "Coder request failed (\(status)): \(message)"
+        case .audioPreparationFailed(let filename, let message):
+            return "Could not prepare \(filename) for transcription: \(message)"
         }
     }
 }
@@ -179,7 +182,12 @@ final class CoderAPIClient {
         let selectedModel = model.trimmingCharacters(in: .whitespacesAndNewlines)
         guard !selectedModel.isEmpty else { throw CoderAPIError.missingModel("transcription") }
         let apiKey = try requiredAPIKey(KeychainHelper.shared.getCoderAPIKey() ?? "")
-        let chunks = try makeAudioChunks(from: fileURL, preserveSpeakerIdentity: diarization)
+        let chunks: [AudioChunk]
+        do {
+            chunks = try makeAudioChunks(from: fileURL, preserveSpeakerIdentity: diarization)
+        } catch {
+            throw CoderAPIError.audioPreparationFailed(fileURL.lastPathComponent, error.localizedDescription)
+        }
         defer {
             for chunk in chunks where chunk.isTemporary {
                 try? FileManager.default.removeItem(at: chunk.url)
@@ -260,13 +268,38 @@ final class CoderAPIClient {
            let size = attributes[.size] as? NSNumber {
             request.setValue(size.stringValue, forHTTPHeaderField: "Content-Length")
         }
-        let (data, response) = try await transcriptionSession.upload(for: request, fromFile: bodyURL)
+        let (data, response) = try await uploadTranscription(request: request, bodyURL: bodyURL)
         try validate(response: response, data: data)
         let decoded = try JSONDecoder().decode(TranscriptionResponse.self, from: data)
         let segments = decoded.segments ?? segments(from: decoded.words ?? [])
         return Transcription(text: decoded.text, segments: segments)
     }
 
+    private func uploadTranscription(request: URLRequest, bodyURL: URL) async throws -> (Data, URLResponse) {
+        let retryDelays: [UInt64] = [3, 10]
+        for attempt in 0...retryDelays.count {
+            do {
+                return try await transcriptionSession.upload(for: request, fromFile: bodyURL)
+            } catch {
+                guard attempt < retryDelays.count, isRetryableTranscriptionError(error) else {
+                    throw error
+                }
+                try await Task.sleep(nanoseconds: retryDelays[attempt] * 1_000_000_000)
+            }
+        }
+        throw CoderAPIError.invalidResponse
+    }
+
+    private func isRetryableTranscriptionError(_ error: Error) -> Bool {
+        guard let urlError = error as? URLError else { return false }
+        switch urlError.code {
+        case .networkConnectionLost, .cannotConnectToHost, .timedOut:
+            return true
+        default:
+            return false
+        }
+    }
+
     private func makeAudioChunks(from fileURL: URL, preserveSpeakerIdentity: Bool) throws -> [AudioChunk] {
         let input = try AVAudioFile(forReading: fileURL)
         let format = input.processingFormat