diff --git a/Meetingnotes.xcodeproj/project.pbxproj b/Meetingnotes.xcodeproj/project.pbxproj index 599e793..09d865e 100644 --- a/Meetingnotes.xcodeproj/project.pbxproj +++ b/Meetingnotes.xcodeproj/project.pbxproj @@ -276,7 +276,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 42; + CURRENT_PROJECT_VERSION = 43; DEVELOPMENT_ASSET_PATHS = "\"meetingnotes/Preview Content\""; DEVELOPMENT_TEAM = G9LVHZAJNX; ENABLE_HARDENED_RUNTIME = YES; @@ -290,7 +290,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 15.0; - MARKETING_VERSION = 1.1.30; + MARKETING_VERSION = 1.1.31; ONLY_ACTIVE_ARCH = NO; OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI"; PRODUCT_BUNDLE_IDENTIFIER = net.jamesbone.meetingnotes; @@ -312,7 +312,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 42; + CURRENT_PROJECT_VERSION = 43; DEVELOPMENT_ASSET_PATHS = "\"meetingnotes/Preview Content\""; DEVELOPMENT_TEAM = G9LVHZAJNX; ENABLE_HARDENED_RUNTIME = YES; @@ -326,7 +326,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 15.0; - MARKETING_VERSION = 1.1.30; + MARKETING_VERSION = 1.1.31; ONLY_ACTIVE_ARCH = YES; OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI"; PRODUCT_BUNDLE_IDENTIFIER = net.jamesbone.meetingnotes; diff --git a/meetingnotes/Managers/AudioManager.swift b/meetingnotes/Managers/AudioManager.swift index c941fc3..07f5609 100644 --- a/meetingnotes/Managers/AudioManager.swift +++ b/meetingnotes/Managers/AudioManager.swift @@ -27,6 +27,7 @@ private struct SystemCaptureDiagnostics { var targetFormat = "unavailable" var selectedInputSampleRate: Double? var targetSampleRate: Double? + var inputRateInference = "unavailable" var firstBufferLayout: String? var callbackCount: UInt64 = 0 var inputFrameCount: UInt64 = 0 @@ -443,18 +444,67 @@ final class AudioManager: NSObject, ObservableObject { systemDiagnostics.targetSampleRate = targetFormat.sampleRate var inputFormat: AVAudioFormat? var converter: AVAudioConverter? + var pendingBuffer: AVAudioPCMBuffer? + var pendingInputTime: AudioTimeStamp? + var pendingNow: AudioTimeStamp? try tap.run(on: tapQueue) { [weak self] inNow, inputData, inInputTime, _, _ in guard let self else { return } if inputFormat == nil { + guard let callbackFormat = self.inputFormat( + for: inputData, + sampleRate: advertisedInputFormat.sampleRate + ), + let currentBuffer = self.copyAudioBuffer(from: inputData, format: callbackFormat) else { + self.systemDiagnostics.discardedCallbackCount += 1 + return + } + + guard let previousBuffer = pendingBuffer, + let previousInputTime = pendingInputTime else { + pendingBuffer = currentBuffer + pendingInputTime = inInputTime.pointee + pendingNow = inNow.pointee + return + } + + let selectedSampleRate = self.effectiveInputSampleRate( + advertisedSampleRate: advertisedInputFormat.sampleRate, + previousFrameLength: previousBuffer.frameLength, + previousTimestamp: previousInputTime, + currentTimestamp: inInputTime.pointee + ) inputFormat = self.inputFormat( for: inputData, - advertisedFormat: advertisedInputFormat + sampleRate: selectedSampleRate ) if let inputFormat { self.systemDiagnostics.selectedInputFormat = self.audioFormatSummary(inputFormat) self.systemDiagnostics.selectedInputSampleRate = inputFormat.sampleRate converter = AVAudioConverter(from: inputFormat, to: targetFormat) } + + guard let inputFormat, let converter else { return } + self.processAudioBuffer( + { self.copyAudioBuffer(from: previousBuffer.audioBufferList, format: inputFormat) }, + converter: converter, + targetFormat: targetFormat, + source: .system, + callbackTimestamp: previousInputTime, + ioTimestamp: pendingNow + ) + pendingBuffer = nil + pendingInputTime = nil + pendingNow = nil + + self.processAudioBuffer( + { self.copyAudioBuffer(from: currentBuffer.audioBufferList, format: inputFormat) }, + converter: converter, + targetFormat: targetFormat, + source: .system, + callbackTimestamp: inInputTime.pointee, + ioTimestamp: inNow.pointee + ) + return } guard let inputFormat, let converter else { return } // The tap queue is serial. Reusing the converter preserves its @@ -475,7 +525,7 @@ final class AudioManager: NSObject, ObservableObject { private func inputFormat( for inputData: UnsafePointer, - advertisedFormat: AVAudioFormat + sampleRate: Double ) -> AVAudioFormat? { let buffers = UnsafeMutableAudioBufferListPointer( UnsafeMutablePointer(mutating: inputData) @@ -492,12 +542,58 @@ final class AudioManager: NSObject, ObservableObject { let isInterleaved = buffers.count == 1 && channelCount > 1 return AVAudioFormat( commonFormat: .pcmFormatFloat32, - sampleRate: advertisedFormat.sampleRate, + sampleRate: sampleRate, channels: AVAudioChannelCount(channelCount), interleaved: isInterleaved ) } + private func effectiveInputSampleRate( + advertisedSampleRate: Double, + previousFrameLength: AVAudioFrameCount, + previousTimestamp: AudioTimeStamp, + currentTimestamp: AudioTimeStamp + ) -> Double { + let timestampsAreValid = previousTimestamp.mFlags.contains(.sampleTimeValid) + && currentTimestamp.mFlags.contains(.sampleTimeValid) + let sampleTimeDelta = currentTimestamp.mSampleTime - previousTimestamp.mSampleTime + guard timestampsAreValid, + advertisedSampleRate > 0, + previousFrameLength > 0, + sampleTimeDelta.isFinite, + sampleTimeDelta > 0 else { + systemDiagnostics.inputRateInference = "advertised (sample timestamps unavailable)" + return advertisedSampleRate + } + + let inferredSampleRate = advertisedSampleRate * Double(previousFrameLength) / sampleTimeDelta + let plausibleRange = (advertisedSampleRate * 0.25)...(advertisedSampleRate * 1.25) + guard inferredSampleRate.isFinite, plausibleRange.contains(inferredSampleRate) else { + systemDiagnostics.inputRateInference = String( + format: "advertised (invalid inference %.3f from %u frames / %.3f sample-time units)", + inferredSampleRate, + previousFrameLength, + sampleTimeDelta + ) + return advertisedSampleRate + } + + let commonSampleRates: [Double] = [8_000, 11_025, 12_000, 16_000, 22_050, 24_000, 32_000, 44_100, 48_000, 88_200, 96_000] + let selectedSampleRate = commonSampleRates + .min(by: { abs($0 - inferredSampleRate) < abs($1 - inferredSampleRate) }) + .flatMap { abs($0 - inferredSampleRate) / $0 <= 0.01 ? $0 : nil } + ?? inferredSampleRate + systemDiagnostics.inputRateInference = String( + format: "advertised=%.3f,inferred=%.3f,selected=%.3f,previousFrames=%u,sampleTimeDelta=%.3f", + advertisedSampleRate, + inferredSampleRate, + selectedSampleRate, + previousFrameLength, + sampleTimeDelta + ) + return selectedSampleRate + } + private func copyAudioBuffer( from inputData: UnsafePointer, format: AVAudioFormat @@ -863,6 +959,7 @@ final class AudioManager: NSObject, ObservableObject { "tapAdvertisedFormat=\(systemDiagnostics.tapAdvertisedFormat)", "aggregateInputFormat=\(systemDiagnostics.aggregateInputFormat)", "selectedInputFormat=\(systemDiagnostics.selectedInputFormat)", + "inputRateInference=\(systemDiagnostics.inputRateInference)", "targetFormat=\(systemDiagnostics.targetFormat)", "firstBufferLayout=\(systemDiagnostics.firstBufferLayout ?? "unavailable")", "callbackCount=\(systemDiagnostics.callbackCount)", diff --git a/meetingnotes/Services/CoderAPIClient.swift b/meetingnotes/Services/CoderAPIClient.swift index 975c817..20af5f7 100644 --- a/meetingnotes/Services/CoderAPIClient.swift +++ b/meetingnotes/Services/CoderAPIClient.swift @@ -30,6 +30,7 @@ enum CoderAPIError: LocalizedError { case missingModel(String) case invalidResponse case serviceError(Int, String) + case audioPreparationFailed(String, String) var errorDescription: String? { switch self { @@ -43,6 +44,8 @@ enum CoderAPIError: LocalizedError { return "Coder returned an invalid response." case .serviceError(let status, let message): return "Coder request failed (\(status)): \(message)" + case .audioPreparationFailed(let filename, let message): + return "Could not prepare \(filename) for transcription: \(message)" } } } @@ -179,7 +182,12 @@ final class CoderAPIClient { let selectedModel = model.trimmingCharacters(in: .whitespacesAndNewlines) guard !selectedModel.isEmpty else { throw CoderAPIError.missingModel("transcription") } let apiKey = try requiredAPIKey(KeychainHelper.shared.getCoderAPIKey() ?? "") - let chunks = try makeAudioChunks(from: fileURL, preserveSpeakerIdentity: diarization) + let chunks: [AudioChunk] + do { + chunks = try makeAudioChunks(from: fileURL, preserveSpeakerIdentity: diarization) + } catch { + throw CoderAPIError.audioPreparationFailed(fileURL.lastPathComponent, error.localizedDescription) + } defer { for chunk in chunks where chunk.isTemporary { try? FileManager.default.removeItem(at: chunk.url) @@ -260,13 +268,38 @@ final class CoderAPIClient { let size = attributes[.size] as? NSNumber { request.setValue(size.stringValue, forHTTPHeaderField: "Content-Length") } - let (data, response) = try await transcriptionSession.upload(for: request, fromFile: bodyURL) + let (data, response) = try await uploadTranscription(request: request, bodyURL: bodyURL) try validate(response: response, data: data) let decoded = try JSONDecoder().decode(TranscriptionResponse.self, from: data) let segments = decoded.segments ?? segments(from: decoded.words ?? []) return Transcription(text: decoded.text, segments: segments) } + private func uploadTranscription(request: URLRequest, bodyURL: URL) async throws -> (Data, URLResponse) { + let retryDelays: [UInt64] = [3, 10] + for attempt in 0...retryDelays.count { + do { + return try await transcriptionSession.upload(for: request, fromFile: bodyURL) + } catch { + guard attempt < retryDelays.count, isRetryableTranscriptionError(error) else { + throw error + } + try await Task.sleep(nanoseconds: retryDelays[attempt] * 1_000_000_000) + } + } + throw CoderAPIError.invalidResponse + } + + private func isRetryableTranscriptionError(_ error: Error) -> Bool { + guard let urlError = error as? URLError else { return false } + switch urlError.code { + case .networkConnectionLost, .cannotConnectToHost, .timedOut: + return true + default: + return false + } + } + private func makeAudioChunks(from fileURL: URL, preserveSpeakerIdentity: Bool) throws -> [AudioChunk] { let input = try AVAudioFile(forReading: fileURL) let format = input.processingFormat