From 58b673ffcc52c09aa3aa9343372e285b142d5aea Mon Sep 17 00:00:00 2001 From: SuperDooper86 Date: Tue, 25 Aug 2026 16:00:36 +0200 Subject: [PATCH] fix: preserve system audio timing --- Meetingnotes.xcodeproj/project.pbxproj | 8 +- meetingnotes/Managers/AudioManager.swift | 136 ++++++++++++++++++++++- 2 files changed, 138 insertions(+), 6 deletions(-) diff --git a/Meetingnotes.xcodeproj/project.pbxproj b/Meetingnotes.xcodeproj/project.pbxproj index 6f8af84..d207f07 100644 --- a/Meetingnotes.xcodeproj/project.pbxproj +++ b/Meetingnotes.xcodeproj/project.pbxproj @@ -276,7 +276,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 33; + CURRENT_PROJECT_VERSION = 34; DEVELOPMENT_ASSET_PATHS = "\"meetingnotes/Preview Content\""; DEVELOPMENT_TEAM = G9LVHZAJNX; ENABLE_HARDENED_RUNTIME = YES; @@ -290,7 +290,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 15.0; - MARKETING_VERSION = 1.1.21; + MARKETING_VERSION = 1.1.22; ONLY_ACTIVE_ARCH = NO; OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI"; PRODUCT_BUNDLE_IDENTIFIER = net.jamesbone.meetingnotes; @@ -312,7 +312,7 @@ CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; - CURRENT_PROJECT_VERSION = 33; + CURRENT_PROJECT_VERSION = 34; DEVELOPMENT_ASSET_PATHS = "\"meetingnotes/Preview Content\""; DEVELOPMENT_TEAM = G9LVHZAJNX; ENABLE_HARDENED_RUNTIME = YES; @@ -326,7 +326,7 @@ "@executable_path/../Frameworks", ); MACOSX_DEPLOYMENT_TARGET = 15.0; - MARKETING_VERSION = 1.1.21; + MARKETING_VERSION = 1.1.22; ONLY_ACTIVE_ARCH = YES; OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI"; PRODUCT_BUNDLE_IDENTIFIER = net.jamesbone.meetingnotes; diff --git a/meetingnotes/Managers/AudioManager.swift b/meetingnotes/Managers/AudioManager.swift index 264a595..a5f99d4 100644 --- a/meetingnotes/Managers/AudioManager.swift +++ b/meetingnotes/Managers/AudioManager.swift @@ -86,6 +86,18 @@ final class AudioManager: NSObject, ObservableObject { isProcessing = false } + repairHalfDurationSystemWAVIfNeeded(in: files) + if let mismatch = captureDurationMismatch(in: files) { + let completedFiles = files.compactMap { $0 } + let audioFolder = preserveAudioFiles(completedFiles, meetingID: completedMeetingID) + lastRecoveryAudioFolderName = audioFolder?.lastPathComponent + let recoveryMessage = audioFolder == nil + ? " The audio remains in the app's temporary folder." + : " Audio was kept so it can be recovered." + errorMessage = "System audio timing was invalid (\(mismatch)). Transcription was stopped to avoid an out-of-order result." + recoveryMessage + return transcriptChunks.filter(\.isFinal) + } + let model = UserDefaultsManager.shared.transcriptionModel async let micResult = transcribe(files[0], model: model, diarization: false) async let systemResult = transcribe(files[1], model: model, diarization: true) @@ -356,13 +368,25 @@ final class AudioManager: NSObject, ObservableObject { private func startTapIO(_ tap: ProcessTap) throws { guard var description = tap.tapStreamDescription, - let inputFormat = AVAudioFormat(streamDescription: &description), + let advertisedInputFormat = AVAudioFormat(streamDescription: &description), let targetFormat = systemAudioFile?.processingFormat, - let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else { + advertisedInputFormat.sampleRate > 0 else { throw NSError(domain: "AudioManager", code: -1, userInfo: [NSLocalizedDescriptionKey: "Unsupported system audio format"]) } + var inputFormat: AVAudioFormat? + var converter: AVAudioConverter? try tap.run(on: tapQueue) { [weak self] _, inputData, _, _, _ in guard let self else { return } + if inputFormat == nil { + inputFormat = self.inputFormat( + for: inputData, + advertisedFormat: advertisedInputFormat + ) + if let inputFormat { + converter = AVAudioConverter(from: inputFormat, to: targetFormat) + } + } + guard let inputFormat, let converter else { return } // The tap queue is serial. Reusing the converter preserves its // resampler state instead of discarding audio at every callback. self.processAudioBuffer( @@ -377,6 +401,30 @@ final class AudioManager: NSObject, ObservableObject { } } + private func inputFormat( + for inputData: UnsafePointer, + advertisedFormat: AVAudioFormat + ) -> AVAudioFormat? { + let buffers = UnsafeMutableAudioBufferListPointer( + UnsafeMutablePointer(mutating: inputData) + ) + let channelCount = buffers.reduce(UInt32(0)) { $0 + $1.mNumberChannels } + guard channelCount > 0 else { return nil } + + // HAL tap metadata can advertise interleaved stereo while the callback + // supplies one mono buffer per channel (or the reverse). Constructing a + // PCM buffer with that mismatched layout halves its frame count and + // produces 2x-speed system audio. The callback's AudioBufferList is the + // authoritative layout for the memory we are copying. + let isInterleaved = buffers.count == 1 && channelCount > 1 + return AVAudioFormat( + commonFormat: advertisedFormat.commonFormat, + sampleRate: advertisedFormat.sampleRate, + channels: AVAudioChannelCount(channelCount), + interleaved: isInterleaved + ) + } + private func copyAudioBuffer( from inputData: UnsafePointer, format: AVAudioFormat @@ -523,6 +571,90 @@ final class AudioManager: NSObject, ObservableObject { private func preserveAudioFiles(_ urls: [URL], meetingID: UUID) -> URL? { LocalStorageManager.shared.preserveAudioFiles(urls, for: meetingID) } + + private func captureDurationMismatch(in files: [URL?]) -> String? { + guard files.count >= 2, + let micDuration = audioDuration(at: files[0]), + let systemDuration = audioDuration(at: files[1]), + micDuration >= 60 else { return nil } + let ratio = systemDuration / micDuration + guard (0.45...0.55).contains(ratio) || (1.8...2.2).contains(ratio) else { return nil } + return String(format: "mic %.1fs, system %.1fs", micDuration, systemDuration) + } + + private func repairHalfDurationSystemWAVIfNeeded(in files: [URL?]) { + guard files.count >= 2, + let micDuration = audioDuration(at: files[0]), + let systemURL = files[1], + let systemDuration = audioDuration(at: systemURL), + micDuration >= 60, + systemURL.pathExtension.caseInsensitiveCompare("wav") == .orderedSame, + (0.48...0.52).contains(systemDuration / micDuration) else { return } + try? halveWAVSampleRate(at: systemURL) + } + + private func halveWAVSampleRate(at url: URL) throws { + let handle = try FileHandle(forUpdating: url) + defer { try? handle.close() } + + try handle.seek(toOffset: 0) + guard let riffHeader = try handle.read(upToCount: 12), + riffHeader.count == 12, + String(data: riffHeader[0..<4], encoding: .ascii) == "RIFF", + String(data: riffHeader[8..<12], encoding: .ascii) == "WAVE" else { + throw NSError(domain: "AudioManager", code: -2, userInfo: [NSLocalizedDescriptionKey: "Invalid WAV header"]) + } + + var offset: UInt64 = 12 + while true { + try handle.seek(toOffset: offset) + guard let chunkHeader = try handle.read(upToCount: 8), chunkHeader.count == 8 else { break } + let chunkID = String(data: chunkHeader[0..<4], encoding: .ascii) + let chunkSize = UInt32(chunkHeader[4]) + | (UInt32(chunkHeader[5]) << 8) + | (UInt32(chunkHeader[6]) << 16) + | (UInt32(chunkHeader[7]) << 24) + let chunkDataOffset = offset + 8 + + if chunkID == "fmt ", chunkSize >= 16 { + try handle.seek(toOffset: chunkDataOffset) + guard let format = try handle.read(upToCount: 16), format.count == 16 else { break } + let audioFormat = UInt16(format[0]) | (UInt16(format[1]) << 8) + let blockAlign = UInt16(format[12]) | (UInt16(format[13]) << 8) + let sampleRate = UInt32(format[4]) + | (UInt32(format[5]) << 8) + | (UInt32(format[6]) << 16) + | (UInt32(format[7]) << 24) + guard audioFormat == 1, sampleRate >= 16_000, sampleRate.isMultiple(of: 2) else { + throw NSError(domain: "AudioManager", code: -3, userInfo: [NSLocalizedDescriptionKey: "Unsupported WAV format"]) + } + + let correctedSampleRate = sampleRate / 2 + let correctedByteRate = correctedSampleRate * UInt32(blockAlign) + try handle.seek(toOffset: chunkDataOffset + 4) + try handle.write(contentsOf: littleEndianData(correctedSampleRate)) + try handle.write(contentsOf: littleEndianData(correctedByteRate)) + try handle.synchronize() + return + } + + offset = chunkDataOffset + UInt64(chunkSize) + UInt64(chunkSize % 2) + } + + throw NSError(domain: "AudioManager", code: -4, userInfo: [NSLocalizedDescriptionKey: "WAV format chunk was not found"]) + } + + private func littleEndianData(_ value: UInt32) -> Data { + var littleEndianValue = value.littleEndian + return withUnsafeBytes(of: &littleEndianValue) { Data($0) } + } + + private func audioDuration(at url: URL?) -> TimeInterval? { + guard let url, + let file = try? AVAudioFile(forReading: url), + file.processingFormat.sampleRate > 0 else { return nil } + return Double(file.length) / file.processingFormat.sampleRate + } private func resetAudioLevels() { micAudioLevel = 0