coder/meetingnotes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

fix: preserve transcription failures and audio diagnostics

Browse Source
This commit is contained in:
coder
2026-08-26 16:43:43 +02:00
parent 66447b2510
commit 299986ab00
7 changed files with 330 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
meetingnotes/Managers/AudioManager.swift
+196 -12
View File
@@ -20,6 +20,26 @@ private enum RecoveryTranscriptionError: LocalizedError {
}
}
private struct SystemCaptureDiagnostics {
var tapAdvertisedFormat = "unavailable"
var aggregateInputFormat = "unavailable"
var selectedInputFormat = "unavailable"
var targetFormat = "unavailable"
var selectedInputSampleRate: Double?
var targetSampleRate: Double?
var firstBufferLayout: String?
var callbackCount: UInt64 = 0
var inputFrameCount: UInt64 = 0
var outputFrameCount: UInt64 = 0
var discardedCallbackCount: UInt64 = 0
var firstSampleTime: Double?
var lastSampleTime: Double?
var firstHostTime: UInt64?
var lastHostTime: UInt64?
var firstCallbackAt: Date?
var lastCallbackAt: Date?
}
/// Captures microphone and system audio locally, then sends completed files to Coder.
@MainActor
final class AudioManager: NSObject, ObservableObject {
@@ -50,6 +70,7 @@ final class AudioManager: NSObject, ObservableObject {
private var micAudioURL: URL?
private var systemAudioURL: URL?
private var recordingStartedAt = Date()
private var systemDiagnostics = SystemCaptureDiagnostics()
private override init() {
super.init()
@@ -67,6 +88,7 @@ final class AudioManager: NSObject, ObservableObject {
sessionID = UUID()
self.meetingID = meetingID
recordingStartedAt = Date()
systemDiagnostics = SystemCaptureDiagnostics()
do {
try prepareAudioFiles()
startMicrophoneTap()
@@ -79,6 +101,7 @@ final class AudioManager: NSObject, ObservableObject {
func stopRecordingAndTranscribe() async -> [TranscriptChunk] {
let completedMeetingID = meetingID
let completedSessionID = sessionID
let captureStartedAt = recordingStartedAt
let files = stopCaptureAndCloseFiles()
isProcessing = true
@@ -86,11 +109,20 @@ final class AudioManager: NSObject, ObservableObject {
isProcessing = false
}
repairHalfDurationSystemWAVIfNeeded(in: files)
let preRepairFileSummaries = files.map(audioFileSummary)
let repairApplied = repairHalfDurationSystemWAVIfNeeded(in: files)
let completedFiles = files.compactMap { $0 }
let audioFolder = preserveAudioFiles(completedFiles, meetingID: completedMeetingID)
let transcriptionFiles = preservedAudioFiles(files, in: audioFolder)
lastRecoveryAudioFolderName = audioFolder?.lastPathComponent
writeCaptureDiagnostics(
sessionID: completedSessionID,
captureStartedAt: captureStartedAt,
preRepairFileSummaries: preRepairFileSummaries,
repairedFiles: transcriptionFiles,
repairApplied: repairApplied,
audioFolder: audioFolder
)
if let mismatch = captureDurationMismatch(in: transcriptionFiles) {
let recoveryMessage = audioFolder == nil
? " The audio remains in the app's temporary folder."
@@ -405,9 +437,13 @@ final class AudioManager: NSObject, ObservableObject {
advertisedInputFormat.sampleRate > 0 else {
throw NSError(domain: "AudioManager", code: -1, userInfo: [NSLocalizedDescriptionKey: "Unsupported system audio format"])
}
systemDiagnostics.tapAdvertisedFormat = streamDescriptionSummary(tap.tapAdvertisedStreamDescription)
systemDiagnostics.aggregateInputFormat = streamDescriptionSummary(tap.aggregateInputStreamDescription)
systemDiagnostics.targetFormat = audioFormatSummary(targetFormat)
systemDiagnostics.targetSampleRate = targetFormat.sampleRate
var inputFormat: AVAudioFormat?
var converter: AVAudioConverter?
try tap.run(on: tapQueue) { [weak self] _, inputData, _, _, _ in
try tap.run(on: tapQueue) { [weak self] inNow, inputData, inInputTime, _, _ in
guard let self else { return }
if inputFormat == nil {
inputFormat = self.inputFormat(
@@ -415,6 +451,8 @@ final class AudioManager: NSObject, ObservableObject {
advertisedFormat: advertisedInputFormat
)
if let inputFormat {
self.systemDiagnostics.selectedInputFormat = self.audioFormatSummary(inputFormat)
self.systemDiagnostics.selectedInputSampleRate = inputFormat.sampleRate
converter = AVAudioConverter(from: inputFormat, to: targetFormat)
}
}
@@ -425,7 +463,9 @@ final class AudioManager: NSObject, ObservableObject {
{ self.copyAudioBuffer(from: inputData, format: inputFormat) },
converter: converter,
targetFormat: targetFormat,
source: .system
source: .system,
callbackTimestamp: inInputTime.pointee,
ioTimestamp: inNow.pointee
)
} invalidationHandler: { [weak self] _ in
guard let self, self.isRecording else { return }
@@ -465,6 +505,11 @@ final class AudioManager: NSObject, ObservableObject {
let sourceBuffers = UnsafeMutableAudioBufferListPointer(
UnsafeMutablePointer(mutating: inputData)
)
if systemDiagnostics.firstBufferLayout == nil {
systemDiagnostics.firstBufferLayout = sourceBuffers.enumerated().map { index, buffer in
"buffer\(index):channels=\(buffer.mNumberChannels),bytes=\(buffer.mDataByteSize)"
}.joined(separator: "; ")
}
let frameLengths = sourceBuffers.compactMap { source -> AVAudioFrameCount? in
let bytesPerFrame = Int(source.mNumberChannels) * MemoryLayout<Float32>.size
guard bytesPerFrame > 0,
@@ -505,20 +550,44 @@ final class AudioManager: NSObject, ObservableObject {
_ inputBufferProvider: () -> AVAudioPCMBuffer?,
converter: AVAudioConverter,
targetFormat: AVAudioFormat,
source: AudioSource
source: AudioSource,
callbackTimestamp: AudioTimeStamp? = nil,
ioTimestamp: AudioTimeStamp? = nil
) {
// The system callback copies its borrowed Core Audio memory while this
// lock prevents teardown, then conversion operates on the owned copy.
audioFileLock.lock()
defer { audioFileLock.unlock() }
guard isAcceptingAudio,
let inputBuffer = inputBufferProvider(),
inputBuffer.frameLength > 0 else { return }
guard isAcceptingAudio else { return }
if source == .system {
systemDiagnostics.callbackCount += 1
let callbackAt = Date()
if systemDiagnostics.firstCallbackAt == nil { systemDiagnostics.firstCallbackAt = callbackAt }
systemDiagnostics.lastCallbackAt = callbackAt
if let callbackTimestamp {
if systemDiagnostics.firstSampleTime == nil { systemDiagnostics.firstSampleTime = callbackTimestamp.mSampleTime }
systemDiagnostics.lastSampleTime = callbackTimestamp.mSampleTime
}
if let ioTimestamp {
if systemDiagnostics.firstHostTime == nil { systemDiagnostics.firstHostTime = ioTimestamp.mHostTime }
systemDiagnostics.lastHostTime = ioTimestamp.mHostTime
}
}
guard let inputBuffer = inputBufferProvider(), inputBuffer.frameLength > 0 else {
if source == .system { systemDiagnostics.discardedCallbackCount += 1 }
return
}
if source == .system {
systemDiagnostics.inputFrameCount += UInt64(inputBuffer.frameLength)
}
updateAudioLevel(inputBuffer, source: source)
let ratio = targetFormat.sampleRate / inputBuffer.format.sampleRate
let capacity = max(1, AVAudioFrameCount(ceil(Double(inputBuffer.frameLength) * ratio)))
guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else { return }
guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
if source == .system { systemDiagnostics.discardedCallbackCount += 1 }
return
}
var suppliedInput = false
var conversionError: NSError?
let status = converter.convert(to: outputBuffer, error: &conversionError) { _, outputStatus in
@@ -530,7 +599,10 @@ final class AudioManager: NSObject, ObservableObject {
outputStatus.pointee = .haveData
return inputBuffer
}
guard status != .error, conversionError == nil, outputBuffer.frameLength > 0 else { return }
guard status != .error, conversionError == nil, outputBuffer.frameLength > 0 else {
if source == .system { systemDiagnostics.discardedCallbackCount += 1 }
return
}
do {
switch source {
@@ -538,6 +610,7 @@ final class AudioManager: NSObject, ObservableObject {
try micAudioFile?.write(from: outputBuffer)
case .system:
try systemAudioFile?.write(from: outputBuffer)
systemDiagnostics.outputFrameCount += UInt64(outputBuffer.frameLength)
}
} catch {
DispatchQueue.main.async { [weak self] in
@@ -631,15 +704,20 @@ final class AudioManager: NSObject, ObservableObject {
return String(format: "mic %.1fs, system %.1fs", micDuration, systemDuration)
}
private func repairHalfDurationSystemWAVIfNeeded(in files: [URL?]) {
private func repairHalfDurationSystemWAVIfNeeded(in files: [URL?]) -> Bool {
guard files.count >= 2,
let micDuration = audioDuration(at: files[0]),
let systemURL = files[1],
let systemDuration = audioDuration(at: systemURL),
micDuration >= 60,
systemURL.pathExtension.caseInsensitiveCompare("wav") == .orderedSame,
(0.48...0.52).contains(systemDuration / micDuration) else { return }
try? halveWAVSampleRate(at: systemURL)
(0.48...0.52).contains(systemDuration / micDuration) else { return false }
do {
try halveWAVSampleRate(at: systemURL)
return true
} catch {
return false
}
}
private func halveWAVSampleRate(at url: URL) throws {
@@ -704,6 +782,112 @@ final class AudioManager: NSObject, ObservableObject {
file.processingFormat.sampleRate > 0 else { return nil }
return Double(file.length) / file.processingFormat.sampleRate
}
private func streamDescriptionSummary(_ description: AudioStreamBasicDescription?) -> String {
guard let description else { return "unavailable" }
return String(
format: "sampleRate=%.3f,formatID=%u,flags=%u,bytesPerPacket=%u,framesPerPacket=%u,bytesPerFrame=%u,channels=%u,bitsPerChannel=%u",
description.mSampleRate,
description.mFormatID,
description.mFormatFlags,
description.mBytesPerPacket,
description.mFramesPerPacket,
description.mBytesPerFrame,
description.mChannelsPerFrame,
description.mBitsPerChannel
)
}
private func audioFormatSummary(_ format: AVAudioFormat) -> String {
"sampleRate=\(format.sampleRate),channels=\(format.channelCount),commonFormat=\(format.commonFormat.rawValue),interleaved=\(format.isInterleaved)"
}
private func audioFileSummary(_ url: URL?) -> String {
guard let url else { return "missing" }
guard let file = try? AVAudioFile(forReading: url), file.processingFormat.sampleRate > 0 else {
return "\(url.lastPathComponent):unreadable"
}
let duration = Double(file.length) / file.processingFormat.sampleRate
return String(
format: "%@:sampleRate=%.3f,channels=%u,frames=%lld,duration=%.6f",
url.lastPathComponent,
file.processingFormat.sampleRate,
file.processingFormat.channelCount,
file.length,
duration
)
}
private func writeCaptureDiagnostics(
sessionID: UUID,
captureStartedAt: Date,
preRepairFileSummaries: [String],
repairedFiles: [URL?],
repairApplied: Bool,
audioFolder: URL?
) {
guard let audioFolder else { return }
let callbackDuration = systemDiagnostics.firstCallbackAt.flatMap { first in
systemDiagnostics.lastCallbackAt.map { $0.timeIntervalSince(first) }
}
let sampleTimeDelta = systemDiagnostics.firstSampleTime.flatMap { first in
systemDiagnostics.lastSampleTime.map { $0 - first }
}
let hostTimeDelta = systemDiagnostics.firstHostTime.flatMap { first in
systemDiagnostics.lastHostTime.map { $0 >= first ? $0 - first : 0 }
}
let inputFrameDuration = systemDiagnostics.selectedInputSampleRate.flatMap { sampleRate in
sampleRate > 0 ? Double(systemDiagnostics.inputFrameCount) / sampleRate : nil
}
let outputFrameDuration = systemDiagnostics.targetSampleRate.flatMap { sampleRate in
sampleRate > 0 ? Double(systemDiagnostics.outputFrameCount) / sampleRate : nil
}
let observedInputRate = callbackDuration.flatMap { duration in
duration > 0 ? Double(systemDiagnostics.inputFrameCount) / duration : nil
}
let preRepairMic = preRepairFileSummaries.indices.contains(0) ? preRepairFileSummaries[0] : "missing"
let preRepairSystem = preRepairFileSummaries.indices.contains(1) ? preRepairFileSummaries[1] : "missing"
let postRepairMic = repairedFiles.indices.contains(0) ? audioFileSummary(repairedFiles[0]) : "missing"
let postRepairSystem = repairedFiles.indices.contains(1) ? audioFileSummary(repairedFiles[1]) : "missing"
let callbackDurationLine = callbackDuration.map { String(format: "callbackWallDuration=%.6f", $0) } ?? "callbackWallDuration=unavailable"
let inputFrameDurationLine = inputFrameDuration.map { String(format: "inputFrameDurationAtSelectedRate=%.6f", $0) } ?? "inputFrameDurationAtSelectedRate=unavailable"
let outputFrameDurationLine = outputFrameDuration.map { String(format: "outputFrameDurationAtTargetRate=%.6f", $0) } ?? "outputFrameDurationAtTargetRate=unavailable"
let observedInputRateLine = observedInputRate.map { String(format: "observedInputFramesPerSecond=%.3f", $0) } ?? "observedInputFramesPerSecond=unavailable"
let sampleTimeDeltaLine = sampleTimeDelta.map { String(format: "sampleTimeDelta=%.6f", $0) } ?? "sampleTimeDelta=unavailable"
let hostTimeDeltaLine = hostTimeDelta.map { "hostTimeDelta=\($0)" } ?? "hostTimeDelta=unavailable"
let lines: [String] = [
"Meetingnotes system capture diagnostics",
"sessionID=\(sessionID.uuidString)",
"createdAt=\(ISO8601DateFormatter().string(from: Date()))",
String(format: "captureWallDuration=%.6f", Date().timeIntervalSince(captureStartedAt)),
"tapAdvertisedFormat=\(systemDiagnostics.tapAdvertisedFormat)",
"aggregateInputFormat=\(systemDiagnostics.aggregateInputFormat)",
"selectedInputFormat=\(systemDiagnostics.selectedInputFormat)",
"targetFormat=\(systemDiagnostics.targetFormat)",
"firstBufferLayout=\(systemDiagnostics.firstBufferLayout ?? "unavailable")",
"callbackCount=\(systemDiagnostics.callbackCount)",
"inputFrameCount=\(systemDiagnostics.inputFrameCount)",
"outputFrameCount=\(systemDiagnostics.outputFrameCount)",
"discardedCallbackCount=\(systemDiagnostics.discardedCallbackCount)",
callbackDurationLine,
inputFrameDurationLine,
outputFrameDurationLine,
observedInputRateLine,
sampleTimeDeltaLine,
hostTimeDeltaLine,
"repairApplied=\(repairApplied)",
"preRepairMic=\(preRepairMic)",
"preRepairSystem=\(preRepairSystem)",
"postRepairMic=\(postRepairMic)",
"postRepairSystem=\(postRepairSystem)"
]
let diagnosticsURL = audioFolder.appendingPathComponent("audio-diagnostics-\(sessionID.uuidString).txt")
try? lines.joined(separator: "\n").appending("\n").write(
to: diagnosticsURL,
atomically: true,
encoding: .utf8
)
}
private func resetAudioLevels() {
micAudioLevel = 0