1018 lines
44 KiB
Swift
1018 lines
44 KiB
Swift
import AVFoundation
|
|
import Combine
|
|
import Foundation
|
|
import SwiftUI
|
|
|
|
private enum RecoveryTranscriptionError: LocalizedError {
|
|
case noAudioFiles
|
|
case noSpeech
|
|
case requestFailed(String)
|
|
|
|
var errorDescription: String? {
|
|
switch self {
|
|
case .noAudioFiles:
|
|
return "The saved recovery audio could not be found."
|
|
case .noSpeech:
|
|
return "No speech was detected in the saved recovery audio."
|
|
case .requestFailed(let details):
|
|
return "Retry transcription failed for \(details)"
|
|
}
|
|
}
|
|
}
|
|
|
|
private struct SystemCaptureDiagnostics {
|
|
var tapAdvertisedFormat = "unavailable"
|
|
var aggregateInputFormat = "unavailable"
|
|
var selectedInputFormat = "unavailable"
|
|
var targetFormat = "unavailable"
|
|
var selectedInputSampleRate: Double?
|
|
var targetSampleRate: Double?
|
|
var inputRateInference = "unavailable"
|
|
var firstBufferLayout: String?
|
|
var callbackCount: UInt64 = 0
|
|
var inputFrameCount: UInt64 = 0
|
|
var outputFrameCount: UInt64 = 0
|
|
var discardedCallbackCount: UInt64 = 0
|
|
var firstSampleTime: Double?
|
|
var lastSampleTime: Double?
|
|
var firstHostTime: UInt64?
|
|
var lastHostTime: UInt64?
|
|
var firstCallbackAt: Date?
|
|
var lastCallbackAt: Date?
|
|
}
|
|
|
|
/// Captures microphone and system audio locally, then sends completed files to Coder.
|
|
@MainActor
|
|
final class AudioManager: NSObject, ObservableObject {
|
|
static let shared = AudioManager()
|
|
|
|
@Published var transcriptChunks: [TranscriptChunk] = []
|
|
@Published var isRecording = false
|
|
@Published var isProcessing = false
|
|
@Published var errorMessage: String?
|
|
@Published var micAudioLevel: Float = 0
|
|
@Published var systemAudioLevel: Float = 0
|
|
private(set) var lastRecoveryAudioFolderName: String?
|
|
|
|
private var audioEngine = AVAudioEngine()
|
|
private var sessionID = UUID()
|
|
private var meetingID = UUID()
|
|
private var processTap: ProcessTap?
|
|
private let permission = AudioRecordingPermission()
|
|
private let tapQueue = DispatchQueue(label: "io.meetingnotes.audiotap", qos: .userInitiated)
|
|
private let audioFileLock = NSLock()
|
|
private var isTapActive = false
|
|
private var isAcceptingAudio = false
|
|
private var micRetryCount = 0
|
|
private var pendingMicRestart: DispatchWorkItem?
|
|
private let maxMicRetries = 3
|
|
private var micAudioFile: AVAudioFile?
|
|
private var systemAudioFile: AVAudioFile?
|
|
private var micAudioURL: URL?
|
|
private var systemAudioURL: URL?
|
|
private var recordingStartedAt = Date()
|
|
private var systemDiagnostics = SystemCaptureDiagnostics()
|
|
|
|
override init() {
|
|
super.init()
|
|
observeAudioEngine()
|
|
}
|
|
|
|
deinit {
|
|
NotificationCenter.default.removeObserver(self)
|
|
}
|
|
|
|
func startRecording(for meetingID: UUID) {
|
|
errorMessage = nil
|
|
lastRecoveryAudioFolderName = nil
|
|
cancelCapture(removeFiles: true)
|
|
sessionID = UUID()
|
|
self.meetingID = meetingID
|
|
recordingStartedAt = Date()
|
|
systemDiagnostics = SystemCaptureDiagnostics()
|
|
do {
|
|
try prepareAudioFiles()
|
|
startMicrophoneTap()
|
|
Task { await startSystemAudioTap() }
|
|
} catch {
|
|
errorMessage = "Could not prepare meeting audio: \(error.localizedDescription)"
|
|
cancelCapture(removeFiles: true)
|
|
}
|
|
}
|
|
|
|
func stopRecordingAndTranscribe() async -> [TranscriptChunk] {
|
|
let completedMeetingID = meetingID
|
|
let completedSessionID = sessionID
|
|
let captureStartedAt = recordingStartedAt
|
|
let files = stopCaptureAndCloseFiles()
|
|
isProcessing = true
|
|
defer {
|
|
isProcessing = false
|
|
}
|
|
|
|
let preRepairFileSummaries = files.map(audioFileSummary)
|
|
let repairApplied = repairHalfDurationSystemWAVIfNeeded(in: files)
|
|
let completedFiles = files.compactMap { $0 }
|
|
let audioFolder = preserveAudioFiles(completedFiles, meetingID: completedMeetingID)
|
|
let transcriptionFiles = preservedAudioFiles(files, in: audioFolder)
|
|
lastRecoveryAudioFolderName = audioFolder?.lastPathComponent
|
|
writeCaptureDiagnostics(
|
|
sessionID: completedSessionID,
|
|
captureStartedAt: captureStartedAt,
|
|
preRepairFileSummaries: preRepairFileSummaries,
|
|
repairedFiles: transcriptionFiles,
|
|
repairApplied: repairApplied,
|
|
audioFolder: audioFolder
|
|
)
|
|
if let mismatch = captureDurationMismatch(in: transcriptionFiles) {
|
|
let recoveryMessage = audioFolder == nil
|
|
? " The audio remains in the app's temporary folder."
|
|
: " Audio was kept so it can be recovered."
|
|
errorMessage = "System audio timing was invalid (\(mismatch)). Transcription was stopped to avoid an out-of-order result." + recoveryMessage
|
|
return transcriptChunks.filter(\.isFinal)
|
|
}
|
|
|
|
let model = UserDefaultsManager.shared.transcriptionModel
|
|
async let micResult = transcribe(transcriptionFiles[0], model: model, diarization: false)
|
|
async let systemResult = transcribe(transcriptionFiles[1], model: model, diarization: true)
|
|
let (micTranscription, systemTranscription) = await (micResult, systemResult)
|
|
let results = [micTranscription, systemTranscription]
|
|
|
|
let (updated, failures) = buildTranscriptChunks(
|
|
from: results,
|
|
captureStartedAt: captureStartedAt,
|
|
existingChunks: transcriptChunks.filter(\.isFinal)
|
|
)
|
|
transcriptChunks = updated
|
|
if !failures.isEmpty {
|
|
let retentionDays = UserDefaultsManager.shared.audioRetentionDays
|
|
let retentionUnit = retentionDays == 1 ? "day" : "days"
|
|
let recoveryMessage = audioFolder == nil
|
|
? " The audio remains in the app's temporary folder."
|
|
: " Audio was kept for \(retentionDays) \(retentionUnit). Use Show Audio Folder in the Meetingnotes menu to find it."
|
|
errorMessage = "Transcription failed for " + failures.joined(separator: "; ") + recoveryMessage
|
|
} else if audioFolder == nil, !completedFiles.isEmpty {
|
|
errorMessage = "The transcript completed, but Meetingnotes could not move the audio into its retention folder."
|
|
}
|
|
return updated
|
|
}
|
|
|
|
func transcribeRecoveryAudio(in folder: URL, captureStartedAt: Date) async throws -> [TranscriptChunk] {
|
|
let recoveryFiles = LocalStorageManager.shared.recoveryAudioFiles(in: folder)
|
|
guard !recoveryFiles.isEmpty else {
|
|
throw RecoveryTranscriptionError.noAudioFiles
|
|
}
|
|
|
|
isProcessing = true
|
|
defer { isProcessing = false }
|
|
let model = UserDefaultsManager.shared.transcriptionModel
|
|
let sessions = Dictionary(grouping: recoveryFiles) { recoverySessionKey(for: $0.url) }
|
|
.values
|
|
.map { files in
|
|
(files: files, startedAt: recoveryCaptureStartedAt(for: files, fallback: captureStartedAt))
|
|
}
|
|
.sorted { $0.startedAt < $1.startedAt }
|
|
|
|
var chunks: [TranscriptChunk] = []
|
|
var failures: [String] = []
|
|
for session in sessions {
|
|
let micURL = session.files.first(where: { $0.source == .mic })?.url
|
|
let systemURL = session.files.first(where: { $0.source == .system })?.url
|
|
async let micResult = transcribe(micURL, model: model, diarization: false)
|
|
async let systemResult = transcribe(systemURL, model: model, diarization: true)
|
|
let (micTranscription, systemTranscription) = await (micResult, systemResult)
|
|
let result = buildTranscriptChunks(
|
|
from: [micTranscription, systemTranscription],
|
|
captureStartedAt: session.startedAt,
|
|
existingChunks: chunks
|
|
)
|
|
chunks = result.0
|
|
failures.append(contentsOf: result.1)
|
|
}
|
|
|
|
if !failures.isEmpty {
|
|
throw RecoveryTranscriptionError.requestFailed(failures.joined(separator: "; "))
|
|
}
|
|
guard !chunks.isEmpty else {
|
|
throw RecoveryTranscriptionError.noSpeech
|
|
}
|
|
return chunks
|
|
}
|
|
|
|
private func recoverySessionKey(for url: URL) -> String {
|
|
let name = url.deletingPathExtension().lastPathComponent
|
|
if name.hasSuffix("-mic") { return String(name.dropLast(4)) }
|
|
if name.hasSuffix("-system") { return String(name.dropLast(7)) }
|
|
return name
|
|
}
|
|
|
|
private func recoveryCaptureStartedAt(
|
|
for files: [(url: URL, source: AudioSource)],
|
|
fallback: Date
|
|
) -> Date {
|
|
let estimatedStarts = files.compactMap { file -> Date? in
|
|
guard let duration = audioDuration(at: file.url),
|
|
let values = try? file.url.resourceValues(forKeys: [.contentModificationDateKey, .creationDateKey]),
|
|
let finishedAt = values.contentModificationDate ?? values.creationDate else {
|
|
return nil
|
|
}
|
|
return finishedAt.addingTimeInterval(-duration)
|
|
}
|
|
return estimatedStarts.min() ?? fallback
|
|
}
|
|
|
|
func cancelRecording() {
|
|
cancelCapture(removeFiles: true)
|
|
lastRecoveryAudioFolderName = nil
|
|
}
|
|
|
|
private func transcribe(
|
|
_ fileURL: URL?,
|
|
model: String,
|
|
diarization: Bool
|
|
) async -> Result<CoderAPIClient.Transcription, Error>? {
|
|
guard let fileURL else { return nil }
|
|
do {
|
|
return .success(try await CoderAPIClient.shared.transcribe(
|
|
fileURL: fileURL,
|
|
model: model,
|
|
diarization: diarization,
|
|
maxSpeakerCount: 4
|
|
))
|
|
} catch {
|
|
return .failure(error)
|
|
}
|
|
}
|
|
|
|
private func buildTranscriptChunks(
|
|
from results: [Result<CoderAPIClient.Transcription, Error>?],
|
|
captureStartedAt: Date,
|
|
existingChunks: [TranscriptChunk]
|
|
) -> ([TranscriptChunk], [String]) {
|
|
var updated = existingChunks
|
|
var failures: [String] = []
|
|
for (source, result) in zip([AudioSource.mic, .system], results) {
|
|
guard let result else { continue }
|
|
switch result {
|
|
case .success(let transcription):
|
|
if transcription.segments.isEmpty {
|
|
let text = transcription.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
if !text.isEmpty {
|
|
updated.append(TranscriptChunk(timestamp: captureStartedAt, source: source, text: text, isFinal: true))
|
|
}
|
|
} else {
|
|
for segment in transcription.segments {
|
|
let text = segment.text.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
guard !text.isEmpty else { continue }
|
|
updated.append(TranscriptChunk(
|
|
timestamp: captureStartedAt.addingTimeInterval(max(0, segment.start)),
|
|
source: source,
|
|
speaker: source == .system ? segment.speaker : nil,
|
|
text: text,
|
|
isFinal: true
|
|
))
|
|
}
|
|
}
|
|
case .failure(let error):
|
|
failures.append("\(source.displayName): \(error.localizedDescription)")
|
|
}
|
|
}
|
|
updated.sort {
|
|
if $0.timestamp != $1.timestamp { return $0.timestamp < $1.timestamp }
|
|
return $0.source.rawValue < $1.source.rawValue
|
|
}
|
|
return (updated, failures)
|
|
}
|
|
|
|
private func prepareAudioFiles() throws {
|
|
let settings: [String: Any] = [
|
|
AVFormatIDKey: kAudioFormatLinearPCM,
|
|
AVSampleRateKey: 16_000,
|
|
AVNumberOfChannelsKey: 1,
|
|
AVLinearPCMBitDepthKey: 16,
|
|
AVLinearPCMIsFloatKey: false,
|
|
AVLinearPCMIsBigEndianKey: false,
|
|
AVLinearPCMIsNonInterleaved: false
|
|
]
|
|
let base = FileManager.default.temporaryDirectory
|
|
let id = sessionID.uuidString
|
|
let micURL = base.appendingPathComponent("meetingnotes-\(id)-mic.wav")
|
|
let systemURL = base.appendingPathComponent("meetingnotes-\(id)-system.wav")
|
|
let newMicAudioFile = try AVAudioFile(
|
|
forWriting: micURL,
|
|
settings: settings,
|
|
commonFormat: .pcmFormatFloat32,
|
|
interleaved: false
|
|
)
|
|
let newSystemAudioFile = try AVAudioFile(
|
|
forWriting: systemURL,
|
|
settings: settings,
|
|
commonFormat: .pcmFormatFloat32,
|
|
interleaved: false
|
|
)
|
|
|
|
audioFileLock.lock()
|
|
micAudioFile = newMicAudioFile
|
|
systemAudioFile = newSystemAudioFile
|
|
micAudioURL = micURL
|
|
systemAudioURL = systemURL
|
|
isAcceptingAudio = true
|
|
audioFileLock.unlock()
|
|
}
|
|
|
|
private func startMicrophoneTap() {
|
|
do {
|
|
let inputNode = audioEngine.inputNode
|
|
let inputFormat = inputNode.outputFormat(forBus: 0)
|
|
guard let targetFormat = micAudioFile?.processingFormat,
|
|
let converter = AVAudioConverter(from: inputFormat, to: targetFormat) else {
|
|
throw NSError(domain: "AudioManager", code: -1, userInfo: [NSLocalizedDescriptionKey: "Unsupported microphone format"])
|
|
}
|
|
inputNode.installTap(onBus: 0, bufferSize: 1024, format: inputFormat) { [weak self] buffer, _ in
|
|
guard let self else { return }
|
|
self.processAudioBuffer(
|
|
{ buffer },
|
|
converter: converter,
|
|
targetFormat: targetFormat,
|
|
source: .mic
|
|
)
|
|
}
|
|
audioEngine.prepare()
|
|
try audioEngine.start()
|
|
micRetryCount = 0
|
|
} catch {
|
|
errorMessage = "Could not start microphone capture: \(error.localizedDescription)"
|
|
restartMicrophone()
|
|
}
|
|
}
|
|
|
|
private func restartMicrophone() {
|
|
guard hasActiveAudioFiles(), micRetryCount < maxMicRetries else { return }
|
|
micRetryCount += 1
|
|
pendingMicRestart?.cancel()
|
|
cleanupAudioEngine()
|
|
|
|
let restart = DispatchWorkItem { [weak self] in
|
|
guard let self, self.hasActiveAudioFiles() else { return }
|
|
self.startMicrophoneTap()
|
|
}
|
|
pendingMicRestart = restart
|
|
DispatchQueue.main.asyncAfter(deadline: .now() + 1, execute: restart)
|
|
}
|
|
|
|
private func cleanupAudioEngine() {
|
|
if audioEngine.isRunning { audioEngine.stop() }
|
|
audioEngine.inputNode.removeTap(onBus: 0)
|
|
audioEngine.reset()
|
|
audioEngine = AVAudioEngine()
|
|
observeAudioEngine()
|
|
}
|
|
|
|
private func observeAudioEngine() {
|
|
NotificationCenter.default.addObserver(
|
|
forName: .AVAudioEngineConfigurationChange,
|
|
object: audioEngine,
|
|
queue: .main
|
|
) { [weak self] _ in
|
|
self?.handleAudioEngineConfigurationChange()
|
|
}
|
|
}
|
|
|
|
private func startSystemAudioTap(isRestart: Bool = false) async {
|
|
if !isRestart, !(await checkSystemAudioPermissions()) {
|
|
errorMessage = "System audio recording permission denied."
|
|
cancelCapture(removeFiles: true)
|
|
return
|
|
}
|
|
|
|
let newTap = ProcessTap(target: .systemAudio)
|
|
newTap.activate()
|
|
if let tapError = newTap.errorMessage {
|
|
errorMessage = "Failed to activate system audio capture: \(tapError)"
|
|
if !isRestart { cancelCapture(removeFiles: true) }
|
|
return
|
|
}
|
|
|
|
processTap = newTap
|
|
isTapActive = true
|
|
do {
|
|
try startTapIO(newTap)
|
|
if !isRestart {
|
|
isRecording = true
|
|
AudioLevelManager.shared.updateRecordingState(true)
|
|
}
|
|
} catch {
|
|
errorMessage = "Failed to capture system audio: \(error.localizedDescription)"
|
|
newTap.invalidate()
|
|
isTapActive = false
|
|
if !isRestart { cancelCapture(removeFiles: true) }
|
|
}
|
|
}
|
|
|
|
private func restartSystemAudioTap() async {
|
|
guard isRecording else { return }
|
|
if isTapActive {
|
|
processTap?.invalidate()
|
|
processTap = nil
|
|
isTapActive = false
|
|
}
|
|
try? await Task.sleep(for: .milliseconds(250))
|
|
guard isRecording else { return }
|
|
await startSystemAudioTap(isRestart: true)
|
|
}
|
|
|
|
private func checkSystemAudioPermissions() async -> Bool {
|
|
if permission.status == .authorized { return true }
|
|
permission.request()
|
|
for _ in 0..<10 {
|
|
if permission.status == .authorized { return true }
|
|
try? await Task.sleep(nanoseconds: 500_000_000)
|
|
}
|
|
return permission.status == .authorized
|
|
}
|
|
|
|
private func startTapIO(_ tap: ProcessTap) throws {
|
|
guard var description = tap.tapStreamDescription,
|
|
let advertisedInputFormat = AVAudioFormat(streamDescription: &description),
|
|
let targetFormat = systemAudioFile?.processingFormat,
|
|
advertisedInputFormat.sampleRate > 0 else {
|
|
throw NSError(domain: "AudioManager", code: -1, userInfo: [NSLocalizedDescriptionKey: "Unsupported system audio format"])
|
|
}
|
|
systemDiagnostics.tapAdvertisedFormat = streamDescriptionSummary(tap.tapAdvertisedStreamDescription)
|
|
systemDiagnostics.aggregateInputFormat = streamDescriptionSummary(tap.aggregateInputStreamDescription)
|
|
systemDiagnostics.targetFormat = audioFormatSummary(targetFormat)
|
|
systemDiagnostics.targetSampleRate = targetFormat.sampleRate
|
|
var inputFormat: AVAudioFormat?
|
|
var converter: AVAudioConverter?
|
|
var pendingBuffer: AVAudioPCMBuffer?
|
|
var pendingInputTime: AudioTimeStamp?
|
|
var pendingNow: AudioTimeStamp?
|
|
try tap.run(on: tapQueue) { [weak self] inNow, inputData, inInputTime, _, _ in
|
|
guard let self else { return }
|
|
if inputFormat == nil {
|
|
guard let callbackFormat = self.inputFormat(
|
|
for: inputData,
|
|
sampleRate: advertisedInputFormat.sampleRate
|
|
),
|
|
let currentBuffer = self.copyAudioBuffer(from: inputData, format: callbackFormat) else {
|
|
self.systemDiagnostics.discardedCallbackCount += 1
|
|
return
|
|
}
|
|
|
|
guard let previousBuffer = pendingBuffer,
|
|
let previousInputTime = pendingInputTime,
|
|
let previousNow = pendingNow else {
|
|
pendingBuffer = currentBuffer
|
|
pendingInputTime = inInputTime.pointee
|
|
pendingNow = inNow.pointee
|
|
return
|
|
}
|
|
|
|
let selectedSampleRate = self.effectiveInputSampleRate(
|
|
advertisedSampleRate: advertisedInputFormat.sampleRate,
|
|
previousFrameLength: previousBuffer.frameLength,
|
|
previousTimestamp: previousInputTime,
|
|
currentTimestamp: inInputTime.pointee,
|
|
previousHostTimestamp: previousNow,
|
|
currentHostTimestamp: inNow.pointee
|
|
)
|
|
inputFormat = self.inputFormat(
|
|
for: inputData,
|
|
sampleRate: selectedSampleRate
|
|
)
|
|
if let inputFormat {
|
|
self.systemDiagnostics.selectedInputFormat = self.audioFormatSummary(inputFormat)
|
|
self.systemDiagnostics.selectedInputSampleRate = inputFormat.sampleRate
|
|
converter = AVAudioConverter(from: inputFormat, to: targetFormat)
|
|
}
|
|
|
|
guard let inputFormat, let converter else { return }
|
|
self.processAudioBuffer(
|
|
{ self.copyAudioBuffer(from: previousBuffer.audioBufferList, format: inputFormat) },
|
|
converter: converter,
|
|
targetFormat: targetFormat,
|
|
source: .system,
|
|
callbackTimestamp: previousInputTime,
|
|
ioTimestamp: pendingNow
|
|
)
|
|
pendingBuffer = nil
|
|
pendingInputTime = nil
|
|
pendingNow = nil
|
|
|
|
self.processAudioBuffer(
|
|
{ self.copyAudioBuffer(from: currentBuffer.audioBufferList, format: inputFormat) },
|
|
converter: converter,
|
|
targetFormat: targetFormat,
|
|
source: .system,
|
|
callbackTimestamp: inInputTime.pointee,
|
|
ioTimestamp: inNow.pointee
|
|
)
|
|
return
|
|
}
|
|
guard let inputFormat, let converter else { return }
|
|
// The tap queue is serial. Reusing the converter preserves its
|
|
// resampler state instead of discarding audio at every callback.
|
|
self.processAudioBuffer(
|
|
{ self.copyAudioBuffer(from: inputData, format: inputFormat) },
|
|
converter: converter,
|
|
targetFormat: targetFormat,
|
|
source: .system,
|
|
callbackTimestamp: inInputTime.pointee,
|
|
ioTimestamp: inNow.pointee
|
|
)
|
|
} invalidationHandler: { [weak self] _ in
|
|
guard let self, self.isRecording else { return }
|
|
Task { await self.restartSystemAudioTap() }
|
|
}
|
|
}
|
|
|
|
private func inputFormat(
|
|
for inputData: UnsafePointer<AudioBufferList>,
|
|
sampleRate: Double
|
|
) -> AVAudioFormat? {
|
|
let buffers = UnsafeMutableAudioBufferListPointer(
|
|
UnsafeMutablePointer(mutating: inputData)
|
|
)
|
|
let channelCount = buffers.reduce(UInt32(0)) { $0 + $1.mNumberChannels }
|
|
guard channelCount > 0 else { return nil }
|
|
|
|
// HAL I/O proc samples use the canonical Float32 representation. The
|
|
// tap's stream description can advertise a different common format;
|
|
// using that to interpret the callback bytes can halve the frame count
|
|
// (for example, treating four-byte Float32 samples as eight-byte
|
|
// Float64 samples). The callback's AudioBufferList is authoritative for
|
|
// its channel layout, while its sample rate comes from the input stream.
|
|
let isInterleaved = buffers.count == 1 && channelCount > 1
|
|
return AVAudioFormat(
|
|
commonFormat: .pcmFormatFloat32,
|
|
sampleRate: sampleRate,
|
|
channels: AVAudioChannelCount(channelCount),
|
|
interleaved: isInterleaved
|
|
)
|
|
}
|
|
|
|
private func effectiveInputSampleRate(
|
|
advertisedSampleRate: Double,
|
|
previousFrameLength: AVAudioFrameCount,
|
|
previousTimestamp: AudioTimeStamp,
|
|
currentTimestamp: AudioTimeStamp,
|
|
previousHostTimestamp: AudioTimeStamp,
|
|
currentHostTimestamp: AudioTimeStamp
|
|
) -> Double {
|
|
let sampleTimeDelta = currentTimestamp.mSampleTime - previousTimestamp.mSampleTime
|
|
guard advertisedSampleRate > 0, previousFrameLength > 0 else {
|
|
systemDiagnostics.inputRateInference = "advertised (invalid advertised rate or frame count)"
|
|
return advertisedSampleRate
|
|
}
|
|
|
|
guard currentHostTimestamp.mHostTime > previousHostTimestamp.mHostTime else {
|
|
systemDiagnostics.inputRateInference = "advertised (host timestamps unavailable)"
|
|
return advertisedSampleRate
|
|
}
|
|
let hostTimeDelta = currentHostTimestamp.mHostTime - previousHostTimestamp.mHostTime
|
|
let hostNanoseconds = AudioConvertHostTimeToNanos(hostTimeDelta)
|
|
guard hostNanoseconds > 0 else {
|
|
systemDiagnostics.inputRateInference = "advertised (host-time conversion failed)"
|
|
return advertisedSampleRate
|
|
}
|
|
|
|
let inferredSampleRate = Double(previousFrameLength) * 1_000_000_000 / Double(hostNanoseconds)
|
|
let plausibleRange = (advertisedSampleRate * 0.25)...(advertisedSampleRate * 1.25)
|
|
guard inferredSampleRate.isFinite, plausibleRange.contains(inferredSampleRate) else {
|
|
systemDiagnostics.inputRateInference = String(
|
|
format: "advertised (invalid host inference %.3f from %u frames / %llu ns; sampleTimeDelta=%.3f)",
|
|
inferredSampleRate,
|
|
previousFrameLength,
|
|
hostNanoseconds,
|
|
sampleTimeDelta
|
|
)
|
|
return advertisedSampleRate
|
|
}
|
|
|
|
let commonSampleRates: [Double] = [8_000, 11_025, 12_000, 16_000, 22_050, 24_000, 32_000, 44_100, 48_000, 88_200, 96_000]
|
|
let selectedSampleRate = commonSampleRates
|
|
.min(by: { abs($0 - inferredSampleRate) < abs($1 - inferredSampleRate) })
|
|
.flatMap { abs($0 - inferredSampleRate) / $0 <= 0.01 ? $0 : nil }
|
|
?? inferredSampleRate
|
|
systemDiagnostics.inputRateInference = String(
|
|
format: "source=hostTime,advertised=%.3f,inferred=%.3f,selected=%.3f,previousFrames=%u,hostNanoseconds=%llu,sampleTimeDelta=%.3f",
|
|
advertisedSampleRate,
|
|
inferredSampleRate,
|
|
selectedSampleRate,
|
|
previousFrameLength,
|
|
hostNanoseconds,
|
|
sampleTimeDelta
|
|
)
|
|
return selectedSampleRate
|
|
}
|
|
|
|
private func copyAudioBuffer(
|
|
from inputData: UnsafePointer<AudioBufferList>,
|
|
format: AVAudioFormat
|
|
) -> AVAudioPCMBuffer? {
|
|
let sourceBuffers = UnsafeMutableAudioBufferListPointer(
|
|
UnsafeMutablePointer(mutating: inputData)
|
|
)
|
|
if systemDiagnostics.firstBufferLayout == nil {
|
|
systemDiagnostics.firstBufferLayout = sourceBuffers.enumerated().map { index, buffer in
|
|
"buffer\(index):channels=\(buffer.mNumberChannels),bytes=\(buffer.mDataByteSize)"
|
|
}.joined(separator: "; ")
|
|
}
|
|
let frameLengths = sourceBuffers.compactMap { source -> AVAudioFrameCount? in
|
|
let bytesPerFrame = Int(source.mNumberChannels) * MemoryLayout<Float32>.size
|
|
guard bytesPerFrame > 0,
|
|
Int(source.mDataByteSize).isMultiple(of: bytesPerFrame) else { return nil }
|
|
return AVAudioFrameCount(Int(source.mDataByteSize) / bytesPerFrame)
|
|
}
|
|
guard frameLengths.count == sourceBuffers.count,
|
|
let frameLength = frameLengths.first,
|
|
frameLength > 0,
|
|
frameLengths.allSatisfy({ $0 == frameLength }),
|
|
let ownedBuffer = AVAudioPCMBuffer(
|
|
pcmFormat: format,
|
|
frameCapacity: frameLength
|
|
) else { return nil }
|
|
|
|
// Set the frame count explicitly instead of asking AVAudioPCMBuffer to
|
|
// infer it from potentially inconsistent tap metadata.
|
|
ownedBuffer.frameLength = frameLength
|
|
let destinationBuffers = UnsafeMutableAudioBufferListPointer(
|
|
ownedBuffer.mutableAudioBufferList
|
|
)
|
|
guard sourceBuffers.count == destinationBuffers.count else { return nil }
|
|
|
|
for index in 0..<sourceBuffers.count {
|
|
let source = sourceBuffers[index]
|
|
let destination = destinationBuffers[index]
|
|
let byteCount = Int(source.mDataByteSize)
|
|
guard byteCount <= Int(destination.mDataByteSize),
|
|
let sourceData = source.mData,
|
|
let destinationData = destination.mData else { return nil }
|
|
memcpy(destinationData, sourceData, byteCount)
|
|
destinationBuffers[index].mDataByteSize = source.mDataByteSize
|
|
}
|
|
return ownedBuffer
|
|
}
|
|
|
|
private func processAudioBuffer(
|
|
_ inputBufferProvider: () -> AVAudioPCMBuffer?,
|
|
converter: AVAudioConverter,
|
|
targetFormat: AVAudioFormat,
|
|
source: AudioSource,
|
|
callbackTimestamp: AudioTimeStamp? = nil,
|
|
ioTimestamp: AudioTimeStamp? = nil
|
|
) {
|
|
// The system callback copies its borrowed Core Audio memory while this
|
|
// lock prevents teardown, then conversion operates on the owned copy.
|
|
audioFileLock.lock()
|
|
defer { audioFileLock.unlock() }
|
|
guard isAcceptingAudio else { return }
|
|
if source == .system {
|
|
systemDiagnostics.callbackCount += 1
|
|
let callbackAt = Date()
|
|
if systemDiagnostics.firstCallbackAt == nil { systemDiagnostics.firstCallbackAt = callbackAt }
|
|
systemDiagnostics.lastCallbackAt = callbackAt
|
|
if let callbackTimestamp {
|
|
if systemDiagnostics.firstSampleTime == nil { systemDiagnostics.firstSampleTime = callbackTimestamp.mSampleTime }
|
|
systemDiagnostics.lastSampleTime = callbackTimestamp.mSampleTime
|
|
}
|
|
if let ioTimestamp {
|
|
if systemDiagnostics.firstHostTime == nil { systemDiagnostics.firstHostTime = ioTimestamp.mHostTime }
|
|
systemDiagnostics.lastHostTime = ioTimestamp.mHostTime
|
|
}
|
|
}
|
|
guard let inputBuffer = inputBufferProvider(), inputBuffer.frameLength > 0 else {
|
|
if source == .system { systemDiagnostics.discardedCallbackCount += 1 }
|
|
return
|
|
}
|
|
if source == .system {
|
|
systemDiagnostics.inputFrameCount += UInt64(inputBuffer.frameLength)
|
|
}
|
|
|
|
updateAudioLevel(inputBuffer, source: source)
|
|
let ratio = targetFormat.sampleRate / inputBuffer.format.sampleRate
|
|
let capacity = max(1, AVAudioFrameCount(ceil(Double(inputBuffer.frameLength) * ratio)))
|
|
guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: capacity) else {
|
|
if source == .system { systemDiagnostics.discardedCallbackCount += 1 }
|
|
return
|
|
}
|
|
var suppliedInput = false
|
|
var conversionError: NSError?
|
|
let status = converter.convert(to: outputBuffer, error: &conversionError) { _, outputStatus in
|
|
if suppliedInput {
|
|
outputStatus.pointee = .noDataNow
|
|
return nil
|
|
}
|
|
suppliedInput = true
|
|
outputStatus.pointee = .haveData
|
|
return inputBuffer
|
|
}
|
|
guard status != .error, conversionError == nil, outputBuffer.frameLength > 0 else {
|
|
if source == .system { systemDiagnostics.discardedCallbackCount += 1 }
|
|
return
|
|
}
|
|
|
|
do {
|
|
switch source {
|
|
case .mic:
|
|
try micAudioFile?.write(from: outputBuffer)
|
|
case .system:
|
|
try systemAudioFile?.write(from: outputBuffer)
|
|
systemDiagnostics.outputFrameCount += UInt64(outputBuffer.frameLength)
|
|
}
|
|
} catch {
|
|
DispatchQueue.main.async { [weak self] in
|
|
self?.errorMessage = "Could not save meeting audio: \(error.localizedDescription)"
|
|
}
|
|
}
|
|
}
|
|
|
|
private func updateAudioLevel(_ buffer: AVAudioPCMBuffer, source: AudioSource) {
|
|
guard let channel = buffer.floatChannelData?[0], buffer.frameLength > 0 else { return }
|
|
let samples = UnsafeBufferPointer(start: channel, count: Int(buffer.frameLength))
|
|
let rms = sqrt(samples.reduce(0) { $0 + ($1 * $1) } / Float(buffer.frameLength))
|
|
DispatchQueue.main.async { [weak self] in
|
|
guard let self else { return }
|
|
switch source {
|
|
case .mic:
|
|
self.micAudioLevel = rms
|
|
AudioLevelManager.shared.updateMicLevel(rms)
|
|
case .system:
|
|
self.systemAudioLevel = rms
|
|
AudioLevelManager.shared.updateSystemLevel(rms)
|
|
}
|
|
}
|
|
}
|
|
|
|
private func stopCaptureAndCloseFiles() -> [URL?] {
|
|
isRecording = false
|
|
pendingMicRestart?.cancel()
|
|
pendingMicRestart = nil
|
|
AudioLevelManager.shared.updateRecordingState(false)
|
|
|
|
// Stop new callbacks and wait for any active conversion/write before
|
|
// invalidating callback-owned buffers or finalizing AVAudioFile.
|
|
audioFileLock.lock()
|
|
isAcceptingAudio = false
|
|
audioFileLock.unlock()
|
|
|
|
if isTapActive {
|
|
processTap?.invalidate()
|
|
processTap = nil
|
|
isTapActive = false
|
|
}
|
|
cleanupAudioEngine()
|
|
micRetryCount = 0
|
|
resetAudioLevels()
|
|
|
|
audioFileLock.lock()
|
|
let micHasAudio = (micAudioFile?.length ?? 0) > 0
|
|
let systemHasAudio = (systemAudioFile?.length ?? 0) > 0
|
|
micAudioFile = nil
|
|
systemAudioFile = nil
|
|
let files: [URL?] = [micHasAudio ? micAudioURL : nil, systemHasAudio ? systemAudioURL : nil]
|
|
audioFileLock.unlock()
|
|
if !micHasAudio, let micAudioURL { try? FileManager.default.removeItem(at: micAudioURL) }
|
|
if !systemHasAudio, let systemAudioURL { try? FileManager.default.removeItem(at: systemAudioURL) }
|
|
micAudioURL = nil
|
|
systemAudioURL = nil
|
|
return files
|
|
}
|
|
|
|
private func cancelCapture(removeFiles: Bool) {
|
|
let files = stopCaptureAndCloseFiles().compactMap { $0 }
|
|
if removeFiles { removeAudioFiles(files) }
|
|
isProcessing = false
|
|
}
|
|
|
|
private func removeAudioFiles(_ urls: [URL]) {
|
|
for url in urls { try? FileManager.default.removeItem(at: url) }
|
|
}
|
|
|
|
private func preserveAudioFiles(_ urls: [URL], meetingID: UUID) -> URL? {
|
|
LocalStorageManager.shared.preserveAudioFiles(urls, for: meetingID)
|
|
}
|
|
|
|
private func preservedAudioFiles(_ urls: [URL?], in folder: URL?) -> [URL?] {
|
|
urls.map { sourceURL in
|
|
guard let sourceURL else { return nil }
|
|
guard let folder else { return sourceURL }
|
|
let preservedURL = folder.appendingPathComponent(sourceURL.lastPathComponent)
|
|
return FileManager.default.fileExists(atPath: preservedURL.path) ? preservedURL : sourceURL
|
|
}
|
|
}
|
|
|
|
private func captureDurationMismatch(in files: [URL?]) -> String? {
|
|
guard files.count >= 2,
|
|
let micDuration = audioDuration(at: files[0]),
|
|
let systemDuration = audioDuration(at: files[1]),
|
|
micDuration >= 60 else { return nil }
|
|
let ratio = systemDuration / micDuration
|
|
guard (0.45...0.55).contains(ratio) || (1.8...2.2).contains(ratio) else { return nil }
|
|
return String(format: "mic %.1fs, system %.1fs", micDuration, systemDuration)
|
|
}
|
|
|
|
private func repairHalfDurationSystemWAVIfNeeded(in files: [URL?]) -> Bool {
|
|
guard files.count >= 2,
|
|
let micDuration = audioDuration(at: files[0]),
|
|
let systemURL = files[1],
|
|
let systemDuration = audioDuration(at: systemURL),
|
|
micDuration >= 60,
|
|
systemURL.pathExtension.caseInsensitiveCompare("wav") == .orderedSame,
|
|
(0.48...0.52).contains(systemDuration / micDuration) else { return false }
|
|
do {
|
|
try halveWAVSampleRate(at: systemURL)
|
|
return true
|
|
} catch {
|
|
return false
|
|
}
|
|
}
|
|
|
|
private func halveWAVSampleRate(at url: URL) throws {
|
|
let handle = try FileHandle(forUpdating: url)
|
|
defer { try? handle.close() }
|
|
|
|
try handle.seek(toOffset: 0)
|
|
guard let riffHeader = try handle.read(upToCount: 12),
|
|
riffHeader.count == 12,
|
|
String(data: riffHeader[0..<4], encoding: .ascii) == "RIFF",
|
|
String(data: riffHeader[8..<12], encoding: .ascii) == "WAVE" else {
|
|
throw NSError(domain: "AudioManager", code: -2, userInfo: [NSLocalizedDescriptionKey: "Invalid WAV header"])
|
|
}
|
|
|
|
var offset: UInt64 = 12
|
|
while true {
|
|
try handle.seek(toOffset: offset)
|
|
guard let chunkHeader = try handle.read(upToCount: 8), chunkHeader.count == 8 else { break }
|
|
let chunkID = String(data: chunkHeader[0..<4], encoding: .ascii)
|
|
let chunkSize = UInt32(chunkHeader[4])
|
|
| (UInt32(chunkHeader[5]) << 8)
|
|
| (UInt32(chunkHeader[6]) << 16)
|
|
| (UInt32(chunkHeader[7]) << 24)
|
|
let chunkDataOffset = offset + 8
|
|
|
|
if chunkID == "fmt ", chunkSize >= 16 {
|
|
try handle.seek(toOffset: chunkDataOffset)
|
|
guard let format = try handle.read(upToCount: 16), format.count == 16 else { break }
|
|
let audioFormat = UInt16(format[0]) | (UInt16(format[1]) << 8)
|
|
let blockAlign = UInt16(format[12]) | (UInt16(format[13]) << 8)
|
|
let sampleRate = UInt32(format[4])
|
|
| (UInt32(format[5]) << 8)
|
|
| (UInt32(format[6]) << 16)
|
|
| (UInt32(format[7]) << 24)
|
|
guard audioFormat == 1, sampleRate >= 16_000, sampleRate.isMultiple(of: 2) else {
|
|
throw NSError(domain: "AudioManager", code: -3, userInfo: [NSLocalizedDescriptionKey: "Unsupported WAV format"])
|
|
}
|
|
|
|
let correctedSampleRate = sampleRate / 2
|
|
let correctedByteRate = correctedSampleRate * UInt32(blockAlign)
|
|
try handle.seek(toOffset: chunkDataOffset + 4)
|
|
try handle.write(contentsOf: littleEndianData(correctedSampleRate))
|
|
try handle.write(contentsOf: littleEndianData(correctedByteRate))
|
|
try handle.synchronize()
|
|
return
|
|
}
|
|
|
|
offset = chunkDataOffset + UInt64(chunkSize) + UInt64(chunkSize % 2)
|
|
}
|
|
|
|
throw NSError(domain: "AudioManager", code: -4, userInfo: [NSLocalizedDescriptionKey: "WAV format chunk was not found"])
|
|
}
|
|
|
|
private func littleEndianData(_ value: UInt32) -> Data {
|
|
var littleEndianValue = value.littleEndian
|
|
return withUnsafeBytes(of: &littleEndianValue) { Data($0) }
|
|
}
|
|
|
|
private func audioDuration(at url: URL?) -> TimeInterval? {
|
|
guard let url,
|
|
let file = try? AVAudioFile(forReading: url),
|
|
file.processingFormat.sampleRate > 0 else { return nil }
|
|
return Double(file.length) / file.processingFormat.sampleRate
|
|
}
|
|
|
|
private func streamDescriptionSummary(_ description: AudioStreamBasicDescription?) -> String {
|
|
guard let description else { return "unavailable" }
|
|
return String(
|
|
format: "sampleRate=%.3f,formatID=%u,flags=%u,bytesPerPacket=%u,framesPerPacket=%u,bytesPerFrame=%u,channels=%u,bitsPerChannel=%u",
|
|
description.mSampleRate,
|
|
description.mFormatID,
|
|
description.mFormatFlags,
|
|
description.mBytesPerPacket,
|
|
description.mFramesPerPacket,
|
|
description.mBytesPerFrame,
|
|
description.mChannelsPerFrame,
|
|
description.mBitsPerChannel
|
|
)
|
|
}
|
|
|
|
private func audioFormatSummary(_ format: AVAudioFormat) -> String {
|
|
"sampleRate=\(format.sampleRate),channels=\(format.channelCount),commonFormat=\(format.commonFormat.rawValue),interleaved=\(format.isInterleaved)"
|
|
}
|
|
|
|
private func audioFileSummary(_ url: URL?) -> String {
|
|
guard let url else { return "missing" }
|
|
guard let file = try? AVAudioFile(forReading: url), file.processingFormat.sampleRate > 0 else {
|
|
return "\(url.lastPathComponent):unreadable"
|
|
}
|
|
let duration = Double(file.length) / file.processingFormat.sampleRate
|
|
return String(
|
|
format: "%@:sampleRate=%.3f,channels=%u,frames=%lld,duration=%.6f",
|
|
url.lastPathComponent,
|
|
file.processingFormat.sampleRate,
|
|
file.processingFormat.channelCount,
|
|
file.length,
|
|
duration
|
|
)
|
|
}
|
|
|
|
private func writeCaptureDiagnostics(
|
|
sessionID: UUID,
|
|
captureStartedAt: Date,
|
|
preRepairFileSummaries: [String],
|
|
repairedFiles: [URL?],
|
|
repairApplied: Bool,
|
|
audioFolder: URL?
|
|
) {
|
|
guard let audioFolder else { return }
|
|
let callbackDuration = systemDiagnostics.firstCallbackAt.flatMap { first in
|
|
systemDiagnostics.lastCallbackAt.map { $0.timeIntervalSince(first) }
|
|
}
|
|
let sampleTimeDelta = systemDiagnostics.firstSampleTime.flatMap { first in
|
|
systemDiagnostics.lastSampleTime.map { $0 - first }
|
|
}
|
|
let hostTimeDelta = systemDiagnostics.firstHostTime.flatMap { first in
|
|
systemDiagnostics.lastHostTime.map { $0 >= first ? $0 - first : 0 }
|
|
}
|
|
let inputFrameDuration = systemDiagnostics.selectedInputSampleRate.flatMap { sampleRate in
|
|
sampleRate > 0 ? Double(systemDiagnostics.inputFrameCount) / sampleRate : nil
|
|
}
|
|
let outputFrameDuration = systemDiagnostics.targetSampleRate.flatMap { sampleRate in
|
|
sampleRate > 0 ? Double(systemDiagnostics.outputFrameCount) / sampleRate : nil
|
|
}
|
|
let observedInputRate = callbackDuration.flatMap { duration in
|
|
duration > 0 ? Double(systemDiagnostics.inputFrameCount) / duration : nil
|
|
}
|
|
let preRepairMic = preRepairFileSummaries.indices.contains(0) ? preRepairFileSummaries[0] : "missing"
|
|
let preRepairSystem = preRepairFileSummaries.indices.contains(1) ? preRepairFileSummaries[1] : "missing"
|
|
let postRepairMic = repairedFiles.indices.contains(0) ? audioFileSummary(repairedFiles[0]) : "missing"
|
|
let postRepairSystem = repairedFiles.indices.contains(1) ? audioFileSummary(repairedFiles[1]) : "missing"
|
|
let callbackDurationLine = callbackDuration.map { String(format: "callbackWallDuration=%.6f", $0) } ?? "callbackWallDuration=unavailable"
|
|
let inputFrameDurationLine = inputFrameDuration.map { String(format: "inputFrameDurationAtSelectedRate=%.6f", $0) } ?? "inputFrameDurationAtSelectedRate=unavailable"
|
|
let outputFrameDurationLine = outputFrameDuration.map { String(format: "outputFrameDurationAtTargetRate=%.6f", $0) } ?? "outputFrameDurationAtTargetRate=unavailable"
|
|
let observedInputRateLine = observedInputRate.map { String(format: "observedInputFramesPerSecond=%.3f", $0) } ?? "observedInputFramesPerSecond=unavailable"
|
|
let sampleTimeDeltaLine = sampleTimeDelta.map { String(format: "sampleTimeDelta=%.6f", $0) } ?? "sampleTimeDelta=unavailable"
|
|
let hostTimeDeltaLine = hostTimeDelta.map { "hostTimeDelta=\($0)" } ?? "hostTimeDelta=unavailable"
|
|
let lines: [String] = [
|
|
"Meetingnotes system capture diagnostics",
|
|
"sessionID=\(sessionID.uuidString)",
|
|
"createdAt=\(ISO8601DateFormatter().string(from: Date()))",
|
|
String(format: "captureWallDuration=%.6f", Date().timeIntervalSince(captureStartedAt)),
|
|
"tapAdvertisedFormat=\(systemDiagnostics.tapAdvertisedFormat)",
|
|
"aggregateInputFormat=\(systemDiagnostics.aggregateInputFormat)",
|
|
"selectedInputFormat=\(systemDiagnostics.selectedInputFormat)",
|
|
"inputRateInference=\(systemDiagnostics.inputRateInference)",
|
|
"targetFormat=\(systemDiagnostics.targetFormat)",
|
|
"firstBufferLayout=\(systemDiagnostics.firstBufferLayout ?? "unavailable")",
|
|
"callbackCount=\(systemDiagnostics.callbackCount)",
|
|
"inputFrameCount=\(systemDiagnostics.inputFrameCount)",
|
|
"outputFrameCount=\(systemDiagnostics.outputFrameCount)",
|
|
"discardedCallbackCount=\(systemDiagnostics.discardedCallbackCount)",
|
|
callbackDurationLine,
|
|
inputFrameDurationLine,
|
|
outputFrameDurationLine,
|
|
observedInputRateLine,
|
|
sampleTimeDeltaLine,
|
|
hostTimeDeltaLine,
|
|
"repairApplied=\(repairApplied)",
|
|
"preRepairMic=\(preRepairMic)",
|
|
"preRepairSystem=\(preRepairSystem)",
|
|
"postRepairMic=\(postRepairMic)",
|
|
"postRepairSystem=\(postRepairSystem)"
|
|
]
|
|
let diagnosticsURL = audioFolder.appendingPathComponent("audio-diagnostics-\(sessionID.uuidString).txt")
|
|
try? lines.joined(separator: "\n").appending("\n").write(
|
|
to: diagnosticsURL,
|
|
atomically: true,
|
|
encoding: .utf8
|
|
)
|
|
}
|
|
|
|
private func resetAudioLevels() {
|
|
micAudioLevel = 0
|
|
systemAudioLevel = 0
|
|
AudioLevelManager.shared.updateMicLevel(0)
|
|
AudioLevelManager.shared.updateSystemLevel(0)
|
|
}
|
|
|
|
private func hasActiveAudioFiles() -> Bool {
|
|
audioFileLock.lock()
|
|
defer { audioFileLock.unlock() }
|
|
return isAcceptingAudio
|
|
}
|
|
|
|
private func handleAudioEngineConfigurationChange() {
|
|
restartMicrophone()
|
|
}
|
|
}
|