fix: make post-meeting transcription reliable (#6)

Co-authored-by: superdooper86 <[email protected]>

Meetingnotes.xcodeproj/project.pbxproj

@@ -276,7 +276,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
 				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 13;
+				CURRENT_PROJECT_VERSION = 14;
 				DEVELOPMENT_ASSET_PATHS = "\"meetingnotes/Preview Content\"";
 				DEVELOPMENT_TEAM = ML6HYR5LUR;
 				ENABLE_HARDENED_RUNTIME = YES;
@@ -290,7 +290,7 @@
 					"@executable_path/../Frameworks",
 				);
 				MACOSX_DEPLOYMENT_TARGET = 15.0;
-				MARKETING_VERSION = 1.1.1;
+				MARKETING_VERSION = 1.1.2;
 				ONLY_ACTIVE_ARCH = NO;
 				OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI";
 				PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes;
@@ -312,7 +312,7 @@
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
 				COMBINE_HIDPI_IMAGES = YES;
-				CURRENT_PROJECT_VERSION = 13;
+				CURRENT_PROJECT_VERSION = 14;
 				DEVELOPMENT_ASSET_PATHS = "\"meetingnotes/Preview Content\"";
 				DEVELOPMENT_TEAM = ML6HYR5LUR;
 				ENABLE_HARDENED_RUNTIME = YES;
@@ -326,7 +326,7 @@
 					"@executable_path/../Frameworks",
 				);
 				MACOSX_DEPLOYMENT_TARGET = 15.0;
-				MARKETING_VERSION = 1.1.1;
+				MARKETING_VERSION = 1.1.2;
 				ONLY_ACTIVE_ARCH = YES;
 				OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI";
 				PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes;

meetingnotes/Managers/AudioManager.swift

@@ -21,9 +21,12 @@ final class AudioManager: NSObject, ObservableObject {
     private let audioProcessController = AudioProcessController()
     private let permission = AudioRecordingPermission()
     private let tapQueue = DispatchQueue(label: "io.meetingnotes.audiotap", qos: .userInitiated)
+    private let audioFileLock = NSLock()
     private var isTapActive = false
     private var isRestartingSystemTap = false
+    private var isAcceptingAudio = false
     private var micRetryCount = 0
+    private var pendingMicRestart: DispatchWorkItem?
     private let maxMicRetries = 3
     private var cancellables = Set<AnyCancellable>()
     
@@ -31,6 +34,7 @@ final class AudioManager: NSObject, ObservableObject {
     private var systemAudioFile: AVAudioFile?
     private var micAudioURL: URL?
     private var systemAudioURL: URL?
+    private var recordingStartedAt = Date()
 
     private override init() {
         super.init()
@@ -50,9 +54,10 @@ final class AudioManager: NSObject, ObservableObject {
     }
 
     func startRecording() {
-        sessionID = UUID()
         errorMessage = nil
         cancelCapture(removeFiles: true)
+        sessionID = UUID()
+        recordingStartedAt = Date()
         do {
             try prepareAudioFiles()
             startMicrophoneTap()
@@ -64,11 +69,12 @@ final class AudioManager: NSObject, ObservableObject {
     }
 
     func stopRecordingAndTranscribe() async -> [TranscriptChunk] {
+        let completedSessionID = sessionID
+        let captureStartedAt = recordingStartedAt
         let files = stopCaptureAndCloseFiles()
         isProcessing = true
         defer {
             isProcessing = false
-            removeAudioFiles(files.compactMap { $0 })
         }
 
         let model = UserDefaultsManager.shared.transcriptionModel
@@ -82,18 +88,42 @@ final class AudioManager: NSObject, ObservableObject {
         for (source, result) in zip([AudioSource.mic, .system], results) {
             guard let result else { continue }
             switch result {
-            case .success(let text):
-                let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
-                if !trimmed.isEmpty {
-                    updated.append(TranscriptChunk(source: source, text: trimmed, isFinal: true))
+            case .success(let transcription):
+                if transcription.segments.isEmpty {
+                    let text = transcription.text.trimmingCharacters(in: .whitespacesAndNewlines)
+                    if !text.isEmpty {
+                        updated.append(TranscriptChunk(timestamp: captureStartedAt, source: source, text: text, isFinal: true))
+                    }
+                } else {
+                    for segment in transcription.segments {
+                        let text = segment.text.trimmingCharacters(in: .whitespacesAndNewlines)
+                        guard !text.isEmpty else { continue }
+                        updated.append(TranscriptChunk(
+                            timestamp: captureStartedAt.addingTimeInterval(max(0, segment.start)),
+                            source: source,
+                            text: text,
+                            isFinal: true
+                        ))
+                    }
                 }
             case .failure(let error):
                 failures.append("\(source.displayName): \(error.localizedDescription)")
             }
         }
+        updated.sort {
+            if $0.timestamp != $1.timestamp { return $0.timestamp < $1.timestamp }
+            return $0.source.rawValue < $1.source.rawValue
+        }
         transcriptChunks = updated
-        if !failures.isEmpty {
-            errorMessage = "Transcription failed for " + failures.joined(separator: "; ")
+        let completedFiles = files.compactMap { $0 }
+        if failures.isEmpty {
+            removeAudioFiles(completedFiles)
+        } else {
+            let recoveryFolder = preserveAudioFiles(completedFiles, sessionID: completedSessionID)
+            let recoveryMessage = recoveryFolder == nil
+                ? " The audio remains in the app's temporary folder."
+                : " Audio was saved in Documents/Meetingnotes-Recovery/\(completedSessionID.uuidString)."
+            errorMessage = "Transcription failed for " + failures.joined(separator: "; ") + recoveryMessage
         }
         return updated
     }
@@ -102,7 +132,7 @@ final class AudioManager: NSObject, ObservableObject {
         cancelCapture(removeFiles: true)
     }
         
-    private func transcribe(_ fileURL: URL?, model: String) async -> Result<String, Error>? {
+    private func transcribe(_ fileURL: URL?, model: String) async -> Result<CoderAPIClient.Transcription, Error>? {
         guard let fileURL else { return nil }
         do {
             return .success(try await CoderAPIClient.shared.transcribe(fileURL: fileURL, model: model))
@@ -122,20 +152,26 @@ final class AudioManager: NSObject, ObservableObject {
         let id = sessionID.uuidString
         let micURL = base.appendingPathComponent("meetingnotes-\(id)-mic.m4a")
         let systemURL = base.appendingPathComponent("meetingnotes-\(id)-system.m4a")
-        micAudioFile = try AVAudioFile(
+        let newMicAudioFile = try AVAudioFile(
             forWriting: micURL,
             settings: settings,
             commonFormat: .pcmFormatFloat32,
             interleaved: false
         )
-        systemAudioFile = try AVAudioFile(
+        let newSystemAudioFile = try AVAudioFile(
             forWriting: systemURL,
             settings: settings,
             commonFormat: .pcmFormatFloat32,
             interleaved: false
         )
+
+        audioFileLock.lock()
+        micAudioFile = newMicAudioFile
+        systemAudioFile = newSystemAudioFile
         micAudioURL = micURL
         systemAudioURL = systemURL
+        isAcceptingAudio = true
+        audioFileLock.unlock()
     }
 
     private func startMicrophoneTap() {
@@ -161,12 +197,17 @@ final class AudioManager: NSObject, ObservableObject {
     }
     
     private func restartMicrophone() {
-        guard (isRecording || micAudioFile != nil), micRetryCount < maxMicRetries else { return }
+        guard hasActiveAudioFiles(), micRetryCount < maxMicRetries else { return }
         micRetryCount += 1
+        pendingMicRestart?.cancel()
         cleanupAudioEngine()
-        DispatchQueue.main.asyncAfter(deadline: .now() + 1) { [weak self] in
-            self?.startMicrophoneTap()
+
+        let restart = DispatchWorkItem { [weak self] in
+            guard let self, self.hasActiveAudioFiles() else { return }
+            self.startMicrophoneTap()
         }
+        pendingMicRestart = restart
+        DispatchQueue.main.asyncAfter(deadline: .now() + 1, execute: restart)
     }
         
     private func cleanupAudioEngine() {
@@ -294,6 +335,12 @@ final class AudioManager: NSObject, ObservableObject {
             return inputBuffer
         }
         guard status != .error, conversionError == nil, outputBuffer.frameLength > 0 else { return }
+
+        audioFileLock.lock()
+        defer { audioFileLock.unlock() }
+        guard isAcceptingAudio else {
+            return
+        }
         do {
             switch source {
             case .mic:
@@ -327,7 +374,16 @@ final class AudioManager: NSObject, ObservableObject {
     
     private func stopCaptureAndCloseFiles() -> [URL?] {
         isRecording = false
+        pendingMicRestart?.cancel()
+        pendingMicRestart = nil
         AudioLevelManager.shared.updateRecordingState(false)
+
+        // Stop new writes and wait for any callback already writing before
+        // AVAudioFile is finalized and released.
+        audioFileLock.lock()
+        isAcceptingAudio = false
+        audioFileLock.unlock()
+
         if isTapActive {
             processTap?.invalidate()
             processTap = nil
@@ -337,11 +393,13 @@ final class AudioManager: NSObject, ObservableObject {
         micRetryCount = 0
         resetAudioLevels()
 
+        audioFileLock.lock()
         let micHasAudio = (micAudioFile?.length ?? 0) > 0
         let systemHasAudio = (systemAudioFile?.length ?? 0) > 0
         micAudioFile = nil
         systemAudioFile = nil
         let files: [URL?] = [micHasAudio ? micAudioURL : nil, systemHasAudio ? systemAudioURL : nil]
+        audioFileLock.unlock()
         if !micHasAudio, let micAudioURL { try? FileManager.default.removeItem(at: micAudioURL) }
         if !systemHasAudio, let systemAudioURL { try? FileManager.default.removeItem(at: systemAudioURL) }
         micAudioURL = nil
@@ -358,6 +416,31 @@ final class AudioManager: NSObject, ObservableObject {
     private func removeAudioFiles(_ urls: [URL]) {
         for url in urls { try? FileManager.default.removeItem(at: url) }
     }
+
+    private func preserveAudioFiles(_ urls: [URL], sessionID: UUID) -> URL? {
+        guard !urls.isEmpty,
+              let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else {
+            return nil
+        }
+        let folder = documents
+            .appendingPathComponent("Meetingnotes-Recovery", isDirectory: true)
+            .appendingPathComponent(sessionID.uuidString, isDirectory: true)
+        do {
+            try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true)
+        } catch {
+            return nil
+        }
+        var preservedCount = 0
+        for url in urls {
+            do {
+                try FileManager.default.moveItem(at: url, to: folder.appendingPathComponent(url.lastPathComponent))
+                preservedCount += 1
+            } catch {
+                continue
+            }
+        }
+        return preservedCount > 0 ? folder : nil
+    }
         
     private func resetAudioLevels() {
         micAudioLevel = 0
@@ -365,6 +448,12 @@ final class AudioManager: NSObject, ObservableObject {
         AudioLevelManager.shared.updateMicLevel(0)
         AudioLevelManager.shared.updateSystemLevel(0)
     }
+
+    private func hasActiveAudioFiles() -> Bool {
+        audioFileLock.lock()
+        defer { audioFileLock.unlock() }
+        return isAcceptingAudio
+    }
     
     private func handleAudioEngineConfigurationChange() {
         restartMicrophone()

meetingnotes/Models/Meeting.swift

@@ -1,5 +1,17 @@
 import Foundation
 
+enum TranscriptTimestampFormatter {
+    static let formatter: DateFormatter = {
+        let formatter = DateFormatter()
+        formatter.dateFormat = "HH:mm:ss"
+        return formatter
+    }()
+
+    static func string(from date: Date) -> String {
+        formatter.string(from: date)
+    }
+}
+
 enum AudioSource: String, Codable, CaseIterable {
     case mic = "MIC"
     case system = "SYS"
@@ -107,81 +119,22 @@ struct Meeting: Codable, Identifiable, Hashable {
     // Formatted transcript for copying with collapsed sequential chunks
     var formattedTranscript: String {
         let finalChunks = transcriptChunks.filter { $0.isFinal }
-        
-        guard !finalChunks.isEmpty else { return "" }
-        
-        var result: [String] = []
-        var currentSource: AudioSource?
-        var currentTexts: [String] = []
-        
-        for chunk in finalChunks {
-            if chunk.source != currentSource {
-                // Finish previous section if exists
-                if let source = currentSource, !currentTexts.isEmpty {
-                    let combinedText = currentTexts.joined(separator: " ")
-                    result.append("\(source.copyPrefix): \(combinedText)")
-                }
-                
-                // Start new section
-                currentSource = chunk.source
-                currentTexts = [chunk.text]
-            } else {
-                // Same source, add to current section
-                currentTexts.append(chunk.text)
-            }
-        }
-        
-        // Finish last section
-        if let source = currentSource, !currentTexts.isEmpty {
-            let combinedText = currentTexts.joined(separator: " ")
-            result.append("\(source.copyPrefix): \(combinedText)")
-        }
-        
-        return result.joined(separator: "  \n")
+
+        return finalChunks.map { chunk in
+            "[\(TranscriptTimestampFormatter.string(from: chunk.timestamp))] \(chunk.source.copyPrefix): \(chunk.text)"
+        }.joined(separator: "\n")
     }
     
     // Collapsed chunks for UI display
     var collapsedTranscriptChunks: [CollapsedTranscriptChunk] {
-        guard !transcriptChunks.isEmpty else { return [] }
-        
-        var result: [CollapsedTranscriptChunk] = []
-        var currentSource: AudioSource?
-        var currentTexts: [String] = []
-        var currentTimestamp: Date?
-        
-        for chunk in transcriptChunks {
-            if chunk.source != currentSource {
-                // Finish previous section if exists
-                if let source = currentSource, !currentTexts.isEmpty, let timestamp = currentTimestamp {
-                    let combinedText = currentTexts.joined(separator: " ")
-                    result.append(CollapsedTranscriptChunk(
-                        timestamp: timestamp,
-                        source: source,
-                        combinedText: combinedText
-                    ))
-                }
-                
-                // Start new section
-                currentSource = chunk.source
-                currentTexts = [chunk.text]
-                currentTimestamp = chunk.timestamp
-            } else {
-                // Same source, add to current section
-                currentTexts.append(chunk.text)
-            }
+        transcriptChunks.filter(\.isFinal).map { chunk in
+            CollapsedTranscriptChunk(
+                id: chunk.id,
+                timestamp: chunk.timestamp,
+                source: chunk.source,
+                combinedText: chunk.text
+            )
         }
-        
-        // Finish last section
-        if let source = currentSource, !currentTexts.isEmpty, let timestamp = currentTimestamp {
-            let combinedText = currentTexts.joined(separator: " ")
-            result.append(CollapsedTranscriptChunk(
-                timestamp: timestamp,
-                source: source,
-                combinedText: combinedText
-            ))
-        }
-        
-        return result
     }
     
     // Separate computed properties for mic and system transcripts
@@ -198,4 +151,4 @@ struct Meeting: Codable, Identifiable, Hashable {
             .map { $0.text }
             .joined(separator: " ")
     }
-} 
+}

meetingnotes/ProcessTap/ProcessTap.swift

@@ -141,7 +141,7 @@ final class ProcessTap {
             if processObjectIDs.isEmpty {
                 logger.warning("System audio tap configured with an empty list of processObjectIDs. This might not capture any audio or behave unexpectedly.")
             }
-            tapDescription = CATapDescription(stereoMixdownOfProcesses: processObjectIDs)
+            tapDescription = CATapDescription(monoMixdownOfProcesses: processObjectIDs)
             logger.debug("Configuring tap for system audio output using \(processObjectIDs.count) explicit processes.")
         }
         
@@ -496,4 +496,4 @@ final class ProcessTapRecorder {
             self.currentAudioLevel = 0.0
         }
     }
-} 
+}

meetingnotes/Services/CoderAPIClient.swift

@@ -48,6 +48,17 @@ enum CoderAPIError: LocalizedError {
 final class CoderAPIClient {
     static let shared = CoderAPIClient()
 
+    struct Transcription {
+        struct Segment: Decodable {
+            let start: TimeInterval
+            let end: TimeInterval
+            let text: String
+        }
+
+        let text: String
+        let segments: [Segment]
+    }
+
     private struct ModelsResponse: Decodable {
         let data: [CoderModel]
     }
@@ -59,9 +70,17 @@ final class CoderAPIClient {
 
     private struct TranscriptionResponse: Decodable {
         let text: String
+        let segments: [Transcription.Segment]?
     }
 
-    private init() {}
+    private let transcriptionSession: URLSession
+
+    private init() {
+        let configuration = URLSessionConfiguration.default
+        configuration.timeoutIntervalForRequest = 2 * 60 * 60
+        configuration.timeoutIntervalForResource = 2 * 60 * 60
+        transcriptionSession = URLSession(configuration: configuration)
+    }
 
     func models(baseURL: String, apiKey: String) async throws -> [CoderModel] {
         var request = URLRequest(url: try endpoint(baseURL: baseURL, path: "models"))
@@ -128,7 +147,7 @@ final class CoderAPIClient {
         }
     }
 
-    func transcribe(fileURL: URL, model: String, language: String = "en") async throws -> String {
+    func transcribe(fileURL: URL, model: String, language: String = "en") async throws -> Transcription {
         let selectedModel = model.trimmingCharacters(in: .whitespacesAndNewlines)
         guard !selectedModel.isEmpty else { throw CoderAPIError.missingModel("transcription") }
         let apiKey = try requiredAPIKey(KeychainHelper.shared.getCoderAPIKey() ?? "")
@@ -149,9 +168,10 @@ final class CoderAPIClient {
            let size = attributes[.size] as? NSNumber {
             request.setValue(size.stringValue, forHTTPHeaderField: "Content-Length")
         }
-        let (data, response) = try await URLSession.shared.upload(for: request, fromFile: bodyURL)
+        let (data, response) = try await transcriptionSession.upload(for: request, fromFile: bodyURL)
         try validate(response: response, data: data)
-        return try JSONDecoder().decode(TranscriptionResponse.self, from: data).text
+        let decoded = try JSONDecoder().decode(TranscriptionResponse.self, from: data)
+        return Transcription(text: decoded.text, segments: decoded.segments ?? [])
     }
 
     private func endpoint(baseURL: String, path: String) throws -> URL {
@@ -194,6 +214,7 @@ final class CoderAPIClient {
         }
         try write("--\(boundary)\r\nContent-Disposition: form-data; name=\"model\"\r\n\r\n\(model)\r\n")
         try write("--\(boundary)\r\nContent-Disposition: form-data; name=\"language\"\r\n\r\n\(language)\r\n")
+        try write("--\(boundary)\r\nContent-Disposition: form-data; name=\"response_format\"\r\n\r\nverbose_json\r\n")
         try write("--\(boundary)\r\nContent-Disposition: form-data; name=\"file\"; filename=\"\(audioURL.lastPathComponent)\"\r\nContent-Type: audio/mp4\r\n\r\n")
         let input = try FileHandle(forReadingFrom: audioURL)
         defer { try? input.close() }

meetingnotes/Views/MeetingListView.swift

@@ -197,6 +197,11 @@ struct CollapsedTranscriptChunkView: View {
     
     var body: some View {
         HStack(alignment: .firstTextBaseline, spacing: 8) {
+            Text(TranscriptTimestampFormatter.string(from: chunk.timestamp))
+                .font(.caption.monospacedDigit())
+                .foregroundColor(.secondary)
+                .frame(width: 58, alignment: .leading)
+
             // Source indicator
             HStack(spacing: 4) {
                 Image(systemName: chunk.source.icon)