From 417d751d7af530050bab3b1c9dd68983af3eb9d2 Mon Sep 17 00:00:00 2001 From: Owen Gretzinger Date: Thu, 10 Jul 2025 13:09:32 -0400 Subject: [PATCH] Basic recording functionality --- .cursor/rules/build-but-dont-run.mdc | 7 + .cursor/rules/overall-objective.mdc | 20 ++ notetaker.xcodeproj/project.pbxproj | 55 +++- .../xcshareddata/swiftpm/Package.resolved | 33 ++ notetaker/AudioManager.swift | 196 ++++++++++++ notetaker/ContentView.swift | 76 ++++- notetaker/Info.plist | 8 + notetaker/KeychainHelper.swift | 42 +++ notetaker/SimpleAudioManager.swift | 298 ++++++++++++++++++ notetaker/notetaker.entitlements | 10 + 10 files changed, 738 insertions(+), 7 deletions(-) create mode 100644 .cursor/rules/build-but-dont-run.mdc create mode 100644 .cursor/rules/overall-objective.mdc create mode 100644 notetaker.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved create mode 100644 notetaker/AudioManager.swift create mode 100644 notetaker/Info.plist create mode 100644 notetaker/KeychainHelper.swift create mode 100644 notetaker/SimpleAudioManager.swift diff --git a/.cursor/rules/build-but-dont-run.mdc b/.cursor/rules/build-but-dont-run.mdc new file mode 100644 index 0000000..2e92747 --- /dev/null +++ b/.cursor/rules/build-but-dont-run.mdc @@ -0,0 +1,7 @@ +--- +alwaysApply: true +--- + +After making changes, ensure there are no errors by creating an Xcode build. + +However, don't run the app, I'll do that myself. \ No newline at end of file diff --git a/.cursor/rules/overall-objective.mdc b/.cursor/rules/overall-objective.mdc new file mode 100644 index 0000000..5b0413a --- /dev/null +++ b/.cursor/rules/overall-objective.mdc @@ -0,0 +1,20 @@ +--- +alwaysApply: true +--- +I want to build an open source version of Granola + +It will be a meeting notetaking macOS application + +Basically it will use your mic and system audio, create a live transcript, and then at the end of the meeting use the transcript + any notes you write in a notepad to generate the meeting notes, with a button to quickly copy the notes or transcript. Can also edit the resulting notes + +The user will provide their own deepgram and openai API keys (stored locally) for transcription and AI generation + +The user will also be able to modify the system prompt in addition to just being able to write a blurb about themself (that gets injected into the system prompt) + +All data will be stored locally on the device + +That will be great for the MVP to start. Later on I will add additional features such as: +- connecting to your Google calendar +- note templates +- AI chat for asking questions about a meeting +- Integrations for email, slack, etc. \ No newline at end of file diff --git a/notetaker.xcodeproj/project.pbxproj b/notetaker.xcodeproj/project.pbxproj index cff8dbd..ae4afbd 100644 --- a/notetaker.xcodeproj/project.pbxproj +++ b/notetaker.xcodeproj/project.pbxproj @@ -6,13 +6,30 @@ objectVersion = 77; objects = { +/* Begin PBXBuildFile section */ + CC5718382E2020490042BA88 /* OpenAI in Frameworks */ = {isa = PBXBuildFile; productRef = CC5718372E2020490042BA88 /* OpenAI */; }; +/* End PBXBuildFile section */ + /* Begin PBXFileReference section */ CC5718182E201D8D0042BA88 /* notetaker.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = notetaker.app; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ +/* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */ + CC5718352E201EAA0042BA88 /* Exceptions for "notetaker" folder in "notetaker" target */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + Info.plist, + ); + target = CC5718172E201D8D0042BA88 /* notetaker */; + }; +/* End PBXFileSystemSynchronizedBuildFileExceptionSet section */ + /* Begin PBXFileSystemSynchronizedRootGroup section */ CC57181A2E201D8D0042BA88 /* notetaker */ = { isa = PBXFileSystemSynchronizedRootGroup; + exceptions = ( + CC5718352E201EAA0042BA88 /* Exceptions for "notetaker" folder in "notetaker" target */, + ); path = notetaker; sourceTree = ""; }; @@ -23,6 +40,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + CC5718382E2020490042BA88 /* OpenAI in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -65,6 +83,7 @@ ); name = notetaker; packageProductDependencies = ( + CC5718372E2020490042BA88 /* OpenAI */, ); productName = notetaker; productReference = CC5718182E201D8D0042BA88 /* notetaker.app */; @@ -94,6 +113,9 @@ ); mainGroup = CC57180F2E201D8D0042BA88; minimizedProjectReferenceProxies = 1; + packageReferences = ( + CC5718362E2020490042BA88 /* XCRemoteSwiftPackageReference "OpenAI" */, + ); preferredProjectObjectVersion = 77; productRefGroup = CC5718192E201D8D0042BA88 /* Products */; projectDirPath = ""; @@ -178,7 +200,7 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; ONLY_ACTIVE_ARCH = YES; @@ -235,7 +257,7 @@ GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MACOSX_DEPLOYMENT_TARGET = 15.0; + MACOSX_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; SDKROOT = macosx; @@ -249,17 +271,22 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_ENTITLEMENTS = notetaker/notetaker.entitlements; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"notetaker/Preview Content\""; + DEVELOPMENT_TEAM = ML6HYR5LUR; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = notetaker/Info.plist; INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INFOPLIST_KEY_NSMicrophoneUsageDescription = "Notetaker needs access to your microphone for transcription."; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", ); + MACOSX_DEPLOYMENT_TARGET = 15.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = owen.notetaker; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -274,17 +301,22 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor; CODE_SIGN_ENTITLEMENTS = notetaker/notetaker.entitlements; + "CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development"; CODE_SIGN_STYLE = Automatic; COMBINE_HIDPI_IMAGES = YES; CURRENT_PROJECT_VERSION = 1; DEVELOPMENT_ASSET_PATHS = "\"notetaker/Preview Content\""; + DEVELOPMENT_TEAM = ML6HYR5LUR; ENABLE_PREVIEWS = YES; GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = notetaker/Info.plist; INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INFOPLIST_KEY_NSMicrophoneUsageDescription = "Notetaker needs access to your microphone for transcription."; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/../Frameworks", ); + MACOSX_DEPLOYMENT_TARGET = 15.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = owen.notetaker; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -315,6 +347,25 @@ defaultConfigurationName = Release; }; /* End XCConfigurationList section */ + +/* Begin XCRemoteSwiftPackageReference section */ + CC5718362E2020490042BA88 /* XCRemoteSwiftPackageReference "OpenAI" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/MacPaw/OpenAI"; + requirement = { + kind = upToNextMajorVersion; + minimumVersion = 0.4.4; + }; + }; +/* End XCRemoteSwiftPackageReference section */ + +/* Begin XCSwiftPackageProductDependency section */ + CC5718372E2020490042BA88 /* OpenAI */ = { + isa = XCSwiftPackageProductDependency; + package = CC5718362E2020490042BA88 /* XCRemoteSwiftPackageReference "OpenAI" */; + productName = OpenAI; + }; +/* End XCSwiftPackageProductDependency section */ }; rootObject = CC5718102E201D8D0042BA88 /* Project object */; } diff --git a/notetaker.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/notetaker.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved new file mode 100644 index 0000000..a5cd505 --- /dev/null +++ b/notetaker.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -0,0 +1,33 @@ +{ + "originHash" : "c351aabe13067917247f7926404d096603b95663248b51124280feb61a2edf30", + "pins" : [ + { + "identity" : "openai", + "kind" : "remoteSourceControl", + "location" : "https://github.com/MacPaw/OpenAI", + "state" : { + "revision" : "cedec2fc80aedafa332f7395b7004f79f547c794", + "version" : "0.4.4" + } + }, + { + "identity" : "swift-http-types", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-http-types", + "state" : { + "revision" : "a0a57e949a8903563aba4615869310c0ebf14c03", + "version" : "1.4.0" + } + }, + { + "identity" : "swift-openapi-runtime", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-openapi-runtime", + "state" : { + "revision" : "8f33cc5dfe81169fb167da73584b9c72c3e8bc23", + "version" : "1.8.2" + } + } + ], + "version" : 3 +} diff --git a/notetaker/AudioManager.swift b/notetaker/AudioManager.swift new file mode 100644 index 0000000..2e3cf57 --- /dev/null +++ b/notetaker/AudioManager.swift @@ -0,0 +1,196 @@ +// AudioManager.swift +import AVFoundation +import Foundation +import SwiftUI +import ScreenCaptureKit + +class AudioManager: NSObject, ObservableObject { + @Published var transcript = "" + @Published var isRecording = false + + private var stream: SCStream? + private var webSocketTask: URLSessionWebSocketTask? + private let deepgramURL = URL(string: "wss://api.deepgram.com/v1/listen?encoding=linear16&sample_rate=48000&channels=1&interim_results=true")! + + func startRecording() { + // Check if ScreenCaptureKit is available + guard #available(macOS 13.0, *) else { + print("ScreenCaptureKit requires macOS 13.0 or later") + return + } + + Task { + do { + // Request permission for screen recording (includes system audio) + let canRecord = await requestScreenRecordingPermission() + if !canRecord { + print("Screen recording permission denied") + DispatchQueue.main.async { + self.isRecording = false + } + return + } + + // Set up screen capture configuration + let config = SCStreamConfiguration() + config.capturesAudio = true + config.captureMicrophone = true + config.sampleRate = 48000 + config.channelCount = 1 + + // Create content filter for desktop audio capture + let availableContent = try await SCShareableContent.current + guard let display = availableContent.displays.first else { + print("No displays available for capture") + DispatchQueue.main.async { + self.isRecording = false + } + return + } + let filter = SCContentFilter(display: display, excludingWindows: []) + + // Create stream + stream = SCStream(filter: filter, configuration: config, delegate: self) + + // Add audio output + try stream?.addStreamOutput(self, type: .audio, sampleHandlerQueue: DispatchQueue(label: "AudioQueue")) + try stream?.addStreamOutput(self, type: .microphone, sampleHandlerQueue: DispatchQueue(label: "MicQueue")) + + // Start capture + try await stream?.startCapture() + + DispatchQueue.main.async { + self.isRecording = true + } + + // Connect to Deepgram + connectToDeepgram() + + } catch { + print("Failed to start recording: \(error)") + DispatchQueue.main.async { + self.isRecording = false + } + } + } + } + + func stopRecording() { + Task { + do { + try await stream?.stopCapture() + stream = nil + webSocketTask?.cancel(with: .normalClosure, reason: nil) + webSocketTask = nil + + DispatchQueue.main.async { + self.isRecording = false + } + } catch { + print("Failed to stop recording: \(error)") + } + } + } + + private func requestScreenRecordingPermission() async -> Bool { + // ScreenCaptureKit handles permission requests automatically + // This will show a system dialog if needed + do { + let _ = try await SCShareableContent.current + return true + } catch { + return false + } + } + + private func connectToDeepgram() { + let session = URLSession(configuration: .default) + var request = URLRequest(url: deepgramURL) + if let key = KeychainHelper.shared.get(forKey: "deepgramKey"), !key.isEmpty { + request.addValue("Token \(key)", forHTTPHeaderField: "Authorization") + } else { + print("No Deepgram key found") + return + } + webSocketTask = session.webSocketTask(with: request) + webSocketTask?.resume() + receiveMessage() + } + + private func sendAudioData(_ data: Data) { + webSocketTask?.send(.data(data)) { error in + if let error = error { + print("Send error: \(error)") + } + } + } + + private func receiveMessage() { + webSocketTask?.receive { result in + switch result { + case .success(let message): + switch message { + case .string(let text): + self.parseTranscription(text) + case .data: + break + @unknown default: + break + } + self.receiveMessage() // Continue receiving + case .failure(let error): + print("Receive error: \(error)") + } + } + } + + private func parseTranscription(_ text: String) { + guard let data = text.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let type = json["type"] as? String, type == "Results", + let channel = json["channel"] as? [String: Any], + let alternatives = channel["alternatives"] as? [[String: Any]], + let alt = alternatives.first, + let transcript = alt["transcript"] as? String, + !transcript.isEmpty else { return } + + DispatchQueue.main.async { + if let isFinal = json["is_final"] as? Bool, isFinal { + self.transcript += transcript + " " + } + } + } +} + +// MARK: - SCStreamDelegate +extension AudioManager: SCStreamDelegate { + func stream(_ stream: SCStream, didStopWithError error: Error) { + print("Stream stopped with error: \(error)") + DispatchQueue.main.async { + self.isRecording = false + } + } +} + +// MARK: - SCStreamOutput +extension AudioManager: SCStreamOutput { + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + // Convert sample buffer to audio data and send to Deepgram + guard let audioBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else { + print("Failed to get data buffer from sample buffer") + return + } + + var length: Int = 0 + var dataPointer: UnsafeMutablePointer? + let status = CMBlockBufferGetDataPointer(audioBuffer, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &length, dataPointerOut: &dataPointer) + + guard status == noErr, let pointer = dataPointer, length > 0 else { + print("Failed to get audio data pointer, status: \(status), length: \(length)") + return + } + + let data = Data(bytes: pointer, count: length) + sendAudioData(data) + } +} \ No newline at end of file diff --git a/notetaker/ContentView.swift b/notetaker/ContentView.swift index a342a1f..c505f9b 100644 --- a/notetaker/ContentView.swift +++ b/notetaker/ContentView.swift @@ -8,14 +8,80 @@ import SwiftUI struct ContentView: View { + @StateObject private var audioManager = SimpleAudioManager() + @State private var deepgramKey = "" + @State private var openAIKey = "" + @State private var userBlurb = "" + @State private var systemPrompt = "Generate meeting notes from transcript and notes." + var body: some View { - VStack { - Image(systemName: "globe") - .imageScale(.large) - .foregroundStyle(.tint) - Text("Hello, world!") + VStack(spacing: 20) { + Text("Notetaker Settings") + .font(.title) + + TextField("Deepgram API Key", text: $deepgramKey) + TextField("OpenAI API Key", text: $openAIKey) + TextField("About Yourself (for prompt)", text: $userBlurb) + TextEditor(text: $systemPrompt) + .frame(height: 100) + + Button("Save") { + saveSettings() + } + + Divider() + + Toggle("Capture System Audio", isOn: $audioManager.captureSystemAudio) + .disabled(audioManager.isRecording) + + HStack { + Button("Start Recording") { + audioManager.startRecording() + } + .disabled(audioManager.isRecording) + + Button("Stop Recording") { + audioManager.stopRecording() + } + .disabled(!audioManager.isRecording) + } + + if audioManager.isRecording { + Text(audioManager.captureSystemAudio ? "🔴 Recording microphone + system audio..." : "🔴 Recording microphone...") + .foregroundColor(.red) + } + + Text("Live Transcript:") + .font(.headline) + + ScrollView { + Text(audioManager.transcript.isEmpty ? "No transcript yet..." : audioManager.transcript) + .frame(maxWidth: .infinity, alignment: .leading) + .padding() + } + .frame(height: 150) + .border(Color.gray) } .padding() + .frame(minWidth: 500, minHeight: 600) + .onAppear { + loadSettings() + } + } + + func saveSettings() { + _ = KeychainHelper.shared.save(deepgramKey, forKey: "deepgramKey") + _ = KeychainHelper.shared.save(openAIKey, forKey: "openAIKey") + _ = KeychainHelper.shared.save(userBlurb, forKey: "userBlurb") + _ = KeychainHelper.shared.save(systemPrompt, forKey: "systemPrompt") + print("Settings saved!") + } + + private func loadSettings() { + deepgramKey = KeychainHelper.shared.get(forKey: "deepgramKey") ?? "" + openAIKey = KeychainHelper.shared.get(forKey: "openAIKey") ?? "" + userBlurb = KeychainHelper.shared.get(forKey: "userBlurb") ?? "" + systemPrompt = KeychainHelper.shared.get(forKey: "systemPrompt") ?? "Generate meeting notes from transcript and notes." } } diff --git a/notetaker/Info.plist b/notetaker/Info.plist new file mode 100644 index 0000000..86e8d89 --- /dev/null +++ b/notetaker/Info.plist @@ -0,0 +1,8 @@ + + + + + NSAudioCaptureUsageDescription + Notetaker needs access to capture system audio for transcription. + + diff --git a/notetaker/KeychainHelper.swift b/notetaker/KeychainHelper.swift new file mode 100644 index 0000000..33478dd --- /dev/null +++ b/notetaker/KeychainHelper.swift @@ -0,0 +1,42 @@ +// KeychainHelper.swift +// Helper for secure storage + +import Foundation +import Security + +class KeychainHelper { + static let shared = KeychainHelper() + + private init() {} + + func save(_ value: String, forKey key: String) -> Bool { + guard let data = value.data(using: .utf8) else { return false } + + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrAccount as String: key, + kSecValueData as String: data, + kSecAttrService as String: "owen.notetaker" // Replace with your bundle ID + ] + + SecItemDelete(query as CFDictionary) // Delete if exists + let status = SecItemAdd(query as CFDictionary, nil) + return status == errSecSuccess + } + + func get(forKey key: String) -> String? { + let query: [String: Any] = [ + kSecClass as String: kSecClassGenericPassword, + kSecAttrAccount as String: key, + kSecAttrService as String: "owen.notetaker", + kSecMatchLimit as String: kSecMatchLimitOne, + kSecReturnData as String: true + ] + + var item: AnyObject? + let status = SecItemCopyMatching(query as CFDictionary, &item) + + guard status == errSecSuccess, let data = item as? Data else { return nil } + return String(data: data, encoding: .utf8) + } +} \ No newline at end of file diff --git a/notetaker/SimpleAudioManager.swift b/notetaker/SimpleAudioManager.swift new file mode 100644 index 0000000..17670fd --- /dev/null +++ b/notetaker/SimpleAudioManager.swift @@ -0,0 +1,298 @@ +// SimpleAudioManager.swift +// Audio manager for microphone and system audio capture + +import AVFoundation +import Foundation +import SwiftUI +import ScreenCaptureKit + +class SimpleAudioManager: NSObject, ObservableObject { + @Published var transcript = "" + @Published var isRecording = false + @Published var captureSystemAudio = true + + private var audioEngine = AVAudioEngine() + private var systemAudioEngine = AVAudioEngine() + private var webSocketTask: URLSessionWebSocketTask? + private let deepgramURL = URL(string: "wss://api.deepgram.com/v1/listen?encoding=linear16&sample_rate=16000&channels=1&interim_results=true")! + + // ScreenCaptureKit properties + private var stream: SCStream? + private var streamConfiguration: SCStreamConfiguration? + + func startRecording() { + print("Starting recording...") + + if captureSystemAudio { + startSystemAudioCapture() + } else { + startMicrophoneOnly() + } + } + + private func startMicrophoneOnly() { + print("Starting microphone-only recording...") + + // Set up audio engine for microphone capture + let inputNode = audioEngine.inputNode + let recordingFormat = inputNode.outputFormat(forBus: 0) + + // Convert to the format Deepgram expects: 16kHz, 16-bit, mono + let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: 16000, + channels: 1, + interleaved: false)! + + let converter = AVAudioConverter(from: recordingFormat, to: targetFormat)! + + inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { [weak self] (buffer, time) in + self?.processAudioBuffer(buffer, converter: converter, targetFormat: targetFormat) + } + + audioEngine.prepare() + + do { + try audioEngine.start() + DispatchQueue.main.async { + self.isRecording = true + } + connectToDeepgram() + print("Microphone recording started") + } catch { + print("Failed to start audio engine: \(error)") + } + } + + private func startSystemAudioCapture() { + print("Starting system audio capture...") + + Task { + do { + // Request screen recording permission + let canRecord = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + + // Get the main display + guard let display = canRecord.displays.first else { + print("❌ No display found") + return + } + + // Configure stream for audio only + let configuration = SCStreamConfiguration() + configuration.capturesAudio = true + configuration.excludesCurrentProcessAudio = false + configuration.sampleRate = 16000 + configuration.channelCount = 1 + + // Create content filter (we only want audio, but need to include display) + let filter = SCContentFilter(display: display, excludingWindows: []) + + // Create and start stream + let stream = SCStream(filter: filter, configuration: configuration, delegate: self) + + try await stream.startCapture() + + self.stream = stream + self.streamConfiguration = configuration + + DispatchQueue.main.async { + self.isRecording = true + } + + connectToDeepgram() + print("System audio capture started") + + } catch { + print("❌ Failed to start system audio capture: \(error)") + // Fallback to microphone only + startMicrophoneOnly() + } + } + } + + func stopRecording() { + print("Stopping recording...") + + // Stop system audio capture + if let stream = stream { + Task { + try? await stream.stopCapture() + } + self.stream = nil + } + + // Stop microphone capture + if audioEngine.isRunning { + audioEngine.stop() + audioEngine.inputNode.removeTap(onBus: 0) + } + + // Close WebSocket + webSocketTask?.cancel(with: .normalClosure, reason: nil) + webSocketTask = nil + + DispatchQueue.main.async { + self.isRecording = false + } + print("Recording stopped") + } + + private func processAudioBuffer(_ buffer: AVAudioPCMBuffer, converter: AVAudioConverter, targetFormat: AVAudioFormat) { + // Create output buffer for conversion + let outputFrameCapacity = AVAudioFrameCount(Double(buffer.frameLength) * targetFormat.sampleRate / buffer.format.sampleRate) + guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outputFrameCapacity) else { + print("❌ Failed to create output buffer") + return + } + + var error: NSError? + let status = converter.convert(to: outputBuffer, error: &error) { _, outStatus in + outStatus.pointee = .haveData + return buffer + } + + guard status == .haveData, error == nil else { + print("❌ Audio conversion failed: \(error?.localizedDescription ?? "Unknown error")") + return + } + + // Convert to Data for Deepgram + guard let channelData = outputBuffer.int16ChannelData?[0] else { + print("❌ No channel data available") + return + } + + let frameCount = Int(outputBuffer.frameLength) + let data = Data(bytes: channelData, count: frameCount * 2) + + print("🎵 Sending audio data: \(data.count) bytes") + sendAudioData(data) + } + + private func connectToDeepgram() { + guard let key = KeychainHelper.shared.get(forKey: "deepgramKey"), !key.isEmpty else { + print("❌ No Deepgram key found") + return + } + + print("🔑 Using Deepgram key: \(String(key.prefix(8)))...") + + let session = URLSession(configuration: .default) + var request = URLRequest(url: deepgramURL) + request.addValue("Token \(key)", forHTTPHeaderField: "Authorization") + + webSocketTask = session.webSocketTask(with: request) + webSocketTask?.resume() + receiveMessage() + print("🌐 Attempting to connect to Deepgram...") + } + + private func sendAudioData(_ data: Data) { + guard webSocketTask?.state == .running else { + print("⚠️ WebSocket not connected, skipping audio data") + return + } + + webSocketTask?.send(.data(data)) { error in + if let error = error { + print("❌ Send error: \(error)") + } + } + } + + private func receiveMessage() { + webSocketTask?.receive { [weak self] result in + switch result { + case .success(let message): + switch message { + case .string(let text): + print("📝 Received: \(text)") + self?.parseTranscription(text) + case .data: + break + @unknown default: + break + } + self?.receiveMessage() // Continue receiving + case .failure(let error): + print("❌ Receive error: \(error)") + // Try to reconnect after a delay + DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) { + if self?.isRecording == true { + print("🔄 Attempting to reconnect...") + self?.connectToDeepgram() + } + } + } + } + } + + private func parseTranscription(_ text: String) { + guard let data = text.data(using: .utf8), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let type = json["type"] as? String, type == "Results", + let channel = json["channel"] as? [String: Any], + let alternatives = channel["alternatives"] as? [[String: Any]], + let alt = alternatives.first, + let transcript = alt["transcript"] as? String, + !transcript.isEmpty else { return } + + DispatchQueue.main.async { + if let isFinal = json["is_final"] as? Bool, isFinal { + self.transcript += transcript + " " + } + } + } +} + +// MARK: - SCStreamDelegate +extension SimpleAudioManager: SCStreamDelegate { + func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { + guard type == .audio else { return } + + // Convert CMSampleBuffer to AVAudioPCMBuffer + guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) else { return } + let audioStreamBasicDescription = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription) + + guard let audioStreamBasicDescription = audioStreamBasicDescription else { return } + + let format = AVAudioFormat(streamDescription: audioStreamBasicDescription) + guard let format = format else { return } + + let frameCount = CMSampleBufferGetNumSamples(sampleBuffer) + guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frameCount)) else { return } + + // Copy audio data + var blockBuffer: CMBlockBuffer? + var audioBufferListOut = AudioBufferList() + var audioBufferListSize = MemoryLayout.size + + let status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer( + sampleBuffer, + bufferListSizeNeededOut: &audioBufferListSize, + bufferListOut: &audioBufferListOut, + bufferListSize: audioBufferListSize, + blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, + flags: 0, + blockBufferOut: &blockBuffer + ) + + guard status == noErr else { return } + + buffer.frameLength = AVAudioFrameCount(frameCount) + + // Convert to target format and send + let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: 16000, + channels: 1, + interleaved: false)! + + let converter = AVAudioConverter(from: format, to: targetFormat)! + processAudioBuffer(buffer, converter: converter, targetFormat: targetFormat) + } + + func stream(_ stream: SCStream, didStopWithError error: Error) { + print("❌ Stream stopped with error: \(error)") + } +} \ No newline at end of file diff --git a/notetaker/notetaker.entitlements b/notetaker/notetaker.entitlements index 18aff0c..e64c6c0 100644 --- a/notetaker/notetaker.entitlements +++ b/notetaker/notetaker.entitlements @@ -4,7 +4,17 @@ com.apple.security.app-sandbox + com.apple.security.device.audio-input + com.apple.security.files.user-selected.read-only + com.apple.security.network.client + + com.apple.security.application-groups + + group.owen.notetaker + + com.apple.security.device.screen-recording +