coder/meetingnotes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Basic recording functionality

Browse Source
This commit is contained in:
Owen Gretzinger
2025-07-10 13:09:32 -04:00
parent b9b1fbedd5
commit 417d751d7a
10 changed files with 738 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cursor/rules/build-but-dont-run.mdc
+7
View File
@@ -0,0 +1,7 @@
---
alwaysApply: true
---
After making changes, ensure there are no errors by creating an Xcode build.
However, don't run the app, I'll do that myself.
.cursor/rules/overall-objective.mdc
+20
View File
@@ -0,0 +1,20 @@
---
alwaysApply: true
---
I want to build an open source version of Granola
It will be a meeting notetaking macOS application
Basically it will use your mic and system audio, create a live transcript, and then at the end of the meeting use the transcript + any notes you write in a notepad to generate the meeting notes, with a button to quickly copy the notes or transcript. Can also edit the resulting notes
The user will provide their own deepgram and openai API keys (stored locally) for transcription and AI generation
The user will also be able to modify the system prompt in addition to just being able to write a blurb about themself (that gets injected into the system prompt)
All data will be stored locally on the device
That will be great for the MVP to start. Later on I will add additional features such as:
- connecting to your Google calendar
- note templates
- AI chat for asking questions about a meeting
- Integrations for email, slack, etc.
notetaker.xcodeproj/project.pbxproj
+53 -2
View File
@@ -6,13 +6,30 @@
objectVersion = 77;
objects = {
/* Begin PBXBuildFile section */
CC5718382E2020490042BA88 /* OpenAI in Frameworks */ = {isa = PBXBuildFile; productRef = CC5718372E2020490042BA88 /* OpenAI */; };
/* End PBXBuildFile section */
/* Begin PBXFileReference section */
CC5718182E201D8D0042BA88 /* notetaker.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = notetaker.app; sourceTree = BUILT_PRODUCTS_DIR; };
/* End PBXFileReference section */
/* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */
CC5718352E201EAA0042BA88 /* Exceptions for "notetaker" folder in "notetaker" target */ = {
isa = PBXFileSystemSynchronizedBuildFileExceptionSet;
membershipExceptions = (
Info.plist,
);
target = CC5718172E201D8D0042BA88 /* notetaker */;
};
/* End PBXFileSystemSynchronizedBuildFileExceptionSet section */
/* Begin PBXFileSystemSynchronizedRootGroup section */
CC57181A2E201D8D0042BA88 /* notetaker */ = {
isa = PBXFileSystemSynchronizedRootGroup;
exceptions = (
CC5718352E201EAA0042BA88 /* Exceptions for "notetaker" folder in "notetaker" target */,
);
path = notetaker;
sourceTree = "<group>";
};
@@ -23,6 +40,7 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
CC5718382E2020490042BA88 /* OpenAI in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -65,6 +83,7 @@
);
name = notetaker;
packageProductDependencies = (
CC5718372E2020490042BA88 /* OpenAI */,
);
productName = notetaker;
productReference = CC5718182E201D8D0042BA88 /* notetaker.app */;
@@ -94,6 +113,9 @@
);
mainGroup = CC57180F2E201D8D0042BA88;
minimizedProjectReferenceProxies = 1;
packageReferences = (
CC5718362E2020490042BA88 /* XCRemoteSwiftPackageReference "OpenAI" */,
);
preferredProjectObjectVersion = 77;
productRefGroup = CC5718192E201D8D0042BA88 /* Products */;
projectDirPath = "";
@@ -178,7 +200,7 @@
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
LOCALIZATION_PREFERS_STRING_CATALOGS = YES;
MACOSX_DEPLOYMENT_TARGET = 15.0;
MACOSX_DEPLOYMENT_TARGET = 13.0;
MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE;
MTL_FAST_MATH = YES;
ONLY_ACTIVE_ARCH = YES;
@@ -235,7 +257,7 @@
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
LOCALIZATION_PREFERS_STRING_CATALOGS = YES;
MACOSX_DEPLOYMENT_TARGET = 15.0;
MACOSX_DEPLOYMENT_TARGET = 13.0;
MTL_ENABLE_DEBUG_INFO = NO;
MTL_FAST_MATH = YES;
SDKROOT = macosx;
@@ -249,17 +271,22 @@
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
CODE_SIGN_ENTITLEMENTS = notetaker/notetaker.entitlements;
"CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development";
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_ASSET_PATHS = "\"notetaker/Preview Content\"";
DEVELOPMENT_TEAM = ML6HYR5LUR;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = notetaker/Info.plist;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Notetaker needs access to your microphone for transcription.";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 15.0;
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = owen.notetaker;
PRODUCT_NAME = "$(TARGET_NAME)";
@@ -274,17 +301,22 @@
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME = AccentColor;
CODE_SIGN_ENTITLEMENTS = notetaker/notetaker.entitlements;
"CODE_SIGN_IDENTITY[sdk=macosx*]" = "Apple Development";
CODE_SIGN_STYLE = Automatic;
COMBINE_HIDPI_IMAGES = YES;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_ASSET_PATHS = "\"notetaker/Preview Content\"";
DEVELOPMENT_TEAM = ML6HYR5LUR;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_FILE = notetaker/Info.plist;
INFOPLIST_KEY_NSHumanReadableCopyright = "";
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Notetaker needs access to your microphone for transcription.";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/../Frameworks",
);
MACOSX_DEPLOYMENT_TARGET = 15.0;
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = owen.notetaker;
PRODUCT_NAME = "$(TARGET_NAME)";
@@ -315,6 +347,25 @@
defaultConfigurationName = Release;
};
/* End XCConfigurationList section */
/* Begin XCRemoteSwiftPackageReference section */
CC5718362E2020490042BA88 /* XCRemoteSwiftPackageReference "OpenAI" */ = {
isa = XCRemoteSwiftPackageReference;
repositoryURL = "https://github.com/MacPaw/OpenAI";
requirement = {
kind = upToNextMajorVersion;
minimumVersion = 0.4.4;
};
};
/* End XCRemoteSwiftPackageReference section */
/* Begin XCSwiftPackageProductDependency section */
CC5718372E2020490042BA88 /* OpenAI */ = {
isa = XCSwiftPackageProductDependency;
package = CC5718362E2020490042BA88 /* XCRemoteSwiftPackageReference "OpenAI" */;
productName = OpenAI;
};
/* End XCSwiftPackageProductDependency section */
};
rootObject = CC5718102E201D8D0042BA88 /* Project object */;
}
notetaker.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
+33
View File
@@ -0,0 +1,33 @@
{
"originHash" : "c351aabe13067917247f7926404d096603b95663248b51124280feb61a2edf30",
"pins" : [
{
"identity" : "openai",
"kind" : "remoteSourceControl",
"location" : "https://github.com/MacPaw/OpenAI",
"state" : {
"revision" : "cedec2fc80aedafa332f7395b7004f79f547c794",
"version" : "0.4.4"
}
},
{
"identity" : "swift-http-types",
"kind" : "remoteSourceControl",
"location" : "https://github.com/apple/swift-http-types",
"state" : {
"revision" : "a0a57e949a8903563aba4615869310c0ebf14c03",
"version" : "1.4.0"
}
},
{
"identity" : "swift-openapi-runtime",
"kind" : "remoteSourceControl",
"location" : "https://github.com/apple/swift-openapi-runtime",
"state" : {
"revision" : "8f33cc5dfe81169fb167da73584b9c72c3e8bc23",
"version" : "1.8.2"
}
}
],
"version" : 3
}
notetaker/AudioManager.swift
+196
View File
@@ -0,0 +1,196 @@
// AudioManager.swift
import AVFoundation
import Foundation
import SwiftUI
import ScreenCaptureKit
class AudioManager: NSObject, ObservableObject {
@Published var transcript = ""
@Published var isRecording = false
private var stream: SCStream?
private var webSocketTask: URLSessionWebSocketTask?
private let deepgramURL = URL(string: "wss://api.deepgram.com/v1/listen?encoding=linear16&sample_rate=48000&channels=1&interim_results=true")!
func startRecording() {
// Check if ScreenCaptureKit is available
guard #available(macOS 13.0, *) else {
print("ScreenCaptureKit requires macOS 13.0 or later")
return
}
Task {
do {
// Request permission for screen recording (includes system audio)
let canRecord = await requestScreenRecordingPermission()
if !canRecord {
print("Screen recording permission denied")
DispatchQueue.main.async {
self.isRecording = false
}
return
}
// Set up screen capture configuration
let config = SCStreamConfiguration()
config.capturesAudio = true
config.captureMicrophone = true
config.sampleRate = 48000
config.channelCount = 1
// Create content filter for desktop audio capture
let availableContent = try await SCShareableContent.current
guard let display = availableContent.displays.first else {
print("No displays available for capture")
DispatchQueue.main.async {
self.isRecording = false
}
return
}
let filter = SCContentFilter(display: display, excludingWindows: [])
// Create stream
stream = SCStream(filter: filter, configuration: config, delegate: self)
// Add audio output
try stream?.addStreamOutput(self, type: .audio, sampleHandlerQueue: DispatchQueue(label: "AudioQueue"))
try stream?.addStreamOutput(self, type: .microphone, sampleHandlerQueue: DispatchQueue(label: "MicQueue"))
// Start capture
try await stream?.startCapture()
DispatchQueue.main.async {
self.isRecording = true
}
// Connect to Deepgram
connectToDeepgram()
} catch {
print("Failed to start recording: \(error)")
DispatchQueue.main.async {
self.isRecording = false
}
}
}
}
func stopRecording() {
Task {
do {
try await stream?.stopCapture()
stream = nil
webSocketTask?.cancel(with: .normalClosure, reason: nil)
webSocketTask = nil
DispatchQueue.main.async {
self.isRecording = false
}
} catch {
print("Failed to stop recording: \(error)")
}
}
}
private func requestScreenRecordingPermission() async -> Bool {
// ScreenCaptureKit handles permission requests automatically
// This will show a system dialog if needed
do {
let _ = try await SCShareableContent.current
return true
} catch {
return false
}
}
private func connectToDeepgram() {
let session = URLSession(configuration: .default)
var request = URLRequest(url: deepgramURL)
if let key = KeychainHelper.shared.get(forKey: "deepgramKey"), !key.isEmpty {
request.addValue("Token \(key)", forHTTPHeaderField: "Authorization")
} else {
print("No Deepgram key found")
return
}
webSocketTask = session.webSocketTask(with: request)
webSocketTask?.resume()
receiveMessage()
}
private func sendAudioData(_ data: Data) {
webSocketTask?.send(.data(data)) { error in
if let error = error {
print("Send error: \(error)")
}
}
}
private func receiveMessage() {
webSocketTask?.receive { result in
switch result {
case .success(let message):
switch message {
case .string(let text):
self.parseTranscription(text)
case .data:
break
@unknown default:
break
}
self.receiveMessage() // Continue receiving
case .failure(let error):
print("Receive error: \(error)")
}
}
}
private func parseTranscription(_ text: String) {
guard let data = text.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let type = json["type"] as? String, type == "Results",
let channel = json["channel"] as? [String: Any],
let alternatives = channel["alternatives"] as? [[String: Any]],
let alt = alternatives.first,
let transcript = alt["transcript"] as? String,
!transcript.isEmpty else { return }
DispatchQueue.main.async {
if let isFinal = json["is_final"] as? Bool, isFinal {
self.transcript += transcript + " "
}
}
}
}
// MARK: - SCStreamDelegate
extension AudioManager: SCStreamDelegate {
func stream(_ stream: SCStream, didStopWithError error: Error) {
print("Stream stopped with error: \(error)")
DispatchQueue.main.async {
self.isRecording = false
}
}
}
// MARK: - SCStreamOutput
extension AudioManager: SCStreamOutput {
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
// Convert sample buffer to audio data and send to Deepgram
guard let audioBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) else {
print("Failed to get data buffer from sample buffer")
return
}
var length: Int = 0
var dataPointer: UnsafeMutablePointer<Int8>?
let status = CMBlockBufferGetDataPointer(audioBuffer, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &length, dataPointerOut: &dataPointer)
guard status == noErr, let pointer = dataPointer, length > 0 else {
print("Failed to get audio data pointer, status: \(status), length: \(length)")
return
}
let data = Data(bytes: pointer, count: length)
sendAudioData(data)
}
}
notetaker/ContentView.swift
+71 -5
View File
@@ -8,14 +8,80 @@
import SwiftUI
struct ContentView: View {
@StateObject private var audioManager = SimpleAudioManager()
@State private var deepgramKey = ""
@State private var openAIKey = ""
@State private var userBlurb = ""
@State private var systemPrompt = "Generate meeting notes from transcript and notes."
var body: some View {
VStack {
Image(systemName: "globe")
.imageScale(.large)
.foregroundStyle(.tint)
Text("Hello, world!")
VStack(spacing: 20) {
Text("Notetaker Settings")
.font(.title)
TextField("Deepgram API Key", text: $deepgramKey)
TextField("OpenAI API Key", text: $openAIKey)
TextField("About Yourself (for prompt)", text: $userBlurb)
TextEditor(text: $systemPrompt)
.frame(height: 100)
Button("Save") {
saveSettings()
}
Divider()
Toggle("Capture System Audio", isOn: $audioManager.captureSystemAudio)
.disabled(audioManager.isRecording)
HStack {
Button("Start Recording") {
audioManager.startRecording()
}
.disabled(audioManager.isRecording)
Button("Stop Recording") {
audioManager.stopRecording()
}
.disabled(!audioManager.isRecording)
}
if audioManager.isRecording {
Text(audioManager.captureSystemAudio ? "🔴 Recording microphone + system audio..." : "🔴 Recording microphone...")
.foregroundColor(.red)
}
Text("Live Transcript:")
.font(.headline)
ScrollView {
Text(audioManager.transcript.isEmpty ? "No transcript yet..." : audioManager.transcript)
.frame(maxWidth: .infinity, alignment: .leading)
.padding()
}
.frame(height: 150)
.border(Color.gray)
}
.padding()
.frame(minWidth: 500, minHeight: 600)
.onAppear {
loadSettings()
}
}
func saveSettings() {
_ = KeychainHelper.shared.save(deepgramKey, forKey: "deepgramKey")
_ = KeychainHelper.shared.save(openAIKey, forKey: "openAIKey")
_ = KeychainHelper.shared.save(userBlurb, forKey: "userBlurb")
_ = KeychainHelper.shared.save(systemPrompt, forKey: "systemPrompt")
print("Settings saved!")
}
private func loadSettings() {
deepgramKey = KeychainHelper.shared.get(forKey: "deepgramKey") ?? ""
openAIKey = KeychainHelper.shared.get(forKey: "openAIKey") ?? ""
userBlurb = KeychainHelper.shared.get(forKey: "userBlurb") ?? ""
systemPrompt = KeychainHelper.shared.get(forKey: "systemPrompt") ?? "Generate meeting notes from transcript and notes."
}
}
notetaker/Info.plist
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>NSAudioCaptureUsageDescription</key>
<string>Notetaker needs access to capture system audio for transcription.</string>
</dict>
</plist>
notetaker/KeychainHelper.swift
+42
View File
@@ -0,0 +1,42 @@
// KeychainHelper.swift
// Helper for secure storage
import Foundation
import Security
class KeychainHelper {
static let shared = KeychainHelper()
private init() {}
func save(_ value: String, forKey key: String) -> Bool {
guard let data = value.data(using: .utf8) else { return false }
let query: [String: Any] = [
kSecClass as String: kSecClassGenericPassword,
kSecAttrAccount as String: key,
kSecValueData as String: data,
kSecAttrService as String: "owen.notetaker" // Replace with your bundle ID
]
SecItemDelete(query as CFDictionary) // Delete if exists
let status = SecItemAdd(query as CFDictionary, nil)
return status == errSecSuccess
}
func get(forKey key: String) -> String? {
let query: [String: Any] = [
kSecClass as String: kSecClassGenericPassword,
kSecAttrAccount as String: key,
kSecAttrService as String: "owen.notetaker",
kSecMatchLimit as String: kSecMatchLimitOne,
kSecReturnData as String: true
]
var item: AnyObject?
let status = SecItemCopyMatching(query as CFDictionary, &item)
guard status == errSecSuccess, let data = item as? Data else { return nil }
return String(data: data, encoding: .utf8)
}
}
notetaker/SimpleAudioManager.swift
+298
View File
@@ -0,0 +1,298 @@
// SimpleAudioManager.swift
// Audio manager for microphone and system audio capture
import AVFoundation
import Foundation
import SwiftUI
import ScreenCaptureKit
class SimpleAudioManager: NSObject, ObservableObject {
@Published var transcript = ""
@Published var isRecording = false
@Published var captureSystemAudio = true
private var audioEngine = AVAudioEngine()
private var systemAudioEngine = AVAudioEngine()
private var webSocketTask: URLSessionWebSocketTask?
private let deepgramURL = URL(string: "wss://api.deepgram.com/v1/listen?encoding=linear16&sample_rate=16000&channels=1&interim_results=true")!
// ScreenCaptureKit properties
private var stream: SCStream?
private var streamConfiguration: SCStreamConfiguration?
func startRecording() {
print("Starting recording...")
if captureSystemAudio {
startSystemAudioCapture()
} else {
startMicrophoneOnly()
}
}
private func startMicrophoneOnly() {
print("Starting microphone-only recording...")
// Set up audio engine for microphone capture
let inputNode = audioEngine.inputNode
let recordingFormat = inputNode.outputFormat(forBus: 0)
// Convert to the format Deepgram expects: 16kHz, 16-bit, mono
let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
sampleRate: 16000,
channels: 1,
interleaved: false)!
let converter = AVAudioConverter(from: recordingFormat, to: targetFormat)!
inputNode.installTap(onBus: 0, bufferSize: 1024, format: recordingFormat) { [weak self] (buffer, time) in
self?.processAudioBuffer(buffer, converter: converter, targetFormat: targetFormat)
}
audioEngine.prepare()
do {
try audioEngine.start()
DispatchQueue.main.async {
self.isRecording = true
}
connectToDeepgram()
print("Microphone recording started")
} catch {
print("Failed to start audio engine: \(error)")
}
}
private func startSystemAudioCapture() {
print("Starting system audio capture...")
Task {
do {
// Request screen recording permission
let canRecord = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true)
// Get the main display
guard let display = canRecord.displays.first else {
print("❌ No display found")
return
}
// Configure stream for audio only
let configuration = SCStreamConfiguration()
configuration.capturesAudio = true
configuration.excludesCurrentProcessAudio = false
configuration.sampleRate = 16000
configuration.channelCount = 1
// Create content filter (we only want audio, but need to include display)
let filter = SCContentFilter(display: display, excludingWindows: [])
// Create and start stream
let stream = SCStream(filter: filter, configuration: configuration, delegate: self)
try await stream.startCapture()
self.stream = stream
self.streamConfiguration = configuration
DispatchQueue.main.async {
self.isRecording = true
}
connectToDeepgram()
print("System audio capture started")
} catch {
print("❌ Failed to start system audio capture: \(error)")
// Fallback to microphone only
startMicrophoneOnly()
}
}
}
func stopRecording() {
print("Stopping recording...")
// Stop system audio capture
if let stream = stream {
Task {
try? await stream.stopCapture()
}
self.stream = nil
}
// Stop microphone capture
if audioEngine.isRunning {
audioEngine.stop()
audioEngine.inputNode.removeTap(onBus: 0)
}
// Close WebSocket
webSocketTask?.cancel(with: .normalClosure, reason: nil)
webSocketTask = nil
DispatchQueue.main.async {
self.isRecording = false
}
print("Recording stopped")
}
private func processAudioBuffer(_ buffer: AVAudioPCMBuffer, converter: AVAudioConverter, targetFormat: AVAudioFormat) {
// Create output buffer for conversion
let outputFrameCapacity = AVAudioFrameCount(Double(buffer.frameLength) * targetFormat.sampleRate / buffer.format.sampleRate)
guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outputFrameCapacity) else {
print("❌ Failed to create output buffer")
return
}
var error: NSError?
let status = converter.convert(to: outputBuffer, error: &error) { _, outStatus in
outStatus.pointee = .haveData
return buffer
}
guard status == .haveData, error == nil else {
print("❌ Audio conversion failed: \(error?.localizedDescription ?? "Unknown error")")
return
}
// Convert to Data for Deepgram
guard let channelData = outputBuffer.int16ChannelData?[0] else {
print("❌ No channel data available")
return
}
let frameCount = Int(outputBuffer.frameLength)
let data = Data(bytes: channelData, count: frameCount * 2)
print("🎵 Sending audio data: \(data.count) bytes")
sendAudioData(data)
}
private func connectToDeepgram() {
guard let key = KeychainHelper.shared.get(forKey: "deepgramKey"), !key.isEmpty else {
print("❌ No Deepgram key found")
return
}
print("🔑 Using Deepgram key: \(String(key.prefix(8)))...")
let session = URLSession(configuration: .default)
var request = URLRequest(url: deepgramURL)
request.addValue("Token \(key)", forHTTPHeaderField: "Authorization")
webSocketTask = session.webSocketTask(with: request)
webSocketTask?.resume()
receiveMessage()
print("🌐 Attempting to connect to Deepgram...")
}
private func sendAudioData(_ data: Data) {
guard webSocketTask?.state == .running else {
print("⚠️ WebSocket not connected, skipping audio data")
return
}
webSocketTask?.send(.data(data)) { error in
if let error = error {
print("❌ Send error: \(error)")
}
}
}
private func receiveMessage() {
webSocketTask?.receive { [weak self] result in
switch result {
case .success(let message):
switch message {
case .string(let text):
print("📝 Received: \(text)")
self?.parseTranscription(text)
case .data:
break
@unknown default:
break
}
self?.receiveMessage() // Continue receiving
case .failure(let error):
print("❌ Receive error: \(error)")
// Try to reconnect after a delay
DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) {
if self?.isRecording == true {
print("🔄 Attempting to reconnect...")
self?.connectToDeepgram()
}
}
}
}
}
private func parseTranscription(_ text: String) {
guard let data = text.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let type = json["type"] as? String, type == "Results",
let channel = json["channel"] as? [String: Any],
let alternatives = channel["alternatives"] as? [[String: Any]],
let alt = alternatives.first,
let transcript = alt["transcript"] as? String,
!transcript.isEmpty else { return }
DispatchQueue.main.async {
if let isFinal = json["is_final"] as? Bool, isFinal {
self.transcript += transcript + " "
}
}
}
}
// MARK: - SCStreamDelegate
extension SimpleAudioManager: SCStreamDelegate {
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
guard type == .audio else { return }
// Convert CMSampleBuffer to AVAudioPCMBuffer
guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer) else { return }
let audioStreamBasicDescription = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription)
guard let audioStreamBasicDescription = audioStreamBasicDescription else { return }
let format = AVAudioFormat(streamDescription: audioStreamBasicDescription)
guard let format = format else { return }
let frameCount = CMSampleBufferGetNumSamples(sampleBuffer)
guard let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frameCount)) else { return }
// Copy audio data
var blockBuffer: CMBlockBuffer?
var audioBufferListOut = AudioBufferList()
var audioBufferListSize = MemoryLayout<AudioBufferList>.size
let status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(
sampleBuffer,
bufferListSizeNeededOut: &audioBufferListSize,
bufferListOut: &audioBufferListOut,
bufferListSize: audioBufferListSize,
blockBufferAllocator: nil,
blockBufferMemoryAllocator: nil,
flags: 0,
blockBufferOut: &blockBuffer
)
guard status == noErr else { return }
buffer.frameLength = AVAudioFrameCount(frameCount)
// Convert to target format and send
let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
sampleRate: 16000,
channels: 1,
interleaved: false)!
let converter = AVAudioConverter(from: format, to: targetFormat)!
processAudioBuffer(buffer, converter: converter, targetFormat: targetFormat)
}
func stream(_ stream: SCStream, didStopWithError error: Error) {
print("❌ Stream stopped with error: \(error)")
}
}
notetaker/notetaker.entitlements
+10
View File
@@ -4,7 +4,17 @@
<dict>
<key>com.apple.security.app-sandbox</key>
<true/>
<key>com.apple.security.device.audio-input</key>
<true/>
<key>com.apple.security.files.user-selected.read-only</key>
<true/>
<key>com.apple.security.network.client</key>
<true/>
<key>com.apple.security.application-groups</key>
<array>
<string>group.owen.notetaker</string>
</array>
<key>com.apple.security.device.screen-recording</key>
<true/>
</dict>
</plist>