coder/meetingnotes
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

feat: use openai for tts instead of deepgram

Browse Source
This commit is contained in:
Owen Gretzinger
2025-07-11 17:31:54 -04:00
parent 263840570d
commit 5ee4fced42
12 changed files with 220 additions and 278 deletions
Download Patch File Download Diff File Expand all files Collapse all files
notetaker/Managers/AudioManager.swift
+103 -49
View File
@@ -6,7 +6,7 @@ import Foundation
import SwiftUI
import ScreenCaptureKit
/// Manages audio capture from microphone and system audio and handles real-time transcription via Deepgram
/// Manages audio capture from microphone and system audio and handles real-time transcription via OpenAI
class AudioManager: NSObject, ObservableObject {
@Published var transcriptChunks: [TranscriptChunk] = []
@Published var isRecording = false
@@ -14,7 +14,7 @@ class AudioManager: NSObject, ObservableObject {
private var audioEngine = AVAudioEngine()
private var micSocketTask: URLSessionWebSocketTask?
private var systemSocketTask: URLSessionWebSocketTask?
private let deepgramURL = URL(string: "wss://api.deepgram.com/v1/listen?encoding=linear16&sample_rate=16000&channels=1&interim_results=true&model=nova-3")!
private let realtimeURL = URL(string: "wss://api.openai.com/v1/realtime?intent=transcription")!
// ScreenCaptureKit properties
private var stream: SCStream?
@@ -22,6 +22,9 @@ class AudioManager: NSObject, ObservableObject {
// Add properties near the top, after existing private vars
private var micRetryCount = 0
private let maxMicRetries = 3
// Add current interim transcripts per source
private var currentInterim: [AudioSource: String] = [.mic: "", .system: ""]
override init() {
super.init()
@@ -93,7 +96,7 @@ class AudioManager: NSObject, ObservableObject {
}
}
/// Starts a microphone tap without creating a new Deepgram connection (used when also capturing system audio)
/// Starts a microphone tap without creating a new OpenAI connection (used when also capturing system audio)
private func startMicrophoneTap() {
print("🎤 Starting microphone tap...")
@@ -102,7 +105,7 @@ class AudioManager: NSObject, ObservableObject {
let recordingFormat = inputNode.outputFormat(forBus: 0)
guard let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
sampleRate: 16000,
sampleRate: 24000,
channels: 1,
interleaved: false) else {
print("❌ Failed to create target audio format for mic tap")
@@ -142,7 +145,7 @@ class AudioManager: NSObject, ObservableObject {
audioEngine.prepare()
try audioEngine.start()
connectToDeepgram(source: .mic)
connectToOpenAIRealtime(source: .mic)
print("✅ Microphone tap started successfully")
micRetryCount = 0 // Reset on success
@@ -222,7 +225,7 @@ class AudioManager: NSObject, ObservableObject {
self.isRecording = true
}
connectToDeepgram(source: .system)
connectToOpenAIRealtime(source: .system)
print("✅ System audio capture started successfully")
} catch {
@@ -263,7 +266,7 @@ class AudioManager: NSObject, ObservableObject {
private func processAudioBuffer(_ buffer: AVAudioPCMBuffer, converter: AVAudioConverter, targetFormat: AVAudioFormat, source: AudioSource) {
let processBuffer = buffer
// Convert to target format (16kHz int16 mono) in a single step – AVAudioConverter will handle resampling and downmixing
// Convert to target format (24kHz int16 mono) in a single step – AVAudioConverter will handle resampling and downmixing
let outputFrameCapacity = AVAudioFrameCount(Double(processBuffer.frameLength) * targetFormat.sampleRate / processBuffer.format.sampleRate)
guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outputFrameCapacity) else {
return
@@ -279,7 +282,7 @@ class AudioManager: NSObject, ObservableObject {
return
}
// Convert to Data for Deepgram
// Convert to Data for OpenAI
guard let channelData = outputBuffer.int16ChannelData?[0] else {
return
}
@@ -290,19 +293,51 @@ class AudioManager: NSObject, ObservableObject {
sendAudioData(data, source: source)
}
private func connectToDeepgram(source: AudioSource) {
guard let key = KeychainHelper.shared.get(forKey: "deepgramKey"), !key.isEmpty else {
print("❌ No Deepgram key found")
private func connectToOpenAIRealtime(source: AudioSource) {
guard let key = KeychainHelper.shared.get(forKey: "openAIKey"), !key.isEmpty else {
print("❌ No OpenAI key found")
return
}
let session = URLSession(configuration: .default)
var request = URLRequest(url: deepgramURL)
request.addValue("Token \(key)", forHTTPHeaderField: "Authorization")
var request = URLRequest(url: realtimeURL)
request.addValue("Bearer \(key)", forHTTPHeaderField: "Authorization")
request.addValue("realtime=v1", forHTTPHeaderField: "OpenAI-Beta")
let task = session.webSocketTask(with: request)
task.resume()
// Send initial configuration
let config: [String: Any] = [
"type": "transcription_session.update",
"session": [
"input_audio_format": "pcm16",
"input_audio_transcription": [
"model": "gpt-4o-mini-transcribe",
"language": "en"
],
"turn_detection": [
"type": "server_vad",
"threshold": 0.5,
"prefix_padding_ms": 300,
"silence_duration_ms": 200
]
]
]
do {
let jsonData = try JSONSerialization.data(withJSONObject: config)
if let jsonStr = String(data: jsonData, encoding: .utf8) {
task.send(.string(jsonStr)) { error in
if let error = error {
print("❌ Config send error: \(error)")
}
}
}
} catch {
print("❌ Config JSON error: \(error)")
}
switch source {
case .mic:
micSocketTask = task
@@ -311,7 +346,7 @@ class AudioManager: NSObject, ObservableObject {
}
receiveMessage(for: source)
print("🌐 Connected to Deepgram (\(source))")
print("🌐 Connected to OpenAI Realtime (\(source))")
}
private func receiveMessage(for source: AudioSource) {
@@ -321,7 +356,7 @@ class AudioManager: NSObject, ObservableObject {
case .success(let message):
switch message {
case .string(let text):
self?.parseTranscription(text, source: source)
self?.parseRealtimeEvent(text, source: source)
case .data:
break
@unknown default:
@@ -333,45 +368,54 @@ class AudioManager: NSObject, ObservableObject {
// Attempt reconnect if still recording
DispatchQueue.main.asyncAfter(deadline: .now() + 2) {
if self?.isRecording == true {
self?.connectToDeepgram(source: source)
self?.connectToOpenAIRealtime(source: source)
}
}
}
}
}
private func parseTranscription(_ text: String, source: AudioSource) {
private func parseRealtimeEvent(_ text: String, source: AudioSource) {
guard let data = text.data(using: .utf8),
let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
let type = json["type"] as? String, type == "Results",
let channel = json["channel"] as? [String: Any],
let alternatives = channel["alternatives"] as? [[String: Any]],
let alt = alternatives.first,
let transcriptText = alt["transcript"] as? String,
!transcriptText.isEmpty else { return }
let type = json["type"] as? String else { return }
let isFinal = json["is_final"] as? Bool ?? false
DispatchQueue.main.async {
let chunk = TranscriptChunk(
timestamp: Date(),
source: source,
text: transcriptText,
isFinal: isFinal
)
// For interim results, replace the last interim chunk from the same source
if !isFinal {
// Remove the last interim chunk from the same source
if let lastIndex = self.transcriptChunks.lastIndex(where: { !$0.isFinal && $0.source == source }) {
self.transcriptChunks.remove(at: lastIndex)
switch type {
case "conversation.item.input_audio_transcription.delta":
if let delta = json["delta"] as? String {
currentInterim[source]! += delta
DispatchQueue.main.async {
// Remove previous interim chunk from the same source
if let lastIndex = self.transcriptChunks.lastIndex(where: { !$0.isFinal && $0.source == source }) {
self.transcriptChunks.remove(at: lastIndex)
}
let chunk = TranscriptChunk(
timestamp: Date(),
source: source,
text: self.currentInterim[source] ?? "",
isFinal: false
)
self.transcriptChunks.append(chunk)
}
self.transcriptChunks.append(chunk)
} else {
// For final results, remove any interim chunks from the same source and add the final chunk
self.transcriptChunks.removeAll { !$0.isFinal && $0.source == source }
self.transcriptChunks.append(chunk)
}
case "conversation.item.input_audio_transcription.completed":
if let transcript = json["transcript"] as? String {
DispatchQueue.main.async {
// Remove any interim chunks from the same source
self.transcriptChunks.removeAll { !$0.isFinal && $0.source == source }
let chunk = TranscriptChunk(
timestamp: Date(),
source: source,
text: transcript,
isFinal: true
)
self.transcriptChunks.append(chunk)
}
currentInterim[source] = ""
}
default:
break
}
}
@@ -380,10 +424,20 @@ class AudioManager: NSObject, ObservableObject {
guard let socket = task, socket.state == .running else { return }
socket.send(.data(data)) { error in
if let error = error {
print("❌ Send error (\(source)): \(error)")
let base64 = data.base64EncodedString()
let message: [String: Any] = ["type": "input_audio_buffer.append", "audio": base64]
do {
let jsonData = try JSONSerialization.data(withJSONObject: message)
if let jsonStr = String(data: jsonData, encoding: .utf8) {
socket.send(.string(jsonStr)) { error in
if let error = error {
print("❌ Send error (\(source)): \(error)")
}
}
}
} catch {
print("❌ JSON send error")
}
}
@@ -402,9 +456,9 @@ extension AudioManager: SCStreamDelegate, SCStreamOutput {
// Convert CMSampleBuffer to AVAudioPCMBuffer
guard let pcmBuffer = sampleBuffer.asPCMBuffer else { return }
// Create converter for Deepgram format
// Create converter for OpenAI format
let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
sampleRate: 16000,
sampleRate: 24000,
channels: 1,
interleaved: false)!
@@ -431,4 +485,4 @@ extension CMSampleBuffer {
return AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: audioBufferList.unsafePointer)
}
}
}
}
notetaker/Models/Settings.swift
+23 -51
View File
@@ -1,72 +1,44 @@
import Foundation
struct Settings: Codable {
var deepgramKey: String
var openAIKey: String
var userBlurb: String
var systemPrompt: String
static let defaultSystemPrompt: String = {
guard let url = Bundle.main.url(forResource: "DefaultSystemPrompt", withExtension: "txt"),
let content = try? String(contentsOf: url) else {
assertionFailure("DefaultSystemPrompt.txt missing from bundle")
return ""
// System prompt default loading
static func defaultSystemPrompt() -> String {
guard let path = Bundle.main.path(forResource: "DefaultSystemPrompt", ofType: "txt"),
let content = try? String(contentsOfFile: path) else {
return "You are a helpful assistant that creates comprehensive meeting notes from transcript data."
}
return content
}()
// Required variables for the template
static let requiredVariables: Set<String> = [
"meeting_title",
"meeting_date",
"transcript",
"user_blurb",
"user_notes"
]
init(deepgramKey: String = "",
openAIKey: String = "",
userBlurb: String = "",
systemPrompt: String = Settings.defaultSystemPrompt) {
self.deepgramKey = deepgramKey
self.openAIKey = openAIKey
self.userBlurb = userBlurb
self.systemPrompt = systemPrompt
}
// MARK: - Template Methods
/// Extracts all template variables from a system prompt string
static func extractTemplateVariables(from template: String) -> Set<String> {
let pattern = #"\{\{(\w+)\}\}"#
let regex = try! NSRegularExpression(pattern: pattern, options: [])
let matches = regex.matches(in: template, options: [], range: NSRange(location: 0, length: template.count))
var variables: Set<String> = []
for match in matches {
if let range = Range(match.range(at: 1), in: template) {
variables.insert(String(template[range]))
}
// Add a computed property for the full prompt
var fullSystemPrompt: String {
let defaultPrompt = Settings.defaultSystemPrompt()
if userBlurb.isEmpty {
return defaultPrompt
}
return variables
return "\(defaultPrompt)\n\nContext about the user: \(userBlurb)"
}
/// Validates that all required variables are present in the system prompt
func validateSystemPrompt() -> (isValid: Bool, missingVariables: Set<String>) {
let presentVariables = Settings.extractTemplateVariables(from: systemPrompt)
let missingVariables = Settings.requiredVariables.subtracting(presentVariables)
return (isValid: missingVariables.isEmpty, missingVariables: missingVariables)
}
/// Replaces template variables in the system prompt with actual values
// Template processing method
static func processTemplate(_ template: String, with variables: [String: String]) -> String {
var result = template
for (key, value) in variables {
let placeholder = "{{\(key)}}"
result = result.replacingOccurrences(of: placeholder, with: value)
result = result.replacingOccurrences(of: "{{\(key)}}", with: value)
}
return result
}
init(openAIKey: String = "",
userBlurb: String = "",
systemPrompt: String = "") {
self.openAIKey = openAIKey
self.userBlurb = userBlurb
self.systemPrompt = systemPrompt.isEmpty ? Settings.defaultSystemPrompt() : systemPrompt
}
}
notetaker/ViewModels/MeetingViewModel.swift
+1 -1
View File
@@ -116,7 +116,7 @@ class MeetingViewModel: ObservableObject {
do {
// Load settings for generation
let userBlurb = KeychainHelper.shared.get(forKey: "userBlurb") ?? ""
let systemPrompt = KeychainHelper.shared.get(forKey: "systemPrompt") ?? Settings.defaultSystemPrompt
let systemPrompt = KeychainHelper.shared.get(forKey: "systemPrompt") ?? Settings.defaultSystemPrompt()
meeting.generatedNotes = try await NotesGenerator.shared.generateNotes(
meeting: meeting,
notetaker/ViewModels/SettingsViewModel.swift
+14 -33
View File
@@ -1,60 +1,41 @@
import Foundation
import SwiftUI
@MainActor
class SettingsViewModel: ObservableObject {
@Published var settings: Settings
@Published var isSaving = false
@Published var saveSuccessful = false
@Published var errorMessage: String?
@Published var settings = Settings()
@Published var saveMessage = ""
@Published var showingSaveMessage = false
init() {
self.settings = Settings()
loadSettings()
}
func loadSettings() {
// Load API keys from Keychain
settings.deepgramKey = KeychainHelper.shared.get(forKey: "deepgramKey") ?? ""
settings.openAIKey = KeychainHelper.shared.get(forKey: "openAIKey") ?? ""
settings.userBlurb = KeychainHelper.shared.get(forKey: "userBlurb") ?? ""
settings.systemPrompt = KeychainHelper.shared.get(forKey: "systemPrompt") ?? Settings.defaultSystemPrompt
settings.systemPrompt = KeychainHelper.shared.get(forKey: "systemPrompt") ?? Settings.defaultSystemPrompt()
}
func saveSettings() {
isSaving = true
errorMessage = nil
saveSuccessful = false
// Validate system prompt template before saving
let validation = settings.validateSystemPrompt()
if !validation.isValid {
let missingVars = validation.missingVariables.joined(separator: ", ")
errorMessage = "System prompt is missing required variables: \(missingVars). Please include these variables in your template using double curly braces (e.g., {{transcript}})."
isSaving = false
return
}
// Save to Keychain
let deepgramSaved = KeychainHelper.shared.save(settings.deepgramKey, forKey: "deepgramKey")
let openAISaved = KeychainHelper.shared.save(settings.openAIKey, forKey: "openAIKey")
let blurbSaved = KeychainHelper.shared.save(settings.userBlurb, forKey: "userBlurb")
let promptSaved = KeychainHelper.shared.save(settings.systemPrompt, forKey: "systemPrompt")
if deepgramSaved && openAISaved && blurbSaved && promptSaved {
saveSuccessful = true
// Show success briefly
DispatchQueue.main.asyncAfter(deadline: .now() + 2) { [weak self] in
self?.saveSuccessful = false
}
if openAISaved && blurbSaved && promptSaved {
saveMessage = "Settings saved successfully!"
} else {
errorMessage = "Failed to save some settings. Please try again."
saveMessage = "Error saving settings"
}
isSaving = false
showingSaveMessage = true
// Hide the message after 3 seconds
DispatchQueue.main.asyncAfter(deadline: .now() + 3) {
self.showingSaveMessage = false
}
}
func resetToDefaults() {
settings.systemPrompt = Settings.defaultSystemPrompt
settings.systemPrompt = Settings.defaultSystemPrompt()
}
}
notetaker/Views/MeetingListView.swift
+1 -1
View File
@@ -71,7 +71,7 @@ struct MeetingListView: View {
}
}
.sheet(isPresented: $showingSettings) {
SettingsView()
SettingsView(viewModel: SettingsViewModel())
}
.navigationDestination(for: Meeting.self) { meeting in
MeetingDetailView(meeting: meeting)
notetaker/Views/SettingsView.swift
+67 -112
View File
@@ -1,146 +1,101 @@
import SwiftUI
struct SettingsView: View {
@StateObject private var viewModel = SettingsViewModel()
@ObservedObject var viewModel: SettingsViewModel
@Environment(\.dismiss) private var dismiss
var body: some View {
NavigationStack {
ScrollView {
VStack(spacing: 32) {
// API Keys Section
VStack(alignment: .leading, spacing: 16) {
Text("API Keys")
VStack(alignment: .leading, spacing: 24) {
// API Configuration Section
VStack(alignment: .leading, spacing: 8) {
Text("OpenAI API Key")
.font(.headline)
.fontWeight(.semibold)
VStack(spacing: 12) {
VStack(alignment: .leading, spacing: 4) {
Text("Deepgram API Key")
.font(.subheadline)
.foregroundColor(.secondary)
SecureField("Enter your Deepgram API key", text: $viewModel.settings.deepgramKey)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
}
VStack(alignment: .leading, spacing: 4) {
Text("OpenAI API Key")
.font(.subheadline)
.foregroundColor(.secondary)
SecureField("Enter your OpenAI API key", text: $viewModel.settings.openAIKey)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
}
}
}
.padding(.horizontal, 24)
Divider()
.padding(.horizontal, 24)
// Personal Information Section
VStack(alignment: .leading, spacing: 16) {
Text("Personal Information")
.font(.headline)
.fontWeight(.semibold)
VStack(alignment: .leading, spacing: 8) {
Text("About Yourself")
.font(.subheadline)
.foregroundColor(.secondary)
Text("This information will be included in the AI context when generating meeting notes.")
.font(.caption)
.foregroundColor(.secondary)
TextEditor(text: $viewModel.settings.userBlurb)
.frame(minHeight: 80, maxHeight: 120)
.scrollContentBackground(.hidden)
.background(Color(.controlBackgroundColor))
.cornerRadius(8)
.overlay(
RoundedRectangle(cornerRadius: 8)
.stroke(Color(.separatorColor), lineWidth: 1)
)
}
}
.padding(.horizontal, 24)
Divider()
.padding(.horizontal, 24)
// AI Generation Section
VStack(alignment: .leading, spacing: 16) {
HStack {
Text("AI Generation")
.font(.headline)
.fontWeight(.semibold)
Spacer()
Button("Reset to Default") {
viewModel.resetToDefaults()
}
.foregroundColor(.primary)
Text("Stored locally and encrypted.")
.font(.caption)
.foregroundColor(.accentColor)
}
.foregroundColor(.secondary)
VStack(alignment: .leading, spacing: 8) {
Text("System Prompt")
.font(.subheadline)
.foregroundColor(.secondary)
Text("Customize how the AI generates your meeting notes.")
.font(.caption)
.foregroundColor(.secondary)
TextEditor(text: $viewModel.settings.systemPrompt)
.frame(minHeight: 100, maxHeight: 160)
.scrollContentBackground(.hidden)
.background(Color(.controlBackgroundColor))
.cornerRadius(8)
.overlay(
RoundedRectangle(cornerRadius: 8)
.stroke(Color(.separatorColor), lineWidth: 1)
)
SecureField("OpenAI API Key", text: $viewModel.settings.openAIKey)
.textFieldStyle(.roundedBorder)
.frame(maxWidth: .infinity)
}
// User Information Section
VStack(alignment: .leading, spacing: 8) {
Text("User Information")
.font(.headline)
.foregroundColor(.primary)
Text("Meetingsnotes works best when it knows a bit about you. You should give your name, role, company, and any other relevant information.")
.font(.caption)
.foregroundColor(.secondary)
TextEditor(text: $viewModel.settings.userBlurb)
.frame(minHeight: 100)
.overlay(
RoundedRectangle(cornerRadius: 8)
.stroke(Color.gray.opacity(0.3), lineWidth: 1)
)
}
// System Prompt Section
VStack(alignment: .leading, spacing: 8) {
Text("System Prompt")
.font(.headline)
.foregroundColor(.primary)
TextEditor(text: $viewModel.settings.systemPrompt)
.frame(minHeight: 200)
.overlay(
RoundedRectangle(cornerRadius: 8)
.stroke(Color.gray.opacity(0.3), lineWidth: 1)
)
}
// GitHub Link Section
VStack(alignment: .leading, spacing: 8) {
Text("About")
.font(.headline)
.foregroundColor(.primary)
Link(destination: URL(string: "https://github.com/owengretzinger/meetingnotes")!) {
HStack {
Image(systemName: "chevron.left.forwardslash.chevron.right")
.foregroundColor(.blue)
Text("View on GitHub")
.foregroundColor(.blue)
}
}
}
.padding(.horizontal, 24)
Spacer(minLength: 20)
}
.padding(.vertical, 24)
.padding(24)
}
.navigationTitle("Settings")
.frame(minWidth: 600, minHeight: 500)
.toolbar {
ToolbarItem(placement: .cancellationAction) {
Button("Cancel") {
dismiss()
}
}
ToolbarItem(placement: .confirmationAction) {
Button("Save") {
viewModel.saveSettings()
dismiss()
}
.disabled(viewModel.isSaving)
.buttonStyle(.borderedProminent)
}
}
.onChange(of: viewModel.saveSuccessful) { _, isSuccessful in
if isSuccessful {
dismiss()
}
}
.alert("Error", isPresented: .constant(viewModel.errorMessage != nil)) {
Button("OK") {
viewModel.errorMessage = nil
}
} message: {
Text(viewModel.errorMessage ?? "")
}
.frame(minWidth: 600, minHeight: 600)
}
.onAppear {
viewModel.loadSettings()
}
}
}
#Preview {
SettingsView()
SettingsView(viewModel: SettingsViewModel())
}