fix: onboarding and bug with opening new apps (#39)

This commit is contained in:
Owen Gretzinger
2025-07-23 23:32:42 -04:00
committed by GitHub
parent 7ddec9c193
commit ad4b61d5c2
6 changed files with 120 additions and 36 deletions
+2
View File
@@ -296,6 +296,7 @@
MACOSX_DEPLOYMENT_TARGET = 15.0; MACOSX_DEPLOYMENT_TARGET = 15.0;
MARKETING_VERSION = 1.0.6; MARKETING_VERSION = 1.0.6;
ONLY_ACTIVE_ARCH = NO; ONLY_ACTIVE_ARCH = NO;
OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI";
PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes; PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes;
PRODUCT_NAME = "$(TARGET_NAME)"; PRODUCT_NAME = "$(TARGET_NAME)";
PROVISIONING_PROFILE_SPECIFIER = ""; PROVISIONING_PROFILE_SPECIFIER = "";
@@ -331,6 +332,7 @@
MACOSX_DEPLOYMENT_TARGET = 15.0; MACOSX_DEPLOYMENT_TARGET = 15.0;
MARKETING_VERSION = 1.0.6; MARKETING_VERSION = 1.0.6;
ONLY_ACTIVE_ARCH = YES; ONLY_ACTIVE_ARCH = YES;
OTHER_SWIFT_FLAGS = "$(inherited) -D ENABLE_TCC_SPI";
PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes; PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes;
PRODUCT_NAME = "$(TARGET_NAME)"; PRODUCT_NAME = "$(TARGET_NAME)";
PROVISIONING_PROFILE_SPECIFIER = ""; PROVISIONING_PROFILE_SPECIFIER = "";
+83 -13
View File
@@ -5,6 +5,7 @@ import AVFoundation
import Foundation import Foundation
import SwiftUI import SwiftUI
import OSLog import OSLog
import Combine
/// Manages audio capture from microphone and system audio and handles real-time transcription via OpenAI /// Manages audio capture from microphone and system audio and handles real-time transcription via OpenAI
@MainActor @MainActor
@@ -29,6 +30,7 @@ class AudioManager: NSObject, ObservableObject {
private let permission = AudioRecordingPermission() private let permission = AudioRecordingPermission()
private let tapQueue = DispatchQueue(label: "io.meetingnotes.audiotap", qos: .userInitiated) private let tapQueue = DispatchQueue(label: "io.meetingnotes.audiotap", qos: .userInitiated)
private var isTapActive = false private var isTapActive = false
private var isRestartingSystemTap = false
// Add properties near the top, after existing private vars // Add properties near the top, after existing private vars
private var micRetryCount = 0 private var micRetryCount = 0
@@ -39,6 +41,7 @@ class AudioManager: NSObject, ObservableObject {
// Add ping timers to keep WebSocket connections alive // Add ping timers to keep WebSocket connections alive
private var pingTimers: [AudioSource: Timer] = [:] private var pingTimers: [AudioSource: Timer] = [:]
private var cancellables = Set<AnyCancellable>()
override init() { override init() {
super.init() super.init()
@@ -50,6 +53,19 @@ class AudioManager: NSObject, ObservableObject {
// Activate the process controller to start monitoring audio-producing apps // Activate the process controller to start monitoring audio-producing apps
audioProcessController.activate() audioProcessController.activate()
// When the list of running applications changes, check if we need to restart the system audio tap
NSWorkspace.shared.publisher(for: \.runningApplications)
.debounce(for: .seconds(1), scheduler: RunLoop.main)
.sink { [weak self] _ in
guard let self, self.isTapActive else { return }
print("🎤 Running applications changed, checking if tap restart is needed.")
Task {
await self.restartSystemAudioTapIfNeeded()
}
}
.store(in: &cancellables)
} }
deinit { deinit {
@@ -208,18 +224,17 @@ class AudioManager: NSObject, ObservableObject {
print("✨ Fresh audio engine created") print("✨ Fresh audio engine created")
} }
private func startSystemAudioTap() async { private func startSystemAudioTap(isRestart: Bool = false) async {
print("🎧 Starting system audio tap...") print(isRestart ? "🎧 Restarting system audio tap logic..." : "🎧 Starting system audio tap for the first time...")
// Ensure we have permission to record system audio. This might prompt the user. if !isRestart {
guard await checkSystemAudioPermissions() else { guard await checkSystemAudioPermissions() else {
let errorMsg = "System audio recording permission denied." let errorMsg = "System audio recording permission denied."
print("\(errorMsg)") print("\(errorMsg)")
DispatchQueue.main.async {
self.errorMessage = errorMsg self.errorMessage = errorMsg
}
return return
} }
}
// Get all running processes that are producing audio // Get all running processes that are producing audio
let allProcessObjectIDs = audioProcessController.processes.map { $0.objectID } let allProcessObjectIDs = audioProcessController.processes.map { $0.objectID }
@@ -236,9 +251,8 @@ class AudioManager: NSObject, ObservableObject {
if let tapError = newTap.errorMessage { if let tapError = newTap.errorMessage {
let errorMsg = "Failed to activate system audio tap: \(tapError)" let errorMsg = "Failed to activate system audio tap: \(tapError)"
print("\(errorMsg)") print("\(errorMsg)")
DispatchQueue.main.async {
self.errorMessage = errorMsg self.errorMessage = errorMsg
} if !isRestart { stopRecording() }
return return
} }
@@ -248,24 +262,73 @@ class AudioManager: NSObject, ObservableObject {
// Start receiving audio data from the tap // Start receiving audio data from the tap
do { do {
try startTapIO(newTap) try startTapIO(newTap)
connectToOpenAIRealtime(source: .system)
print("✅ System audio tap started successfully")
DispatchQueue.main.async { if !isRestart {
connectToOpenAIRealtime(source: .system)
self.isRecording = true self.isRecording = true
AudioLevelManager.shared.updateRecordingState(true) AudioLevelManager.shared.updateRecordingState(true)
} }
print("✅ System audio tap started successfully (isRestart: \(isRestart))")
} catch { } catch {
let errorMsg = "Failed to start system audio tap IO: \(error.localizedDescription)" let errorMsg = "Failed to start system audio tap IO: \(error.localizedDescription)"
print("\(errorMsg)") print("\(errorMsg)")
DispatchQueue.main.async {
self.errorMessage = errorMsg self.errorMessage = errorMsg
}
newTap.invalidate() newTap.invalidate()
self.isTapActive = false self.isTapActive = false
if !isRestart { stopRecording() }
} }
} }
private func restartSystemAudioTapIfNeeded() async {
let newProcessObjectIDs = Set(audioProcessController.processes.map { $0.objectID })
let currentProcessObjectIDs: Set<AudioObjectID>
if case .systemAudio(let processObjectIDs) = self.processTap?.target {
currentProcessObjectIDs = Set(processObjectIDs)
} else {
currentProcessObjectIDs = []
}
if newProcessObjectIDs != currentProcessObjectIDs {
print("Process list has changed. Restarting system audio tap.")
await restartSystemAudioTap()
} else {
print("Process list is the same. No restart needed.")
}
}
private func restartSystemAudioTap() async {
print("🔄 Restarting system audio tap...")
guard isRecording else {
print("Recording was stopped, aborting tap restart.")
return
}
isRestartingSystemTap = true
defer { isRestartingSystemTap = false }
// 1. Invalidate existing tap
if isTapActive {
processTap?.invalidate()
processTap = nil
isTapActive = false
print("System audio tap invalidated for restart.")
}
// A small delay to let things settle.
try? await Task.sleep(for: .milliseconds(250))
guard self.isRecording else {
print("Recording was stopped during tap restart. Aborting.")
return
}
// 2. Start a new one, but don't re-connect to OpenAI or change recording state
await startSystemAudioTap(isRestart: true)
}
@MainActor @MainActor
private func checkSystemAudioPermissions() async -> Bool { private func checkSystemAudioPermissions() async -> Bool {
if permission.status == .authorized { if permission.status == .authorized {
@@ -325,9 +388,16 @@ class AudioManager: NSObject, ObservableObject {
self.processAudioBuffer(buffer, converter: converter, targetFormat: targetFormat, source: .system) self.processAudioBuffer(buffer, converter: converter, targetFormat: targetFormat, source: .system)
} invalidationHandler: { [weak self] _ in } invalidationHandler: { [weak self] _ in
guard let self else { return }
print("Audio tap was invalidated.") print("Audio tap was invalidated.")
if !self.isRestartingSystemTap {
DispatchQueue.main.async { DispatchQueue.main.async {
self?.stopRecording() print("Tap invalidated unexpectedly. Stopping recording.")
self.stopRecording()
}
} else {
print("Tap invalidated as part of a restart. Not stopping recording.")
} }
} }
} }
@@ -49,7 +49,7 @@ extension String: @retroactive LocalizedError {
@Observable @Observable
final class AudioProcessController { final class AudioProcessController {
private let logger = Logger(subsystem: "codes.rambo.AudioCap", category: String(describing: AudioProcessController.self)) private let logger = Logger(subsystem: "owen.meetingnotes", category: String(describing: AudioProcessController.self))
private(set) var processes = [AudioProcess]() { private(set) var processes = [AudioProcess]() {
didSet { didSet {
@@ -5,7 +5,7 @@ import OSLog
/// Uses TCC SPI in order to check/request system audio recording permission. /// Uses TCC SPI in order to check/request system audio recording permission.
@Observable @Observable
final class AudioRecordingPermission { final class AudioRecordingPermission {
private let logger = Logger(subsystem: "codes.rambo.AudioCap", category: String(describing: AudioRecordingPermission.self)) private let logger = Logger(subsystem: "owen.meetingnotes", category: String(describing: AudioRecordingPermission.self))
enum Status: String { enum Status: String {
case unknown case unknown
+2 -2
View File
@@ -51,7 +51,7 @@ final class ProcessTap {
init(target: TapTarget, muteWhenRunning: Bool = false) { init(target: TapTarget, muteWhenRunning: Bool = false) {
self.target = target self.target = target
self.muteWhenRunning = muteWhenRunning self.muteWhenRunning = muteWhenRunning
self.logger = Logger(subsystem: "codes.rambo.AudioCap", category: "\(String(describing: ProcessTap.self))(\(target.loggingProcessName))") self.logger = Logger(subsystem: "owen.meetingnotes", category: "\(String(describing: ProcessTap.self))(\(target.loggingProcessName))")
} }
@ObservationIgnored @ObservationIgnored
@@ -327,7 +327,7 @@ final class ProcessTapRecorder {
self.tapDisplayName = tap.displayName self.tapDisplayName = tap.displayName
self.fileURL = fileURL self.fileURL = fileURL
self._tap = tap self._tap = tap
self.logger = Logger(subsystem: "codes.rambo.AudioCap", category: "\(String(describing: ProcessTapRecorder.self))(\(fileURL.lastPathComponent))") self.logger = Logger(subsystem: "owen.meetingnotes", category: "\(String(describing: ProcessTapRecorder.self))(\(fileURL.lastPathComponent))")
self.icon = tap.target.iconImage self.icon = tap.target.iconImage
} }
+25 -13
View File
@@ -6,10 +6,13 @@ struct OnboardingView: View {
@State private var apiKey = "" @State private var apiKey = ""
@State private var hasAcceptedTerms = false @State private var hasAcceptedTerms = false
@State private var micPermissionGranted = false @State private var micPermissionGranted = false
@State private var screenPermissionGranted = false @State private var systemAudioPermissionGranted = false
@State private var showingPermissionAlert = false @State private var showingPermissionAlert = false
@State private var permissionAlertMessage = "" @State private var permissionAlertMessage = ""
// Add AudioRecordingPermission instance
@State private var audioRecordingPermission = AudioRecordingPermission()
var body: some View { var body: some View {
GeometryReader { geometry in GeometryReader { geometry in
VStack(spacing: 0) { VStack(spacing: 0) {
@@ -31,10 +34,10 @@ struct OnboardingView: View {
) )
PermissionRow( PermissionRow(
title: "System Recording", title: "System Audio Recording",
description: "Required to transcribe what others say in meetings", description: "Required to transcribe what others say in meetings",
isGranted: screenPermissionGranted, isGranted: systemAudioPermissionGranted,
action: requestScreenPermission action: requestSystemAudioPermission
) )
} }
} }
@@ -155,11 +158,19 @@ struct OnboardingView: View {
apiKey = settingsViewModel.settings.openAIKey apiKey = settingsViewModel.settings.openAIKey
hasAcceptedTerms = settingsViewModel.settings.hasAcceptedTerms hasAcceptedTerms = settingsViewModel.settings.hasAcceptedTerms
} }
.onChange(of: audioRecordingPermission.status) { oldValue, newValue in
// Update permission status when it changes
systemAudioPermissionGranted = (newValue == .authorized)
}
.onReceive(NotificationCenter.default.publisher(for: NSApplication.didBecomeActiveNotification)) { _ in
// Re-check permissions when app becomes active (in case they were changed in System Settings)
checkPermissions()
}
} }
private var canProceed: Bool { private var canProceed: Bool {
return micPermissionGranted && return micPermissionGranted &&
screenPermissionGranted && systemAudioPermissionGranted &&
!apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty && !apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty &&
hasAcceptedTerms hasAcceptedTerms
} }
@@ -168,8 +179,8 @@ struct OnboardingView: View {
// Check microphone permission using AVCaptureDevice (macOS compatible) // Check microphone permission using AVCaptureDevice (macOS compatible)
micPermissionGranted = AVCaptureDevice.authorizationStatus(for: .audio) == .authorized micPermissionGranted = AVCaptureDevice.authorizationStatus(for: .audio) == .authorized
// Check screen recording permission // Check system audio recording permission
screenPermissionGranted = CGPreflightScreenCaptureAccess() systemAudioPermissionGranted = (audioRecordingPermission.status == .authorized)
} }
private func requestMicrophonePermission() { private func requestMicrophonePermission() {
@@ -184,12 +195,13 @@ struct OnboardingView: View {
} }
} }
private func requestScreenPermission() { private func requestSystemAudioPermission() {
let success = CGRequestScreenCaptureAccess() audioRecordingPermission.request()
DispatchQueue.main.async {
screenPermissionGranted = success // Show alert if permission is denied after request
if !success { DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
permissionAlertMessage = "System recording access is required to capture system audio. Please enable it in System Preferences > Security & Privacy > Privacy > Screen Recording." if audioRecordingPermission.status == .denied {
permissionAlertMessage = "System audio recording access is required to capture what others say in meetings. Please enable 'meetingnotes' in System Preferences > Security & Privacy > Privacy > Microphone."
showingPermissionAlert = true showingPermissionAlert = true
} }
} }