From d537810f1b8cc8538ae5c25efdfb877fb7c8bf48 Mon Sep 17 00:00:00 2001 From: Owen Gretzinger Date: Wed, 23 Jul 2025 23:14:32 -0400 Subject: [PATCH] feat: proper system audio recording instead of screen recording (#36) --- Meetingnotes.xcodeproj/project.pbxproj | 4 +- meetingnotes/Info.plist | 4 +- meetingnotes/Managers/AudioManager.swift | 218 ++++---- .../ProcessTap/AudioProcessController.swift | 239 +++++++++ .../ProcessTap/AudioRecordingPermission.swift | 125 +++++ meetingnotes/ProcessTap/CoreAudioUtils.swift | 310 +++++++++++ meetingnotes/ProcessTap/ProcessTap.swift | 499 ++++++++++++++++++ 7 files changed, 1297 insertions(+), 102 deletions(-) create mode 100644 meetingnotes/ProcessTap/AudioProcessController.swift create mode 100644 meetingnotes/ProcessTap/AudioRecordingPermission.swift create mode 100644 meetingnotes/ProcessTap/CoreAudioUtils.swift create mode 100644 meetingnotes/ProcessTap/ProcessTap.swift diff --git a/Meetingnotes.xcodeproj/project.pbxproj b/Meetingnotes.xcodeproj/project.pbxproj index 4c2fa9f..02c6f39 100644 --- a/Meetingnotes.xcodeproj/project.pbxproj +++ b/Meetingnotes.xcodeproj/project.pbxproj @@ -293,7 +293,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 14.0; + MACOSX_DEPLOYMENT_TARGET = 14.4; MARKETING_VERSION = 1.0.6; ONLY_ACTIVE_ARCH = NO; PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes; @@ -328,7 +328,7 @@ "$(inherited)", "@executable_path/../Frameworks", ); - MACOSX_DEPLOYMENT_TARGET = 14.0; + MACOSX_DEPLOYMENT_TARGET = 14.4; MARKETING_VERSION = 1.0.6; ONLY_ACTIVE_ARCH = YES; PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes; diff --git a/meetingnotes/Info.plist b/meetingnotes/Info.plist index af9fdad..b1fb75b 100644 --- a/meetingnotes/Info.plist +++ b/meetingnotes/Info.plist @@ -4,8 +4,8 @@ NSAudioCaptureUsageDescription Meetingnotes needs access to capture system audio for transcription. - NSScreenCaptureUsageDescription - Meetingnotes needs screen recording permission to capture system audio for transcription. + NSMicrophoneUsageDescription + Meetingnotes needs access to your microphone for transcription. SUFeedURL https://raw.githubusercontent.com/owengretzinger/meetingnotes/main/appcast.xml SUPublicEDKey diff --git a/meetingnotes/Managers/AudioManager.swift b/meetingnotes/Managers/AudioManager.swift index 5549ae4..4a94369 100644 --- a/meetingnotes/Managers/AudioManager.swift +++ b/meetingnotes/Managers/AudioManager.swift @@ -4,9 +4,10 @@ import AVFoundation import Foundation import SwiftUI -import ScreenCaptureKit +import OSLog /// Manages audio capture from microphone and system audio and handles real-time transcription via OpenAI +@MainActor class AudioManager: NSObject, ObservableObject { @Published var transcriptChunks: [TranscriptChunk] = [] @Published var isRecording = false @@ -20,8 +21,12 @@ class AudioManager: NSObject, ObservableObject { // Unique identifier for the current recording session private var sessionID = UUID() - // ScreenCaptureKit properties - private var stream: SCStream? + // ProcessTap properties + private var processTap: ProcessTap? + private let audioProcessController = AudioProcessController() + private let permission = AudioRecordingPermission() + private let tapQueue = DispatchQueue(label: "io.meetingnotes.audiotap", qos: .userInitiated) + private var isTapActive = false // Add properties near the top, after existing private vars private var micRetryCount = 0 @@ -40,6 +45,9 @@ class AudioManager: NSObject, ObservableObject { queue: .main) { [weak self] _ in self?.handleAudioEngineConfigurationChange() } + + // Activate the process controller to start monitoring audio-producing apps + audioProcessController.activate() } deinit { @@ -66,7 +74,7 @@ class AudioManager: NSObject, ObservableObject { self.startMicrophoneTap() // Start system audio capture asynchronously Task { - await self.startSystemAudioCapture() + await self.startSystemAudioTap() } } } @@ -75,10 +83,11 @@ class AudioManager: NSObject, ObservableObject { print("Internal cleanup...") // Stop system audio capture - if let stream = stream { - stream.stopCapture() - self.stream = nil - print("System audio capture stopped") + if isTapActive { + self.processTap?.invalidate() + self.processTap = nil + isTapActive = false + print("System audio tap invalidated") } // Stop microphone capture @@ -195,61 +204,112 @@ class AudioManager: NSObject, ObservableObject { print("✨ Fresh audio engine created") } - private func startSystemAudioCapture() async { - print("🎧 Starting system audio capture...") + private func startSystemAudioTap() async { + print("🎧 Starting system audio tap...") + // Ensure we have permission to record system audio. This might prompt the user. + guard await checkSystemAudioPermissions() else { + let errorMsg = "System audio recording permission denied." + print("❌ \(errorMsg)") + DispatchQueue.main.async { + self.errorMessage = errorMsg + } + return + } + + // Get all running processes that are producing audio + let allProcessObjectIDs = audioProcessController.processes.map { $0.objectID } + if allProcessObjectIDs.isEmpty { + print("⚠️ No audio-producing processes found. System audio tap might not capture anything.") + } + + // Configure the tap for system-wide audio + let target = TapTarget.systemAudio(processObjectIDs: allProcessObjectIDs) + let newTap = ProcessTap(target: target) + newTap.activate() + + // Check for activation errors + if let tapError = newTap.errorMessage { + let errorMsg = "Failed to activate system audio tap: \(tapError)" + print("❌ \(errorMsg)") + DispatchQueue.main.async { + self.errorMessage = errorMsg + } + return + } + + self.processTap = newTap + self.isTapActive = true + + // Start receiving audio data from the tap do { - // Request screen capture permission - let content = try await SCShareableContent.excludingDesktopWindows(true, onScreenWindowsOnly: true) - - // Exclude self to avoid feedback - let excludedApps = content.applications.filter { app in - Bundle.main.bundleIdentifier == app.bundleIdentifier - } - - guard let display = content.displays.first else { - print("❌ No display found") - return - } - - // Create filter - let filter = SCContentFilter(display: display, excludingApplications: excludedApps, exceptingWindows: []) - - // Configure stream - let configuration = SCStreamConfiguration() - configuration.width = 2 // Minimal video settings - configuration.height = 2 - configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale.max) - configuration.capturesAudio = true - configuration.sampleRate = 48000 - configuration.channelCount = 2 - - // Create stream - let stream = SCStream(filter: filter, configuration: configuration, delegate: self) - - // Add stream output for audio processing - try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: .global(qos: .userInitiated)) - // Add a minimal screen output so SCStream doesn't complain about missing video output - try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: .global(qos: .userInitiated)) - - // Start capture - try await stream.startCapture() - - // Store reference - self.stream = stream + try startTapIO(newTap) + connectToOpenAIRealtime(source: .system) + print("✅ System audio tap started successfully") DispatchQueue.main.async { self.isRecording = true } - - connectToOpenAIRealtime(source: .system) - print("✅ System audio capture started successfully") - } catch { - print("❌ Failed to start system audio capture: \(error)") + let errorMsg = "Failed to start system audio tap IO: \(error.localizedDescription)" + print("❌ \(errorMsg)") + DispatchQueue.main.async { + self.errorMessage = errorMsg + } + newTap.invalidate() + self.isTapActive = false + } + } + + @MainActor + private func checkSystemAudioPermissions() async -> Bool { + if permission.status == .authorized { + return true + } + + permission.request() + + // Poll for a short time to see if permission is granted + for _ in 0..<10 { + if permission.status == .authorized { + return true + } + try? await Task.sleep(nanoseconds: 500_000_000) // 0.5 seconds + } + + return permission.status == .authorized + } + + private func startTapIO(_ tap: ProcessTap) throws { + guard var streamDescription = tap.tapStreamDescription else { + throw NSError(domain: "AudioManager", code: -1, userInfo: [NSLocalizedDescriptionKey: "Failed to get audio format from tap."]) + } + + guard let format = AVAudioFormat(streamDescription: &streamDescription) else { + throw NSError(domain: "AudioManager", code: -1, userInfo: [NSLocalizedDescriptionKey: "Failed to create AVAudioFormat from tap."]) + } + + try tap.run(on: tapQueue) { [weak self] _, inInputData, _, _, _ in + guard let self = self, + let buffer = AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: inInputData, deallocator: nil) else { + return + } + + let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, + sampleRate: 24000, + channels: 1, + interleaved: false)! + + guard let converter = AVAudioConverter(from: format, to: targetFormat) else { + return + } - if case SCStreamError.userDeclined = error { - print("📍 Permission denied. User needs to enable screen recording in System Settings.") + self.processAudioBuffer(buffer, converter: converter, targetFormat: targetFormat, source: .system) + + } invalidationHandler: { [weak self] _ in + print("Audio tap was invalidated.") + DispatchQueue.main.async { + self?.stopRecording() } } } @@ -260,9 +320,11 @@ class AudioManager: NSObject, ObservableObject { print("Stopping recording...") // Stop system audio capture - if let stream = stream { - stream.stopCapture() - self.stream = nil + if isTapActive { + self.processTap?.invalidate() + self.processTap = nil + isTapActive = false + print("System audio tap invalidated") } // Stop microphone capture @@ -557,44 +619,4 @@ class AudioManager: NSObject, ObservableObject { print("🔔 Audio engine configuration changed - restarting mic") restartMicrophone() } -} - -// MARK: - SCStreamDelegate & SCStreamOutput -extension AudioManager: SCStreamDelegate, SCStreamOutput { - func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) { - guard type == .audio else { return } - guard sampleBuffer.isValid else { return } - - // Convert CMSampleBuffer to AVAudioPCMBuffer - guard let pcmBuffer = sampleBuffer.asPCMBuffer else { return } - - // Create converter for OpenAI format - let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, - sampleRate: 24000, - channels: 1, - interleaved: false)! - - guard let converter = AVAudioConverter(from: pcmBuffer.format, to: targetFormat) else { return } - - processAudioBuffer(pcmBuffer, converter: converter, targetFormat: targetFormat, source: .system) - } - - func stream(_ stream: SCStream, didStopWithError error: Error) { - print("❌ Stream stopped with error: \(error)") - DispatchQueue.main.async { - self.stream = nil - self.isRecording = false - } - } -} - -// MARK: - CMSampleBuffer Extension -extension CMSampleBuffer { - var asPCMBuffer: AVAudioPCMBuffer? { - try? self.withAudioBufferList { audioBufferList, _ -> AVAudioPCMBuffer? in - guard let absd = self.formatDescription?.audioStreamBasicDescription else { return nil } - guard let format = AVAudioFormat(standardFormatWithSampleRate: absd.mSampleRate, channels: absd.mChannelsPerFrame) else { return nil } - return AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: audioBufferList.unsafePointer) - } - } } diff --git a/meetingnotes/ProcessTap/AudioProcessController.swift b/meetingnotes/ProcessTap/AudioProcessController.swift new file mode 100644 index 0000000..a44231c --- /dev/null +++ b/meetingnotes/ProcessTap/AudioProcessController.swift @@ -0,0 +1,239 @@ +import SwiftUI +import AudioToolbox +import OSLog +import Combine + +struct AudioProcess: Identifiable, Hashable, Sendable { + enum Kind: String, Sendable { + case process + case app + } + var id: pid_t + var kind: Kind + var name: String + var audioActive: Bool + var bundleID: String? + var bundleURL: URL? + var objectID: AudioObjectID +} + +struct AudioProcessGroup: Identifiable, Hashable, Sendable { + var id: String + var title: String + var processes: [AudioProcess] +} + +extension AudioProcess.Kind { + var defaultIcon: NSImage { + switch self { + case .process: NSWorkspace.shared.icon(for: .unixExecutable) + case .app: NSWorkspace.shared.icon(for: .applicationBundle) + } + } +} + +extension AudioProcess { + var icon: NSImage { + guard let bundleURL else { return kind.defaultIcon } + let image = NSWorkspace.shared.icon(forFile: bundleURL.path) + image.size = NSSize(width: 32, height: 32) + return image + } +} + +extension String: @retroactive LocalizedError { + public var errorDescription: String? { self } +} + +@MainActor +@Observable +final class AudioProcessController { + + private let logger = Logger(subsystem: "codes.rambo.AudioCap", category: String(describing: AudioProcessController.self)) + + private(set) var processes = [AudioProcess]() { + didSet { + guard processes != oldValue else { return } + processGroups = AudioProcessGroup.groups(with: processes) + } + } + + private(set) var processGroups = [AudioProcessGroup]() + + private var cancellables = Set() + + func activate() { + logger.debug(#function) + + NSWorkspace.shared + .publisher(for: \.runningApplications, options: [.initial, .new]) + .map { $0.filter({ $0.processIdentifier != ProcessInfo.processInfo.processIdentifier }) } + .sink { [weak self] apps in + guard let self else { return } + self.reload(apps: apps) + } + .store(in: &cancellables) + } + + fileprivate func reload(apps: [NSRunningApplication]) { + logger.debug(#function) + + do { + let objectIdentifiers = try AudioObjectID.readProcessList() + + let updatedProcesses: [AudioProcess] = objectIdentifiers.compactMap { objectID in + do { + let proc = try AudioProcess(objectID: objectID, runningApplications: apps) + + #if DEBUG + if UserDefaults.standard.bool(forKey: "ACDumpProcessInfo") { + logger.debug("[PROCESS] \(String(describing: proc))") + } + #endif + + return proc + } catch { + logger.warning("Failed to initialize process with object ID #\(objectID, privacy: .public): \(error, privacy: .public)") + return nil + } + } + + self.processes = updatedProcesses + .sorted { + if $0.name.localizedStandardCompare($1.name) == .orderedAscending { + $1.audioActive && !$0.audioActive ? false : true + } else { + $0.audioActive && !$1.audioActive ? true : false + } + } + } catch { + logger.error("Error reading process list: \(error, privacy: .public)") + } + } + +} + +private extension AudioProcess { + init(app: NSRunningApplication, objectID: AudioObjectID) { + let name = app.localizedName ?? app.bundleURL?.deletingPathExtension().lastPathComponent ?? app.bundleIdentifier?.components(separatedBy: ".").last ?? "Unknown \(app.processIdentifier)" + + self.init( + id: app.processIdentifier, + kind: .app, + name: name, + audioActive: objectID.readProcessIsRunning(), + bundleID: app.bundleIdentifier, + bundleURL: app.bundleURL, + objectID: objectID + ) + } + + init(objectID: AudioObjectID, runningApplications apps: [NSRunningApplication]) throws { + let pid: pid_t = try objectID.read(kAudioProcessPropertyPID, defaultValue: -1) + + if let app = apps.first(where: { $0.processIdentifier == pid }) { + self.init(app: app, objectID: objectID) + } else { + try self.init(objectID: objectID, pid: pid) + } + } + + init(objectID: AudioObjectID, pid: pid_t) throws { + let bundleID = objectID.readProcessBundleID() + let bundleURL: URL? + let name: String + + (name, bundleURL) = if let info = processInfo(for: pid) { + (info.name, URL(fileURLWithPath: info.path).parentBundleURL()) + } else if let id = bundleID?.lastReverseDNSComponent { + (id, nil) + } else { + ("Unknown (\(pid))", nil) + } + + self.init( + id: pid, + kind: bundleURL?.isApp == true ? .app : .process, + name: name, + audioActive: objectID.readProcessIsRunning(), + bundleID: bundleID.flatMap { $0.isEmpty ? nil : $0 }, + bundleURL: bundleURL, + objectID: objectID + ) + } +} + +extension AudioProcessGroup { + static func groups(with processes: [AudioProcess]) -> [AudioProcessGroup] { + var byKind = [AudioProcess.Kind: AudioProcessGroup]() + + for process in processes { + byKind[process.kind, default: .init(for: process.kind)].processes.append(process) + } + + return byKind.values.sorted(by: { $0.title.localizedStandardCompare($1.title) == .orderedAscending }) + } +} + +extension AudioProcessGroup { + init(for kind: AudioProcess.Kind) { + self.init(id: kind.rawValue, title: kind.groupTitle, processes: []) + } +} + +extension AudioProcess.Kind { + var groupTitle: String { + switch self { + case .process: "Processes" + case .app: "Apps" + } + } +} + +private func processInfo(for pid: pid_t) -> (name: String, path: String)? { + let nameBuffer = UnsafeMutablePointer.allocate(capacity: Int(MAXPATHLEN)) + let pathBuffer = UnsafeMutablePointer.allocate(capacity: Int(MAXPATHLEN)) + + defer { + nameBuffer.deallocate() + pathBuffer.deallocate() + } + + let nameLength = proc_name(pid, nameBuffer, UInt32(MAXPATHLEN)) + let pathLength = proc_pidpath(pid, pathBuffer, UInt32(MAXPATHLEN)) + + guard nameLength > 0, pathLength > 0 else { + return nil + } + + let name = String(cString: nameBuffer) + let path = String(cString: pathBuffer) + + return (name, path) +} + +private extension String { + var lastReverseDNSComponent: String? { + components(separatedBy: ".").last.flatMap { $0.isEmpty ? nil : $0 } + } +} + +private extension URL { + func parentBundleURL(maxDepth: Int = 8) -> URL? { + var depth = 0 + var url = deletingLastPathComponent() + while depth < maxDepth, !url.isBundle { + url = url.deletingLastPathComponent() + depth += 1 + } + return url.isBundle ? url : nil + } + + var isBundle: Bool { + (try? resourceValues(forKeys: [.contentTypeKey]))?.contentType?.conforms(to: .bundle) == true + } + + var isApp: Bool { + (try? resourceValues(forKeys: [.contentTypeKey]))?.contentType?.conforms(to: .application) == true + } +} \ No newline at end of file diff --git a/meetingnotes/ProcessTap/AudioRecordingPermission.swift b/meetingnotes/ProcessTap/AudioRecordingPermission.swift new file mode 100644 index 0000000..6842c79 --- /dev/null +++ b/meetingnotes/ProcessTap/AudioRecordingPermission.swift @@ -0,0 +1,125 @@ +import SwiftUI +import Observation +import OSLog + +/// Uses TCC SPI in order to check/request system audio recording permission. +@Observable +final class AudioRecordingPermission { + private let logger = Logger(subsystem: "codes.rambo.AudioCap", category: String(describing: AudioRecordingPermission.self)) + + enum Status: String { + case unknown + case denied + case authorized + } + + private(set) var status: Status = .unknown + + init() { + #if ENABLE_TCC_SPI + NotificationCenter.default.addObserver(forName: NSApplication.didBecomeActiveNotification, object: nil, queue: .main) { [weak self] _ in + guard let self else { return } + self.updateStatus() + } + + updateStatus() + #else + status = .authorized + #endif // ENABLE_TCC_SPI + } + + func request() { + #if ENABLE_TCC_SPI + logger.debug(#function) + + guard let request = Self.requestSPI else { + logger.fault("Request SPI missing") + return + } + + request("kTCCServiceAudioCapture" as CFString, nil) { [weak self] granted in + guard let self else { return } + + self.logger.info("Request finished with result: \(granted, privacy: .public)") + + DispatchQueue.main.async { + if granted { + self.status = .authorized + } else { + self.status = .denied + } + } + } + #endif // ENABLE_TCC_SPI + } + + private func updateStatus() { + #if ENABLE_TCC_SPI + logger.debug(#function) + + guard let preflight = Self.preflightSPI else { + logger.fault("Preflight SPI missing") + return + } + + let result = preflight("kTCCServiceAudioCapture" as CFString, nil) + + if result == 1 { + status = .denied + } else if result == 0 { + status = .authorized + } else { + status = .unknown + } + #endif // ENABLE_TCC_SPI + } + + #if ENABLE_TCC_SPI + private typealias PreflightFuncType = @convention(c) (CFString, CFDictionary?) -> Int + private typealias RequestFuncType = @convention(c) (CFString, CFDictionary?, @escaping (Bool) -> Void) -> Void + + /// `dlopen` handle to the TCC framework. + private static let apiHandle: UnsafeMutableRawPointer? = { + let tccPath = "/System/Library/PrivateFrameworks/TCC.framework/Versions/A/TCC" + + guard let handle = dlopen(tccPath, RTLD_NOW) else { + assertionFailure("dlopen failed") + return nil + } + + return handle + }() + + /// `dlsym` function handle for `TCCAccessPreflight`. + private static let preflightSPI: PreflightFuncType? = { + guard let apiHandle else { return nil } + + let fnName = "TCCAccessPreflight" + + guard let funcSym = dlsym(apiHandle, fnName) else { + assertionFailure("Couldn't find symbol") + return nil + } + + let fn = unsafeBitCast(funcSym, to: PreflightFuncType.self) + + return fn + }() + + /// `dlsym` function handle for `TCCAccessRequest`. + private static let requestSPI: RequestFuncType? = { + guard let apiHandle else { return nil } + + let fnName = "TCCAccessRequest" + + guard let funcSym = dlsym(apiHandle, fnName) else { + assertionFailure("Couldn't find symbol") + return nil + } + + let fn = unsafeBitCast(funcSym, to: RequestFuncType.self) + + return fn + }() + #endif // ENABLE_TCC_SPI +} \ No newline at end of file diff --git a/meetingnotes/ProcessTap/CoreAudioUtils.swift b/meetingnotes/ProcessTap/CoreAudioUtils.swift new file mode 100644 index 0000000..6baa3b0 --- /dev/null +++ b/meetingnotes/ProcessTap/CoreAudioUtils.swift @@ -0,0 +1,310 @@ +import Foundation +import AudioToolbox + +// MARK: - Constants + +extension AudioObjectID { + /// Convenience for `kAudioObjectSystemObject`. + static let system = AudioObjectID(kAudioObjectSystemObject) + /// Convenience for `kAudioObjectUnknown`. + static let unknown = kAudioObjectUnknown + + /// `true` if this object has the value of `kAudioObjectUnknown`. + var isUnknown: Bool { self == .unknown } + + /// `false` if this object has the value of `kAudioObjectUnknown`. + var isValid: Bool { !isUnknown } +} + +// MARK: - Concrete Property Helpers + +extension AudioObjectID { + /// Reads the value for `kAudioHardwarePropertyDefaultSystemOutputDevice`. + static func readDefaultSystemOutputDevice() throws -> AudioDeviceID { + try AudioObjectID.system.readDefaultSystemOutputDevice() + } + + static func readProcessList() throws -> [AudioObjectID] { + try AudioObjectID.system.readProcessList() + } + + /// Reads `kAudioHardwarePropertyTranslatePIDToProcessObject` for the specific pid. + static func translatePIDToProcessObjectID(pid: pid_t) throws -> AudioObjectID { + try AudioObjectID.system.translatePIDToProcessObjectID(pid: pid) + } + + /// Reads `kAudioHardwarePropertyProcessObjectList`. + func readProcessList() throws -> [AudioObjectID] { + try requireSystemObject() + + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyProcessObjectList, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + + var dataSize: UInt32 = 0 + + var err = AudioObjectGetPropertyDataSize(self, &address, 0, nil, &dataSize) + + guard err == noErr else { throw "Error reading data size for \(address): \(err)" } + + var value = [AudioObjectID](repeating: .unknown, count: Int(dataSize) / MemoryLayout.size) + + err = AudioObjectGetPropertyData(self, &address, 0, nil, &dataSize, &value) + + guard err == noErr else { throw "Error reading array for \(address): \(err)" } + + return value + } + + /// Reads `kAudioHardwarePropertyTranslatePIDToProcessObject` for the specific pid, should only be called on the system object. + func translatePIDToProcessObjectID(pid: pid_t) throws -> AudioObjectID { + try requireSystemObject() + + let processObject = try read( + kAudioHardwarePropertyTranslatePIDToProcessObject, + defaultValue: AudioObjectID.unknown, + qualifier: pid + ) + + guard processObject.isValid else { + throw "Invalid process identifier: \(pid)" + } + + return processObject + } + + func readProcessBundleID() -> String? { + if let result = try? readString(kAudioProcessPropertyBundleID) { + result.isEmpty ? nil : result + } else { + nil + } + } + + func readProcessIsRunning() -> Bool { + (try? readBool(kAudioProcessPropertyIsRunning)) ?? false + } + + /* + public var kAudioProcessPropertyPID: AudioObjectPropertySelector { get } + + public var kAudioProcessPropertyBundleID: AudioObjectPropertySelector { get } + + public var kAudioProcessPropertyDevices: AudioObjectPropertySelector { get } + + public var kAudioProcessPropertyIsRunning: AudioObjectPropertySelector { get } + + public var kAudioProcessPropertyIsRunningInput: AudioObjectPropertySelector { get } + + public var kAudioProcessPropertyIsRunningOutput: AudioObjectPropertySelector { get } + */ + + /// Reads the value for `kAudioHardwarePropertyDefaultSystemOutputDevice`, should only be called on the system object. + func readDefaultSystemOutputDevice() throws -> AudioDeviceID { + try requireSystemObject() + + return try read(kAudioHardwarePropertyDefaultSystemOutputDevice, defaultValue: AudioDeviceID.unknown) + } + + /// Reads the value for `kAudioDevicePropertyDeviceUID` for the device represented by this audio object ID. + func readDeviceUID() throws -> String { try readString(kAudioDevicePropertyDeviceUID) } + + /// Reads the value for `kAudioTapPropertyFormat` for the device represented by this audio object ID. + func readAudioTapStreamBasicDescription() throws -> AudioStreamBasicDescription { + try read(kAudioTapPropertyFormat, defaultValue: AudioStreamBasicDescription()) + } + + private func requireSystemObject() throws { + if self != .system { throw "Only supported for the system object." } + } +} + +// MARK: - Generic Property Access + +extension AudioObjectID { + func read(_ selector: AudioObjectPropertySelector, + scope: AudioObjectPropertyScope = kAudioObjectPropertyScopeGlobal, + element: AudioObjectPropertyElement = kAudioObjectPropertyElementMain, + defaultValue: T, + qualifier: Q) throws -> T + { + try read(AudioObjectPropertyAddress(mSelector: selector, mScope: scope, mElement: element), defaultValue: defaultValue, qualifier: qualifier) + } + + func read(_ selector: AudioObjectPropertySelector, + scope: AudioObjectPropertyScope = kAudioObjectPropertyScopeGlobal, + element: AudioObjectPropertyElement = kAudioObjectPropertyElementMain, + defaultValue: T) throws -> T + { + try read(AudioObjectPropertyAddress(mSelector: selector, mScope: scope, mElement: element), defaultValue: defaultValue) + } + + func read(_ address: AudioObjectPropertyAddress, defaultValue: T, qualifier: Q) throws -> T { + var inQualifier = qualifier + let qualifierSize = UInt32(MemoryLayout.size(ofValue: qualifier)) + return try withUnsafeMutablePointer(to: &inQualifier) { qualifierPtr in + try read(address, defaultValue: defaultValue, inQualifierSize: qualifierSize, inQualifierData: qualifierPtr) + } + } + + func read(_ address: AudioObjectPropertyAddress, defaultValue: T) throws -> T { + try read(address, defaultValue: defaultValue, inQualifierSize: 0, inQualifierData: nil) + } + + func readString(_ selector: AudioObjectPropertySelector, scope: AudioObjectPropertyScope = kAudioObjectPropertyScopeGlobal, element: AudioObjectPropertyElement = kAudioObjectPropertyElementMain) throws -> String { + try read(AudioObjectPropertyAddress(mSelector: selector, mScope: scope, mElement: element), defaultValue: "" as CFString) as String + } + + func readBool(_ selector: AudioObjectPropertySelector, scope: AudioObjectPropertyScope = kAudioObjectPropertyScopeGlobal, element: AudioObjectPropertyElement = kAudioObjectPropertyElementMain) throws -> Bool { + let value: Int = try read(AudioObjectPropertyAddress(mSelector: selector, mScope: scope, mElement: element), defaultValue: 0) + return value == 1 + } + + private func read(_ inAddress: AudioObjectPropertyAddress, defaultValue: T, inQualifierSize: UInt32 = 0, inQualifierData: UnsafeRawPointer? = nil) throws -> T { + var address = inAddress + + var dataSize: UInt32 = 0 + + var err = AudioObjectGetPropertyDataSize(self, &address, inQualifierSize, inQualifierData, &dataSize) + + guard err == noErr else { + throw "Error reading data size for \(inAddress): \(err)" + } + + var value: T = defaultValue + err = withUnsafeMutablePointer(to: &value) { ptr in + AudioObjectGetPropertyData(self, &address, inQualifierSize, inQualifierData, &dataSize, ptr) + } + + guard err == noErr else { + throw "Error reading data for \(inAddress): \(err)" + } + + return value + } +} + +// MARK: - Debugging Helpers + +private extension UInt32 { + var fourCharString: String { + String(cString: [ + UInt8((self >> 24) & 0xFF), + UInt8((self >> 16) & 0xFF), + UInt8((self >> 8) & 0xFF), + UInt8(self & 0xFF), + 0 + ]) + } +} + +extension AudioObjectPropertyAddress: @retroactive CustomStringConvertible { + public var description: String { + let elementDescription = mElement == kAudioObjectPropertyElementMain ? "main" : mElement.fourCharString + return "\(mSelector.fourCharString)/\(mScope.fourCharString)/\(elementDescription)" + } +} + +public struct AudioInputDevice: Identifiable, Hashable { + public let id: AudioDeviceID // AudioDeviceID (UInt32) is Hashable and can serve as Identifiable's id. + public let uid: String // Unique identifier for the device (persistent across reboots) + public let name: String + + public init(id: AudioDeviceID, uid: String, name: String) { + self.id = id + self.uid = uid + self.name = name + } +} + +extension AudioObjectID { + static func getAllInputDevices() throws -> [AudioInputDevice] { + let allDeviceIDs = try AudioObjectID.system.getAllHardwareDevices() + var inputDevices: [AudioInputDevice] = [] + + for deviceID in allDeviceIDs { + do { + let inputChannelCount = try deviceID.getTotalInputChannelCount() + if inputChannelCount > 0 { + let deviceName = try deviceID.getDeviceName() + let deviceUID = try deviceID.readDeviceUID() + inputDevices.append(AudioInputDevice(id: deviceID, uid: deviceUID, name: deviceName)) + } + } catch { + print("CoreAudioUtils: Could not fully query device \(deviceID): \(error)") + } + } + return inputDevices + } + + func getAllHardwareDevices() throws -> [AudioDeviceID] { + try requireSystemObject() + + var address = AudioObjectPropertyAddress( + mSelector: kAudioHardwarePropertyDevices, + mScope: kAudioObjectPropertyScopeGlobal, + mElement: kAudioObjectPropertyElementMain + ) + + var dataSize: UInt32 = 0 + var err = AudioObjectGetPropertyDataSize(self, &address, 0, nil, &dataSize) + guard err == noErr else { + throw "CoreAudioUtils: Error reading data size for \(kAudioHardwarePropertyDevices.fourCharString): \(err)" + } + + let deviceCount = Int(dataSize) / MemoryLayout.size + if deviceCount == 0 { + return [] + } + var deviceIDs = [AudioDeviceID](repeating: .unknown, count: deviceCount) + + err = AudioObjectGetPropertyData(self, &address, 0, nil, &dataSize, &deviceIDs) + guard err == noErr else { + throw "CoreAudioUtils: Error reading device array for \(kAudioHardwarePropertyDevices.fourCharString): \(err)" + } + + return deviceIDs + } + + func getTotalInputChannelCount() throws -> UInt32 { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreamConfiguration, + mScope: kAudioDevicePropertyScopeInput, + mElement: kAudioObjectPropertyElementMain + ) + + var dataSize: UInt32 = 0 + var err = AudioObjectGetPropertyDataSize(self, &address, 0, nil, &dataSize) + + if err == kAudioHardwareUnknownPropertyError || dataSize == 0 { + return 0 + } + guard err == noErr else { + throw "CoreAudioUtils: Error reading data size for input stream configuration on device \(self): \(err)" + } + + let bufferListPtr = UnsafeMutableRawPointer.allocate(byteCount: Int(dataSize), alignment: MemoryLayout.alignment) + defer { bufferListPtr.deallocate() } + + err = AudioObjectGetPropertyData(self, &address, 0, nil, &dataSize, bufferListPtr) + guard err == noErr else { + throw "CoreAudioUtils: Error reading input stream configuration for device \(self): \(err)" + } + + let audioBufferList = bufferListPtr.assumingMemoryBound(to: AudioBufferList.self) + var totalInputChannels: UInt32 = 0 + + let buffers = UnsafeMutableAudioBufferListPointer(audioBufferList) + for i in 0.. String { + return try readString(kAudioDevicePropertyDeviceNameCFString) + } +} \ No newline at end of file diff --git a/meetingnotes/ProcessTap/ProcessTap.swift b/meetingnotes/ProcessTap/ProcessTap.swift new file mode 100644 index 0000000..e7a1fad --- /dev/null +++ b/meetingnotes/ProcessTap/ProcessTap.swift @@ -0,0 +1,499 @@ +import SwiftUI +import AudioToolbox +import OSLog +import AVFoundation + +enum TapTarget { + case singleProcess(AudioProcess) + case systemAudio(processObjectIDs: [AudioObjectID]) + + var displayName: String { + switch self { + case .singleProcess(let process): + return process.name + case .systemAudio: + return "System Audio Output" + } + } + + var iconImage: NSImage { + switch self { + case .singleProcess(let process): + return process.icon + case .systemAudio: + let genericAppIcon = NSWorkspace.shared.icon(for: .applicationBundle) + genericAppIcon.size = NSSize(width: 32, height: 32) + return genericAppIcon + } + } + + var loggingProcessName: String { + switch self { + case .singleProcess(let process): + return process.name + case .systemAudio: + return "SystemAudioOutput" + } + } +} + +@Observable +final class ProcessTap { + + typealias InvalidationHandler = (ProcessTap) -> Void + + let target: TapTarget + let muteWhenRunning: Bool + private let logger: Logger + + private(set) var errorMessage: String? = nil + + init(target: TapTarget, muteWhenRunning: Bool = false) { + self.target = target + self.muteWhenRunning = muteWhenRunning + self.logger = Logger(subsystem: "codes.rambo.AudioCap", category: "\(String(describing: ProcessTap.self))(\(target.loggingProcessName))") + } + + @ObservationIgnored + private var processTapID: AudioObjectID = .unknown + @ObservationIgnored + private var aggregateDeviceID = AudioObjectID.unknown + @ObservationIgnored + private var deviceProcID: AudioDeviceIOProcID? + @ObservationIgnored + private(set) var tapStreamDescription: AudioStreamBasicDescription? + @ObservationIgnored + private var invalidationHandler: InvalidationHandler? + + @ObservationIgnored + private(set) var activated = false + + var displayName: String { + target.displayName + } + + @MainActor + func activate() { + guard !activated else { return } + activated = true + + logger.debug(#function) + self.errorMessage = nil + + do { + try prepare() + } catch { + logger.error("\(error, privacy: .public)") + self.errorMessage = error.localizedDescription + } + } + + func invalidate() { + guard activated else { return } + defer { activated = false } + + logger.debug(#function) + + invalidationHandler?(self) + self.invalidationHandler = nil + + if aggregateDeviceID.isValid { + var err: OSStatus + + err = AudioDeviceStop(aggregateDeviceID, deviceProcID) + if err != noErr { + logger.warning("Failed to stop aggregate device: \(err, privacy: .public)") + } + + if let deviceProcID { + err = AudioDeviceDestroyIOProcID(aggregateDeviceID, deviceProcID) + if err != noErr { + logger.warning("Failed to destroy device I/O proc: \(err, privacy: .public)") + } + self.deviceProcID = nil + } + + err = AudioHardwareDestroyAggregateDevice(aggregateDeviceID) + if err != noErr { + logger.warning("Failed to destroy aggregate device: \(err, privacy: .public)") + } + aggregateDeviceID = .unknown + } + + if processTapID.isValid { + let errTapDestroy = AudioHardwareDestroyProcessTap(processTapID) + if errTapDestroy != noErr { + logger.warning("Failed to destroy audio tap: \(errTapDestroy, privacy: .public)") + } + self.processTapID = .unknown + } + } + + private func prepare() throws { + errorMessage = nil + + let tapDescription: CATapDescription + switch self.target { + case .singleProcess(let process): + tapDescription = CATapDescription(stereoMixdownOfProcesses: [process.objectID]) + logger.debug("Configuring tap for single process objectID: \(process.objectID)") + case .systemAudio(let processObjectIDs): + if processObjectIDs.isEmpty { + logger.warning("System audio tap configured with an empty list of processObjectIDs. This might not capture any audio or behave unexpectedly.") + } + tapDescription = CATapDescription(stereoMixdownOfProcesses: processObjectIDs) + logger.debug("Configuring tap for system audio output using \(processObjectIDs.count) explicit processes.") + } + + tapDescription.uuid = UUID() + tapDescription.muteBehavior = muteWhenRunning ? .mutedWhenTapped : .unmuted + var tapID: AUAudioObjectID = .unknown + let errTapCreation = AudioHardwareCreateProcessTap(tapDescription, &tapID) + + guard errTapCreation == noErr else { + errorMessage = "Process/System tap creation failed with error \(errTapCreation)" + throw errorMessage ?? "Unknown error creating tap." + } + + logger.debug("Created process/system tap #\(tapID, privacy: .public). Associated UUID: \(tapDescription.uuid.uuidString)") + self.processTapID = tapID + + let allDeviceIDs = try AudioObjectID.system.getAllHardwareDevices() + var outputUIDs: [String] = [] + var outputDeviceIDs: [AudioDeviceID] = [] + for devID in allDeviceIDs { + do { + let outputChans = try devID.getTotalOutputChannelCount() + if outputChans > 0 { + let devUID = try devID.readDeviceUID() + outputUIDs.append(devUID) + outputDeviceIDs.append(devID) + } + } catch { + logger.warning("Ignored device \(devID): \(error.localizedDescription)") + } + } + + if outputUIDs.isEmpty { + throw "No hardware output devices found!" + } + + let systemOutputID: AudioDeviceID + do { + logger.debug("Attempting to read default system output device ID...") + systemOutputID = try AudioDeviceID.readDefaultSystemOutputDevice() + logger.debug("Successfully read default system output device ID: \(systemOutputID)") + } catch { + logger.error("Failed to read default system output device ID: \(error)") + throw error // Propagate error + } + + let mainSubdeviceUID: String + do { + logger.debug("Attempting to read device UID for systemOutputID: \(systemOutputID)...") + mainSubdeviceUID = try systemOutputID.readDeviceUID() + logger.debug("Successfully read mainSubdeviceUID: \(mainSubdeviceUID)") + } catch { + logger.error("Failed to read device UID for systemOutputID \(systemOutputID): \(error)") + throw error // Propagate error + } + + let subDeviceListForAggregate: [[String: Any]] + let aggregateDeviceName: String + let aggregateUID = UUID().uuidString + + switch self.target { + case .systemAudio: + aggregateDeviceName = "Tap-SysAgg-\(mainSubdeviceUID.prefix(8))" + subDeviceListForAggregate = [ + [kAudioSubDeviceUIDKey: mainSubdeviceUID] + ] + logger.debug("System mode: mainSubdeviceUID for aggregate: \(mainSubdeviceUID). Aggregate name: \(aggregateDeviceName)") + case .singleProcess: + aggregateDeviceName = "Tap-\(self.displayName)-Agg" + subDeviceListForAggregate = outputUIDs.map { [kAudioSubDeviceUIDKey: $0] } + logger.debug("Process mode: Aggregate subDeviceList from outputUIDs. Aggregate name: \(aggregateDeviceName)") + } + + let descriptionForAggregate: [String: Any] = [ + kAudioAggregateDeviceNameKey: aggregateDeviceName, + kAudioAggregateDeviceUIDKey: aggregateUID, + kAudioAggregateDeviceMainSubDeviceKey: mainSubdeviceUID, + kAudioAggregateDeviceIsPrivateKey: true, + kAudioAggregateDeviceIsStackedKey: false, + kAudioAggregateDeviceTapAutoStartKey: true, + kAudioAggregateDeviceSubDeviceListKey: subDeviceListForAggregate, + kAudioAggregateDeviceTapListKey: [ + [ + kAudioSubTapDriftCompensationKey: true, + kAudioSubTapUIDKey: tapDescription.uuid.uuidString + ] + ] + ] + logger.debug("Aggregate device description prepared. Main sub-device UID: \(mainSubdeviceUID), Tap UUID: \(tapDescription.uuid.uuidString)") + + aggregateDeviceID = AudioObjectID.unknown + do { + logger.debug("Calling AudioHardwareCreateAggregateDevice...") + let errAggDeviceCreation = AudioHardwareCreateAggregateDevice(descriptionForAggregate as CFDictionary, &aggregateDeviceID) + if errAggDeviceCreation != noErr { + logger.error("AudioHardwareCreateAggregateDevice failed with error: \(errAggDeviceCreation).") + throw "Failed to create aggregate device: \(errAggDeviceCreation)" + } + logger.debug("Successfully created aggregate device #\(self.aggregateDeviceID, privacy: .public)") + } catch { + logger.error("EXCEPTION during AudioHardwareCreateAggregateDevice block: \(error)") + throw error // Propagate error + } + + do { + logger.debug("Attempting to read audio tap stream basic description for tapID #\(tapID)...") + self.tapStreamDescription = try tapID.readAudioTapStreamBasicDescription() + logger.debug("Successfully read tap stream description: \(String(describing: self.tapStreamDescription))") + } catch { + logger.error("Failed to read audio tap stream basic description for tapID #\(tapID): \(error)") + throw error // Propagate error + } + } + + func run(on queue: DispatchQueue, ioBlock: @escaping AudioDeviceIOBlock, invalidationHandler: @escaping InvalidationHandler) throws { + assert(activated, "\(#function) called with inactive tap!") + assert(self.invalidationHandler == nil, "\(#function) called with tap already active!") + + errorMessage = nil + logger.debug("Run tap!") + self.invalidationHandler = invalidationHandler + + var err = AudioDeviceCreateIOProcIDWithBlock(&deviceProcID, aggregateDeviceID, queue, ioBlock) + guard err == noErr else { throw "Failed to create device I/O proc: \(err)" } + + err = AudioDeviceStart(aggregateDeviceID, deviceProcID) + guard err == noErr else { throw "Failed to start audio device: \(err)" } + } + + deinit { invalidate() } + +} + +private extension AudioDeviceID { + func getTotalOutputChannelCount() throws -> UInt32 { + var address = AudioObjectPropertyAddress( + mSelector: kAudioDevicePropertyStreamConfiguration, + mScope: kAudioDevicePropertyScopeOutput, + mElement: kAudioObjectPropertyElementMain + ) + var dataSize: UInt32 = 0 + var err = AudioObjectGetPropertyDataSize(self, &address, 0, nil, &dataSize) + if err == kAudioHardwareUnknownPropertyError || dataSize == 0 { + return 0 + } + guard err == noErr else { + throw "Error reading data size for output stream configuration: \(err)" + } + let bufferListPtr = UnsafeMutableRawPointer.allocate(byteCount: Int(dataSize), alignment: MemoryLayout.alignment) + defer { bufferListPtr.deallocate() } + err = AudioObjectGetPropertyData(self, &address, 0, nil, &dataSize, bufferListPtr) + guard err == noErr else { + throw "Error reading output stream configuration: \(err)" + } + let audioBufferList = bufferListPtr.assumingMemoryBound(to: AudioBufferList.self) + var totalOutputChannels: UInt32 = 0 + let buffers = UnsafeMutableAudioBufferListPointer(audioBufferList) + for i in 0.. 0 { + let channelData = floatChannelData[0] + let frameLength = Int(buffer.frameLength) + var sumOfSquares: Float = 0.0 + for i in 0..