feat: proper system audio recording instead of screen recording (#36)

This commit is contained in:
Owen Gretzinger
2025-07-23 23:14:32 -04:00
committed by GitHub
parent d496ca3285
commit d537810f1b
7 changed files with 1297 additions and 102 deletions
+2 -2
View File
@@ -293,7 +293,7 @@
"$(inherited)", "$(inherited)",
"@executable_path/../Frameworks", "@executable_path/../Frameworks",
); );
MACOSX_DEPLOYMENT_TARGET = 14.0; MACOSX_DEPLOYMENT_TARGET = 14.4;
MARKETING_VERSION = 1.0.6; MARKETING_VERSION = 1.0.6;
ONLY_ACTIVE_ARCH = NO; ONLY_ACTIVE_ARCH = NO;
PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes; PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes;
@@ -328,7 +328,7 @@
"$(inherited)", "$(inherited)",
"@executable_path/../Frameworks", "@executable_path/../Frameworks",
); );
MACOSX_DEPLOYMENT_TARGET = 14.0; MACOSX_DEPLOYMENT_TARGET = 14.4;
MARKETING_VERSION = 1.0.6; MARKETING_VERSION = 1.0.6;
ONLY_ACTIVE_ARCH = YES; ONLY_ACTIVE_ARCH = YES;
PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes; PRODUCT_BUNDLE_IDENTIFIER = owen.meetingnotes;
+2 -2
View File
@@ -4,8 +4,8 @@
<dict> <dict>
<key>NSAudioCaptureUsageDescription</key> <key>NSAudioCaptureUsageDescription</key>
<string>Meetingnotes needs access to capture system audio for transcription.</string> <string>Meetingnotes needs access to capture system audio for transcription.</string>
<key>NSScreenCaptureUsageDescription</key> <key>NSMicrophoneUsageDescription</key>
<string>Meetingnotes needs screen recording permission to capture system audio for transcription.</string> <string>Meetingnotes needs access to your microphone for transcription.</string>
<key>SUFeedURL</key> <key>SUFeedURL</key>
<string>https://raw.githubusercontent.com/owengretzinger/meetingnotes/main/appcast.xml</string> <string>https://raw.githubusercontent.com/owengretzinger/meetingnotes/main/appcast.xml</string>
<key>SUPublicEDKey</key> <key>SUPublicEDKey</key>
+120 -98
View File
@@ -4,9 +4,10 @@
import AVFoundation import AVFoundation
import Foundation import Foundation
import SwiftUI import SwiftUI
import ScreenCaptureKit import OSLog
/// Manages audio capture from microphone and system audio and handles real-time transcription via OpenAI /// Manages audio capture from microphone and system audio and handles real-time transcription via OpenAI
@MainActor
class AudioManager: NSObject, ObservableObject { class AudioManager: NSObject, ObservableObject {
@Published var transcriptChunks: [TranscriptChunk] = [] @Published var transcriptChunks: [TranscriptChunk] = []
@Published var isRecording = false @Published var isRecording = false
@@ -20,8 +21,12 @@ class AudioManager: NSObject, ObservableObject {
// Unique identifier for the current recording session // Unique identifier for the current recording session
private var sessionID = UUID() private var sessionID = UUID()
// ScreenCaptureKit properties // ProcessTap properties
private var stream: SCStream? private var processTap: ProcessTap?
private let audioProcessController = AudioProcessController()
private let permission = AudioRecordingPermission()
private let tapQueue = DispatchQueue(label: "io.meetingnotes.audiotap", qos: .userInitiated)
private var isTapActive = false
// Add properties near the top, after existing private vars // Add properties near the top, after existing private vars
private var micRetryCount = 0 private var micRetryCount = 0
@@ -40,6 +45,9 @@ class AudioManager: NSObject, ObservableObject {
queue: .main) { [weak self] _ in queue: .main) { [weak self] _ in
self?.handleAudioEngineConfigurationChange() self?.handleAudioEngineConfigurationChange()
} }
// Activate the process controller to start monitoring audio-producing apps
audioProcessController.activate()
} }
deinit { deinit {
@@ -66,7 +74,7 @@ class AudioManager: NSObject, ObservableObject {
self.startMicrophoneTap() self.startMicrophoneTap()
// Start system audio capture asynchronously // Start system audio capture asynchronously
Task { Task {
await self.startSystemAudioCapture() await self.startSystemAudioTap()
} }
} }
} }
@@ -75,10 +83,11 @@ class AudioManager: NSObject, ObservableObject {
print("Internal cleanup...") print("Internal cleanup...")
// Stop system audio capture // Stop system audio capture
if let stream = stream { if isTapActive {
stream.stopCapture() self.processTap?.invalidate()
self.stream = nil self.processTap = nil
print("System audio capture stopped") isTapActive = false
print("System audio tap invalidated")
} }
// Stop microphone capture // Stop microphone capture
@@ -195,61 +204,112 @@ class AudioManager: NSObject, ObservableObject {
print("✨ Fresh audio engine created") print("✨ Fresh audio engine created")
} }
private func startSystemAudioCapture() async { private func startSystemAudioTap() async {
print("🎧 Starting system audio capture...") print("🎧 Starting system audio tap...")
// Ensure we have permission to record system audio. This might prompt the user.
guard await checkSystemAudioPermissions() else {
let errorMsg = "System audio recording permission denied."
print("\(errorMsg)")
DispatchQueue.main.async {
self.errorMessage = errorMsg
}
return
}
// Get all running processes that are producing audio
let allProcessObjectIDs = audioProcessController.processes.map { $0.objectID }
if allProcessObjectIDs.isEmpty {
print("⚠️ No audio-producing processes found. System audio tap might not capture anything.")
}
// Configure the tap for system-wide audio
let target = TapTarget.systemAudio(processObjectIDs: allProcessObjectIDs)
let newTap = ProcessTap(target: target)
newTap.activate()
// Check for activation errors
if let tapError = newTap.errorMessage {
let errorMsg = "Failed to activate system audio tap: \(tapError)"
print("\(errorMsg)")
DispatchQueue.main.async {
self.errorMessage = errorMsg
}
return
}
self.processTap = newTap
self.isTapActive = true
// Start receiving audio data from the tap
do { do {
// Request screen capture permission try startTapIO(newTap)
let content = try await SCShareableContent.excludingDesktopWindows(true, onScreenWindowsOnly: true) connectToOpenAIRealtime(source: .system)
print("✅ System audio tap started successfully")
// Exclude self to avoid feedback
let excludedApps = content.applications.filter { app in
Bundle.main.bundleIdentifier == app.bundleIdentifier
}
guard let display = content.displays.first else {
print("❌ No display found")
return
}
// Create filter
let filter = SCContentFilter(display: display, excludingApplications: excludedApps, exceptingWindows: [])
// Configure stream
let configuration = SCStreamConfiguration()
configuration.width = 2 // Minimal video settings
configuration.height = 2
configuration.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale.max)
configuration.capturesAudio = true
configuration.sampleRate = 48000
configuration.channelCount = 2
// Create stream
let stream = SCStream(filter: filter, configuration: configuration, delegate: self)
// Add stream output for audio processing
try stream.addStreamOutput(self, type: .audio, sampleHandlerQueue: .global(qos: .userInitiated))
// Add a minimal screen output so SCStream doesn't complain about missing video output
try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: .global(qos: .userInitiated))
// Start capture
try await stream.startCapture()
// Store reference
self.stream = stream
DispatchQueue.main.async { DispatchQueue.main.async {
self.isRecording = true self.isRecording = true
} }
connectToOpenAIRealtime(source: .system)
print("✅ System audio capture started successfully")
} catch { } catch {
print("Failed to start system audio capture: \(error)") let errorMsg = "Failed to start system audio tap IO: \(error.localizedDescription)"
print("\(errorMsg)")
DispatchQueue.main.async {
self.errorMessage = errorMsg
}
newTap.invalidate()
self.isTapActive = false
}
}
if case SCStreamError.userDeclined = error { @MainActor
print("📍 Permission denied. User needs to enable screen recording in System Settings.") private func checkSystemAudioPermissions() async -> Bool {
if permission.status == .authorized {
return true
}
permission.request()
// Poll for a short time to see if permission is granted
for _ in 0..<10 {
if permission.status == .authorized {
return true
}
try? await Task.sleep(nanoseconds: 500_000_000) // 0.5 seconds
}
return permission.status == .authorized
}
private func startTapIO(_ tap: ProcessTap) throws {
guard var streamDescription = tap.tapStreamDescription else {
throw NSError(domain: "AudioManager", code: -1, userInfo: [NSLocalizedDescriptionKey: "Failed to get audio format from tap."])
}
guard let format = AVAudioFormat(streamDescription: &streamDescription) else {
throw NSError(domain: "AudioManager", code: -1, userInfo: [NSLocalizedDescriptionKey: "Failed to create AVAudioFormat from tap."])
}
try tap.run(on: tapQueue) { [weak self] _, inInputData, _, _, _ in
guard let self = self,
let buffer = AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: inInputData, deallocator: nil) else {
return
}
let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
sampleRate: 24000,
channels: 1,
interleaved: false)!
guard let converter = AVAudioConverter(from: format, to: targetFormat) else {
return
}
self.processAudioBuffer(buffer, converter: converter, targetFormat: targetFormat, source: .system)
} invalidationHandler: { [weak self] _ in
print("Audio tap was invalidated.")
DispatchQueue.main.async {
self?.stopRecording()
} }
} }
} }
@@ -260,9 +320,11 @@ class AudioManager: NSObject, ObservableObject {
print("Stopping recording...") print("Stopping recording...")
// Stop system audio capture // Stop system audio capture
if let stream = stream { if isTapActive {
stream.stopCapture() self.processTap?.invalidate()
self.stream = nil self.processTap = nil
isTapActive = false
print("System audio tap invalidated")
} }
// Stop microphone capture // Stop microphone capture
@@ -558,43 +620,3 @@ class AudioManager: NSObject, ObservableObject {
restartMicrophone() restartMicrophone()
} }
} }
// MARK: - SCStreamDelegate & SCStreamOutput
extension AudioManager: SCStreamDelegate, SCStreamOutput {
func stream(_ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType) {
guard type == .audio else { return }
guard sampleBuffer.isValid else { return }
// Convert CMSampleBuffer to AVAudioPCMBuffer
guard let pcmBuffer = sampleBuffer.asPCMBuffer else { return }
// Create converter for OpenAI format
let targetFormat = AVAudioFormat(commonFormat: .pcmFormatInt16,
sampleRate: 24000,
channels: 1,
interleaved: false)!
guard let converter = AVAudioConverter(from: pcmBuffer.format, to: targetFormat) else { return }
processAudioBuffer(pcmBuffer, converter: converter, targetFormat: targetFormat, source: .system)
}
func stream(_ stream: SCStream, didStopWithError error: Error) {
print("❌ Stream stopped with error: \(error)")
DispatchQueue.main.async {
self.stream = nil
self.isRecording = false
}
}
}
// MARK: - CMSampleBuffer Extension
extension CMSampleBuffer {
var asPCMBuffer: AVAudioPCMBuffer? {
try? self.withAudioBufferList { audioBufferList, _ -> AVAudioPCMBuffer? in
guard let absd = self.formatDescription?.audioStreamBasicDescription else { return nil }
guard let format = AVAudioFormat(standardFormatWithSampleRate: absd.mSampleRate, channels: absd.mChannelsPerFrame) else { return nil }
return AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: audioBufferList.unsafePointer)
}
}
}
@@ -0,0 +1,239 @@
import SwiftUI
import AudioToolbox
import OSLog
import Combine
struct AudioProcess: Identifiable, Hashable, Sendable {
enum Kind: String, Sendable {
case process
case app
}
var id: pid_t
var kind: Kind
var name: String
var audioActive: Bool
var bundleID: String?
var bundleURL: URL?
var objectID: AudioObjectID
}
struct AudioProcessGroup: Identifiable, Hashable, Sendable {
var id: String
var title: String
var processes: [AudioProcess]
}
extension AudioProcess.Kind {
var defaultIcon: NSImage {
switch self {
case .process: NSWorkspace.shared.icon(for: .unixExecutable)
case .app: NSWorkspace.shared.icon(for: .applicationBundle)
}
}
}
extension AudioProcess {
var icon: NSImage {
guard let bundleURL else { return kind.defaultIcon }
let image = NSWorkspace.shared.icon(forFile: bundleURL.path)
image.size = NSSize(width: 32, height: 32)
return image
}
}
extension String: @retroactive LocalizedError {
public var errorDescription: String? { self }
}
@MainActor
@Observable
final class AudioProcessController {
private let logger = Logger(subsystem: "codes.rambo.AudioCap", category: String(describing: AudioProcessController.self))
private(set) var processes = [AudioProcess]() {
didSet {
guard processes != oldValue else { return }
processGroups = AudioProcessGroup.groups(with: processes)
}
}
private(set) var processGroups = [AudioProcessGroup]()
private var cancellables = Set<AnyCancellable>()
func activate() {
logger.debug(#function)
NSWorkspace.shared
.publisher(for: \.runningApplications, options: [.initial, .new])
.map { $0.filter({ $0.processIdentifier != ProcessInfo.processInfo.processIdentifier }) }
.sink { [weak self] apps in
guard let self else { return }
self.reload(apps: apps)
}
.store(in: &cancellables)
}
fileprivate func reload(apps: [NSRunningApplication]) {
logger.debug(#function)
do {
let objectIdentifiers = try AudioObjectID.readProcessList()
let updatedProcesses: [AudioProcess] = objectIdentifiers.compactMap { objectID in
do {
let proc = try AudioProcess(objectID: objectID, runningApplications: apps)
#if DEBUG
if UserDefaults.standard.bool(forKey: "ACDumpProcessInfo") {
logger.debug("[PROCESS] \(String(describing: proc))")
}
#endif
return proc
} catch {
logger.warning("Failed to initialize process with object ID #\(objectID, privacy: .public): \(error, privacy: .public)")
return nil
}
}
self.processes = updatedProcesses
.sorted {
if $0.name.localizedStandardCompare($1.name) == .orderedAscending {
$1.audioActive && !$0.audioActive ? false : true
} else {
$0.audioActive && !$1.audioActive ? true : false
}
}
} catch {
logger.error("Error reading process list: \(error, privacy: .public)")
}
}
}
private extension AudioProcess {
init(app: NSRunningApplication, objectID: AudioObjectID) {
let name = app.localizedName ?? app.bundleURL?.deletingPathExtension().lastPathComponent ?? app.bundleIdentifier?.components(separatedBy: ".").last ?? "Unknown \(app.processIdentifier)"
self.init(
id: app.processIdentifier,
kind: .app,
name: name,
audioActive: objectID.readProcessIsRunning(),
bundleID: app.bundleIdentifier,
bundleURL: app.bundleURL,
objectID: objectID
)
}
init(objectID: AudioObjectID, runningApplications apps: [NSRunningApplication]) throws {
let pid: pid_t = try objectID.read(kAudioProcessPropertyPID, defaultValue: -1)
if let app = apps.first(where: { $0.processIdentifier == pid }) {
self.init(app: app, objectID: objectID)
} else {
try self.init(objectID: objectID, pid: pid)
}
}
init(objectID: AudioObjectID, pid: pid_t) throws {
let bundleID = objectID.readProcessBundleID()
let bundleURL: URL?
let name: String
(name, bundleURL) = if let info = processInfo(for: pid) {
(info.name, URL(fileURLWithPath: info.path).parentBundleURL())
} else if let id = bundleID?.lastReverseDNSComponent {
(id, nil)
} else {
("Unknown (\(pid))", nil)
}
self.init(
id: pid,
kind: bundleURL?.isApp == true ? .app : .process,
name: name,
audioActive: objectID.readProcessIsRunning(),
bundleID: bundleID.flatMap { $0.isEmpty ? nil : $0 },
bundleURL: bundleURL,
objectID: objectID
)
}
}
extension AudioProcessGroup {
static func groups(with processes: [AudioProcess]) -> [AudioProcessGroup] {
var byKind = [AudioProcess.Kind: AudioProcessGroup]()
for process in processes {
byKind[process.kind, default: .init(for: process.kind)].processes.append(process)
}
return byKind.values.sorted(by: { $0.title.localizedStandardCompare($1.title) == .orderedAscending })
}
}
extension AudioProcessGroup {
init(for kind: AudioProcess.Kind) {
self.init(id: kind.rawValue, title: kind.groupTitle, processes: [])
}
}
extension AudioProcess.Kind {
var groupTitle: String {
switch self {
case .process: "Processes"
case .app: "Apps"
}
}
}
private func processInfo(for pid: pid_t) -> (name: String, path: String)? {
let nameBuffer = UnsafeMutablePointer<UInt8>.allocate(capacity: Int(MAXPATHLEN))
let pathBuffer = UnsafeMutablePointer<UInt8>.allocate(capacity: Int(MAXPATHLEN))
defer {
nameBuffer.deallocate()
pathBuffer.deallocate()
}
let nameLength = proc_name(pid, nameBuffer, UInt32(MAXPATHLEN))
let pathLength = proc_pidpath(pid, pathBuffer, UInt32(MAXPATHLEN))
guard nameLength > 0, pathLength > 0 else {
return nil
}
let name = String(cString: nameBuffer)
let path = String(cString: pathBuffer)
return (name, path)
}
private extension String {
var lastReverseDNSComponent: String? {
components(separatedBy: ".").last.flatMap { $0.isEmpty ? nil : $0 }
}
}
private extension URL {
func parentBundleURL(maxDepth: Int = 8) -> URL? {
var depth = 0
var url = deletingLastPathComponent()
while depth < maxDepth, !url.isBundle {
url = url.deletingLastPathComponent()
depth += 1
}
return url.isBundle ? url : nil
}
var isBundle: Bool {
(try? resourceValues(forKeys: [.contentTypeKey]))?.contentType?.conforms(to: .bundle) == true
}
var isApp: Bool {
(try? resourceValues(forKeys: [.contentTypeKey]))?.contentType?.conforms(to: .application) == true
}
}
@@ -0,0 +1,125 @@
import SwiftUI
import Observation
import OSLog
/// Uses TCC SPI in order to check/request system audio recording permission.
@Observable
final class AudioRecordingPermission {
private let logger = Logger(subsystem: "codes.rambo.AudioCap", category: String(describing: AudioRecordingPermission.self))
enum Status: String {
case unknown
case denied
case authorized
}
private(set) var status: Status = .unknown
init() {
#if ENABLE_TCC_SPI
NotificationCenter.default.addObserver(forName: NSApplication.didBecomeActiveNotification, object: nil, queue: .main) { [weak self] _ in
guard let self else { return }
self.updateStatus()
}
updateStatus()
#else
status = .authorized
#endif // ENABLE_TCC_SPI
}
func request() {
#if ENABLE_TCC_SPI
logger.debug(#function)
guard let request = Self.requestSPI else {
logger.fault("Request SPI missing")
return
}
request("kTCCServiceAudioCapture" as CFString, nil) { [weak self] granted in
guard let self else { return }
self.logger.info("Request finished with result: \(granted, privacy: .public)")
DispatchQueue.main.async {
if granted {
self.status = .authorized
} else {
self.status = .denied
}
}
}
#endif // ENABLE_TCC_SPI
}
private func updateStatus() {
#if ENABLE_TCC_SPI
logger.debug(#function)
guard let preflight = Self.preflightSPI else {
logger.fault("Preflight SPI missing")
return
}
let result = preflight("kTCCServiceAudioCapture" as CFString, nil)
if result == 1 {
status = .denied
} else if result == 0 {
status = .authorized
} else {
status = .unknown
}
#endif // ENABLE_TCC_SPI
}
#if ENABLE_TCC_SPI
private typealias PreflightFuncType = @convention(c) (CFString, CFDictionary?) -> Int
private typealias RequestFuncType = @convention(c) (CFString, CFDictionary?, @escaping (Bool) -> Void) -> Void
/// `dlopen` handle to the TCC framework.
private static let apiHandle: UnsafeMutableRawPointer? = {
let tccPath = "/System/Library/PrivateFrameworks/TCC.framework/Versions/A/TCC"
guard let handle = dlopen(tccPath, RTLD_NOW) else {
assertionFailure("dlopen failed")
return nil
}
return handle
}()
/// `dlsym` function handle for `TCCAccessPreflight`.
private static let preflightSPI: PreflightFuncType? = {
guard let apiHandle else { return nil }
let fnName = "TCCAccessPreflight"
guard let funcSym = dlsym(apiHandle, fnName) else {
assertionFailure("Couldn't find symbol")
return nil
}
let fn = unsafeBitCast(funcSym, to: PreflightFuncType.self)
return fn
}()
/// `dlsym` function handle for `TCCAccessRequest`.
private static let requestSPI: RequestFuncType? = {
guard let apiHandle else { return nil }
let fnName = "TCCAccessRequest"
guard let funcSym = dlsym(apiHandle, fnName) else {
assertionFailure("Couldn't find symbol")
return nil
}
let fn = unsafeBitCast(funcSym, to: RequestFuncType.self)
return fn
}()
#endif // ENABLE_TCC_SPI
}
@@ -0,0 +1,310 @@
import Foundation
import AudioToolbox
// MARK: - Constants
extension AudioObjectID {
/// Convenience for `kAudioObjectSystemObject`.
static let system = AudioObjectID(kAudioObjectSystemObject)
/// Convenience for `kAudioObjectUnknown`.
static let unknown = kAudioObjectUnknown
/// `true` if this object has the value of `kAudioObjectUnknown`.
var isUnknown: Bool { self == .unknown }
/// `false` if this object has the value of `kAudioObjectUnknown`.
var isValid: Bool { !isUnknown }
}
// MARK: - Concrete Property Helpers
extension AudioObjectID {
/// Reads the value for `kAudioHardwarePropertyDefaultSystemOutputDevice`.
static func readDefaultSystemOutputDevice() throws -> AudioDeviceID {
try AudioObjectID.system.readDefaultSystemOutputDevice()
}
static func readProcessList() throws -> [AudioObjectID] {
try AudioObjectID.system.readProcessList()
}
/// Reads `kAudioHardwarePropertyTranslatePIDToProcessObject` for the specific pid.
static func translatePIDToProcessObjectID(pid: pid_t) throws -> AudioObjectID {
try AudioObjectID.system.translatePIDToProcessObjectID(pid: pid)
}
/// Reads `kAudioHardwarePropertyProcessObjectList`.
func readProcessList() throws -> [AudioObjectID] {
try requireSystemObject()
var address = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyProcessObjectList,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
var dataSize: UInt32 = 0
var err = AudioObjectGetPropertyDataSize(self, &address, 0, nil, &dataSize)
guard err == noErr else { throw "Error reading data size for \(address): \(err)" }
var value = [AudioObjectID](repeating: .unknown, count: Int(dataSize) / MemoryLayout<AudioObjectID>.size)
err = AudioObjectGetPropertyData(self, &address, 0, nil, &dataSize, &value)
guard err == noErr else { throw "Error reading array for \(address): \(err)" }
return value
}
/// Reads `kAudioHardwarePropertyTranslatePIDToProcessObject` for the specific pid, should only be called on the system object.
func translatePIDToProcessObjectID(pid: pid_t) throws -> AudioObjectID {
try requireSystemObject()
let processObject = try read(
kAudioHardwarePropertyTranslatePIDToProcessObject,
defaultValue: AudioObjectID.unknown,
qualifier: pid
)
guard processObject.isValid else {
throw "Invalid process identifier: \(pid)"
}
return processObject
}
func readProcessBundleID() -> String? {
if let result = try? readString(kAudioProcessPropertyBundleID) {
result.isEmpty ? nil : result
} else {
nil
}
}
func readProcessIsRunning() -> Bool {
(try? readBool(kAudioProcessPropertyIsRunning)) ?? false
}
/*
public var kAudioProcessPropertyPID: AudioObjectPropertySelector { get }
public var kAudioProcessPropertyBundleID: AudioObjectPropertySelector { get }
public var kAudioProcessPropertyDevices: AudioObjectPropertySelector { get }
public var kAudioProcessPropertyIsRunning: AudioObjectPropertySelector { get }
public var kAudioProcessPropertyIsRunningInput: AudioObjectPropertySelector { get }
public var kAudioProcessPropertyIsRunningOutput: AudioObjectPropertySelector { get }
*/
/// Reads the value for `kAudioHardwarePropertyDefaultSystemOutputDevice`, should only be called on the system object.
func readDefaultSystemOutputDevice() throws -> AudioDeviceID {
try requireSystemObject()
return try read(kAudioHardwarePropertyDefaultSystemOutputDevice, defaultValue: AudioDeviceID.unknown)
}
/// Reads the value for `kAudioDevicePropertyDeviceUID` for the device represented by this audio object ID.
func readDeviceUID() throws -> String { try readString(kAudioDevicePropertyDeviceUID) }
/// Reads the value for `kAudioTapPropertyFormat` for the device represented by this audio object ID.
func readAudioTapStreamBasicDescription() throws -> AudioStreamBasicDescription {
try read(kAudioTapPropertyFormat, defaultValue: AudioStreamBasicDescription())
}
private func requireSystemObject() throws {
if self != .system { throw "Only supported for the system object." }
}
}
// MARK: - Generic Property Access
extension AudioObjectID {
func read<T, Q>(_ selector: AudioObjectPropertySelector,
scope: AudioObjectPropertyScope = kAudioObjectPropertyScopeGlobal,
element: AudioObjectPropertyElement = kAudioObjectPropertyElementMain,
defaultValue: T,
qualifier: Q) throws -> T
{
try read(AudioObjectPropertyAddress(mSelector: selector, mScope: scope, mElement: element), defaultValue: defaultValue, qualifier: qualifier)
}
func read<T>(_ selector: AudioObjectPropertySelector,
scope: AudioObjectPropertyScope = kAudioObjectPropertyScopeGlobal,
element: AudioObjectPropertyElement = kAudioObjectPropertyElementMain,
defaultValue: T) throws -> T
{
try read(AudioObjectPropertyAddress(mSelector: selector, mScope: scope, mElement: element), defaultValue: defaultValue)
}
func read<T, Q>(_ address: AudioObjectPropertyAddress, defaultValue: T, qualifier: Q) throws -> T {
var inQualifier = qualifier
let qualifierSize = UInt32(MemoryLayout<Q>.size(ofValue: qualifier))
return try withUnsafeMutablePointer(to: &inQualifier) { qualifierPtr in
try read(address, defaultValue: defaultValue, inQualifierSize: qualifierSize, inQualifierData: qualifierPtr)
}
}
func read<T>(_ address: AudioObjectPropertyAddress, defaultValue: T) throws -> T {
try read(address, defaultValue: defaultValue, inQualifierSize: 0, inQualifierData: nil)
}
func readString(_ selector: AudioObjectPropertySelector, scope: AudioObjectPropertyScope = kAudioObjectPropertyScopeGlobal, element: AudioObjectPropertyElement = kAudioObjectPropertyElementMain) throws -> String {
try read(AudioObjectPropertyAddress(mSelector: selector, mScope: scope, mElement: element), defaultValue: "" as CFString) as String
}
func readBool(_ selector: AudioObjectPropertySelector, scope: AudioObjectPropertyScope = kAudioObjectPropertyScopeGlobal, element: AudioObjectPropertyElement = kAudioObjectPropertyElementMain) throws -> Bool {
let value: Int = try read(AudioObjectPropertyAddress(mSelector: selector, mScope: scope, mElement: element), defaultValue: 0)
return value == 1
}
private func read<T>(_ inAddress: AudioObjectPropertyAddress, defaultValue: T, inQualifierSize: UInt32 = 0, inQualifierData: UnsafeRawPointer? = nil) throws -> T {
var address = inAddress
var dataSize: UInt32 = 0
var err = AudioObjectGetPropertyDataSize(self, &address, inQualifierSize, inQualifierData, &dataSize)
guard err == noErr else {
throw "Error reading data size for \(inAddress): \(err)"
}
var value: T = defaultValue
err = withUnsafeMutablePointer(to: &value) { ptr in
AudioObjectGetPropertyData(self, &address, inQualifierSize, inQualifierData, &dataSize, ptr)
}
guard err == noErr else {
throw "Error reading data for \(inAddress): \(err)"
}
return value
}
}
// MARK: - Debugging Helpers
private extension UInt32 {
var fourCharString: String {
String(cString: [
UInt8((self >> 24) & 0xFF),
UInt8((self >> 16) & 0xFF),
UInt8((self >> 8) & 0xFF),
UInt8(self & 0xFF),
0
])
}
}
extension AudioObjectPropertyAddress: @retroactive CustomStringConvertible {
public var description: String {
let elementDescription = mElement == kAudioObjectPropertyElementMain ? "main" : mElement.fourCharString
return "\(mSelector.fourCharString)/\(mScope.fourCharString)/\(elementDescription)"
}
}
public struct AudioInputDevice: Identifiable, Hashable {
public let id: AudioDeviceID // AudioDeviceID (UInt32) is Hashable and can serve as Identifiable's id.
public let uid: String // Unique identifier for the device (persistent across reboots)
public let name: String
public init(id: AudioDeviceID, uid: String, name: String) {
self.id = id
self.uid = uid
self.name = name
}
}
extension AudioObjectID {
static func getAllInputDevices() throws -> [AudioInputDevice] {
let allDeviceIDs = try AudioObjectID.system.getAllHardwareDevices()
var inputDevices: [AudioInputDevice] = []
for deviceID in allDeviceIDs {
do {
let inputChannelCount = try deviceID.getTotalInputChannelCount()
if inputChannelCount > 0 {
let deviceName = try deviceID.getDeviceName()
let deviceUID = try deviceID.readDeviceUID()
inputDevices.append(AudioInputDevice(id: deviceID, uid: deviceUID, name: deviceName))
}
} catch {
print("CoreAudioUtils: Could not fully query device \(deviceID): \(error)")
}
}
return inputDevices
}
func getAllHardwareDevices() throws -> [AudioDeviceID] {
try requireSystemObject()
var address = AudioObjectPropertyAddress(
mSelector: kAudioHardwarePropertyDevices,
mScope: kAudioObjectPropertyScopeGlobal,
mElement: kAudioObjectPropertyElementMain
)
var dataSize: UInt32 = 0
var err = AudioObjectGetPropertyDataSize(self, &address, 0, nil, &dataSize)
guard err == noErr else {
throw "CoreAudioUtils: Error reading data size for \(kAudioHardwarePropertyDevices.fourCharString): \(err)"
}
let deviceCount = Int(dataSize) / MemoryLayout<AudioDeviceID>.size
if deviceCount == 0 {
return []
}
var deviceIDs = [AudioDeviceID](repeating: .unknown, count: deviceCount)
err = AudioObjectGetPropertyData(self, &address, 0, nil, &dataSize, &deviceIDs)
guard err == noErr else {
throw "CoreAudioUtils: Error reading device array for \(kAudioHardwarePropertyDevices.fourCharString): \(err)"
}
return deviceIDs
}
func getTotalInputChannelCount() throws -> UInt32 {
var address = AudioObjectPropertyAddress(
mSelector: kAudioDevicePropertyStreamConfiguration,
mScope: kAudioDevicePropertyScopeInput,
mElement: kAudioObjectPropertyElementMain
)
var dataSize: UInt32 = 0
var err = AudioObjectGetPropertyDataSize(self, &address, 0, nil, &dataSize)
if err == kAudioHardwareUnknownPropertyError || dataSize == 0 {
return 0
}
guard err == noErr else {
throw "CoreAudioUtils: Error reading data size for input stream configuration on device \(self): \(err)"
}
let bufferListPtr = UnsafeMutableRawPointer.allocate(byteCount: Int(dataSize), alignment: MemoryLayout<AudioBufferList>.alignment)
defer { bufferListPtr.deallocate() }
err = AudioObjectGetPropertyData(self, &address, 0, nil, &dataSize, bufferListPtr)
guard err == noErr else {
throw "CoreAudioUtils: Error reading input stream configuration for device \(self): \(err)"
}
let audioBufferList = bufferListPtr.assumingMemoryBound(to: AudioBufferList.self)
var totalInputChannels: UInt32 = 0
let buffers = UnsafeMutableAudioBufferListPointer(audioBufferList)
for i in 0..<Int(buffers.count) {
totalInputChannels += buffers[i].mNumberChannels
}
return totalInputChannels
}
func getDeviceName() throws -> String {
return try readString(kAudioDevicePropertyDeviceNameCFString)
}
}
+499
View File
@@ -0,0 +1,499 @@
import SwiftUI
import AudioToolbox
import OSLog
import AVFoundation
enum TapTarget {
case singleProcess(AudioProcess)
case systemAudio(processObjectIDs: [AudioObjectID])
var displayName: String {
switch self {
case .singleProcess(let process):
return process.name
case .systemAudio:
return "System Audio Output"
}
}
var iconImage: NSImage {
switch self {
case .singleProcess(let process):
return process.icon
case .systemAudio:
let genericAppIcon = NSWorkspace.shared.icon(for: .applicationBundle)
genericAppIcon.size = NSSize(width: 32, height: 32)
return genericAppIcon
}
}
var loggingProcessName: String {
switch self {
case .singleProcess(let process):
return process.name
case .systemAudio:
return "SystemAudioOutput"
}
}
}
@Observable
final class ProcessTap {
typealias InvalidationHandler = (ProcessTap) -> Void
let target: TapTarget
let muteWhenRunning: Bool
private let logger: Logger
private(set) var errorMessage: String? = nil
init(target: TapTarget, muteWhenRunning: Bool = false) {
self.target = target
self.muteWhenRunning = muteWhenRunning
self.logger = Logger(subsystem: "codes.rambo.AudioCap", category: "\(String(describing: ProcessTap.self))(\(target.loggingProcessName))")
}
@ObservationIgnored
private var processTapID: AudioObjectID = .unknown
@ObservationIgnored
private var aggregateDeviceID = AudioObjectID.unknown
@ObservationIgnored
private var deviceProcID: AudioDeviceIOProcID?
@ObservationIgnored
private(set) var tapStreamDescription: AudioStreamBasicDescription?
@ObservationIgnored
private var invalidationHandler: InvalidationHandler?
@ObservationIgnored
private(set) var activated = false
var displayName: String {
target.displayName
}
@MainActor
func activate() {
guard !activated else { return }
activated = true
logger.debug(#function)
self.errorMessage = nil
do {
try prepare()
} catch {
logger.error("\(error, privacy: .public)")
self.errorMessage = error.localizedDescription
}
}
func invalidate() {
guard activated else { return }
defer { activated = false }
logger.debug(#function)
invalidationHandler?(self)
self.invalidationHandler = nil
if aggregateDeviceID.isValid {
var err: OSStatus
err = AudioDeviceStop(aggregateDeviceID, deviceProcID)
if err != noErr {
logger.warning("Failed to stop aggregate device: \(err, privacy: .public)")
}
if let deviceProcID {
err = AudioDeviceDestroyIOProcID(aggregateDeviceID, deviceProcID)
if err != noErr {
logger.warning("Failed to destroy device I/O proc: \(err, privacy: .public)")
}
self.deviceProcID = nil
}
err = AudioHardwareDestroyAggregateDevice(aggregateDeviceID)
if err != noErr {
logger.warning("Failed to destroy aggregate device: \(err, privacy: .public)")
}
aggregateDeviceID = .unknown
}
if processTapID.isValid {
let errTapDestroy = AudioHardwareDestroyProcessTap(processTapID)
if errTapDestroy != noErr {
logger.warning("Failed to destroy audio tap: \(errTapDestroy, privacy: .public)")
}
self.processTapID = .unknown
}
}
private func prepare() throws {
errorMessage = nil
let tapDescription: CATapDescription
switch self.target {
case .singleProcess(let process):
tapDescription = CATapDescription(stereoMixdownOfProcesses: [process.objectID])
logger.debug("Configuring tap for single process objectID: \(process.objectID)")
case .systemAudio(let processObjectIDs):
if processObjectIDs.isEmpty {
logger.warning("System audio tap configured with an empty list of processObjectIDs. This might not capture any audio or behave unexpectedly.")
}
tapDescription = CATapDescription(stereoMixdownOfProcesses: processObjectIDs)
logger.debug("Configuring tap for system audio output using \(processObjectIDs.count) explicit processes.")
}
tapDescription.uuid = UUID()
tapDescription.muteBehavior = muteWhenRunning ? .mutedWhenTapped : .unmuted
var tapID: AUAudioObjectID = .unknown
let errTapCreation = AudioHardwareCreateProcessTap(tapDescription, &tapID)
guard errTapCreation == noErr else {
errorMessage = "Process/System tap creation failed with error \(errTapCreation)"
throw errorMessage ?? "Unknown error creating tap."
}
logger.debug("Created process/system tap #\(tapID, privacy: .public). Associated UUID: \(tapDescription.uuid.uuidString)")
self.processTapID = tapID
let allDeviceIDs = try AudioObjectID.system.getAllHardwareDevices()
var outputUIDs: [String] = []
var outputDeviceIDs: [AudioDeviceID] = []
for devID in allDeviceIDs {
do {
let outputChans = try devID.getTotalOutputChannelCount()
if outputChans > 0 {
let devUID = try devID.readDeviceUID()
outputUIDs.append(devUID)
outputDeviceIDs.append(devID)
}
} catch {
logger.warning("Ignored device \(devID): \(error.localizedDescription)")
}
}
if outputUIDs.isEmpty {
throw "No hardware output devices found!"
}
let systemOutputID: AudioDeviceID
do {
logger.debug("Attempting to read default system output device ID...")
systemOutputID = try AudioDeviceID.readDefaultSystemOutputDevice()
logger.debug("Successfully read default system output device ID: \(systemOutputID)")
} catch {
logger.error("Failed to read default system output device ID: \(error)")
throw error // Propagate error
}
let mainSubdeviceUID: String
do {
logger.debug("Attempting to read device UID for systemOutputID: \(systemOutputID)...")
mainSubdeviceUID = try systemOutputID.readDeviceUID()
logger.debug("Successfully read mainSubdeviceUID: \(mainSubdeviceUID)")
} catch {
logger.error("Failed to read device UID for systemOutputID \(systemOutputID): \(error)")
throw error // Propagate error
}
let subDeviceListForAggregate: [[String: Any]]
let aggregateDeviceName: String
let aggregateUID = UUID().uuidString
switch self.target {
case .systemAudio:
aggregateDeviceName = "Tap-SysAgg-\(mainSubdeviceUID.prefix(8))"
subDeviceListForAggregate = [
[kAudioSubDeviceUIDKey: mainSubdeviceUID]
]
logger.debug("System mode: mainSubdeviceUID for aggregate: \(mainSubdeviceUID). Aggregate name: \(aggregateDeviceName)")
case .singleProcess:
aggregateDeviceName = "Tap-\(self.displayName)-Agg"
subDeviceListForAggregate = outputUIDs.map { [kAudioSubDeviceUIDKey: $0] }
logger.debug("Process mode: Aggregate subDeviceList from outputUIDs. Aggregate name: \(aggregateDeviceName)")
}
let descriptionForAggregate: [String: Any] = [
kAudioAggregateDeviceNameKey: aggregateDeviceName,
kAudioAggregateDeviceUIDKey: aggregateUID,
kAudioAggregateDeviceMainSubDeviceKey: mainSubdeviceUID,
kAudioAggregateDeviceIsPrivateKey: true,
kAudioAggregateDeviceIsStackedKey: false,
kAudioAggregateDeviceTapAutoStartKey: true,
kAudioAggregateDeviceSubDeviceListKey: subDeviceListForAggregate,
kAudioAggregateDeviceTapListKey: [
[
kAudioSubTapDriftCompensationKey: true,
kAudioSubTapUIDKey: tapDescription.uuid.uuidString
]
]
]
logger.debug("Aggregate device description prepared. Main sub-device UID: \(mainSubdeviceUID), Tap UUID: \(tapDescription.uuid.uuidString)")
aggregateDeviceID = AudioObjectID.unknown
do {
logger.debug("Calling AudioHardwareCreateAggregateDevice...")
let errAggDeviceCreation = AudioHardwareCreateAggregateDevice(descriptionForAggregate as CFDictionary, &aggregateDeviceID)
if errAggDeviceCreation != noErr {
logger.error("AudioHardwareCreateAggregateDevice failed with error: \(errAggDeviceCreation).")
throw "Failed to create aggregate device: \(errAggDeviceCreation)"
}
logger.debug("Successfully created aggregate device #\(self.aggregateDeviceID, privacy: .public)")
} catch {
logger.error("EXCEPTION during AudioHardwareCreateAggregateDevice block: \(error)")
throw error // Propagate error
}
do {
logger.debug("Attempting to read audio tap stream basic description for tapID #\(tapID)...")
self.tapStreamDescription = try tapID.readAudioTapStreamBasicDescription()
logger.debug("Successfully read tap stream description: \(String(describing: self.tapStreamDescription))")
} catch {
logger.error("Failed to read audio tap stream basic description for tapID #\(tapID): \(error)")
throw error // Propagate error
}
}
func run(on queue: DispatchQueue, ioBlock: @escaping AudioDeviceIOBlock, invalidationHandler: @escaping InvalidationHandler) throws {
assert(activated, "\(#function) called with inactive tap!")
assert(self.invalidationHandler == nil, "\(#function) called with tap already active!")
errorMessage = nil
logger.debug("Run tap!")
self.invalidationHandler = invalidationHandler
var err = AudioDeviceCreateIOProcIDWithBlock(&deviceProcID, aggregateDeviceID, queue, ioBlock)
guard err == noErr else { throw "Failed to create device I/O proc: \(err)" }
err = AudioDeviceStart(aggregateDeviceID, deviceProcID)
guard err == noErr else { throw "Failed to start audio device: \(err)" }
}
deinit { invalidate() }
}
private extension AudioDeviceID {
func getTotalOutputChannelCount() throws -> UInt32 {
var address = AudioObjectPropertyAddress(
mSelector: kAudioDevicePropertyStreamConfiguration,
mScope: kAudioDevicePropertyScopeOutput,
mElement: kAudioObjectPropertyElementMain
)
var dataSize: UInt32 = 0
var err = AudioObjectGetPropertyDataSize(self, &address, 0, nil, &dataSize)
if err == kAudioHardwareUnknownPropertyError || dataSize == 0 {
return 0
}
guard err == noErr else {
throw "Error reading data size for output stream configuration: \(err)"
}
let bufferListPtr = UnsafeMutableRawPointer.allocate(byteCount: Int(dataSize), alignment: MemoryLayout<AudioBufferList>.alignment)
defer { bufferListPtr.deallocate() }
err = AudioObjectGetPropertyData(self, &address, 0, nil, &dataSize, bufferListPtr)
guard err == noErr else {
throw "Error reading output stream configuration: \(err)"
}
let audioBufferList = bufferListPtr.assumingMemoryBound(to: AudioBufferList.self)
var totalOutputChannels: UInt32 = 0
let buffers = UnsafeMutableAudioBufferListPointer(audioBufferList)
for i in 0..<Int(buffers.count) {
totalOutputChannels += buffers[i].mNumberChannels
}
return totalOutputChannels
}
}
@Observable
final class ProcessTapRecorder {
let fileURL: URL
let tapDisplayName: String
let icon: NSImage
private(set) var currentAudioLevel: Float = 0.0
private let queue = DispatchQueue(label: "ProcessTapRecorder", qos: .userInitiated)
private let logger: Logger
@ObservationIgnored
private weak var _tap: ProcessTap?
private(set) var isRecording = false
init(fileURL: URL, tap: ProcessTap) {
self.tapDisplayName = tap.displayName
self.fileURL = fileURL
self._tap = tap
self.logger = Logger(subsystem: "codes.rambo.AudioCap", category: "\(String(describing: ProcessTapRecorder.self))(\(fileURL.lastPathComponent))")
self.icon = tap.target.iconImage
}
private var tap: ProcessTap {
get throws {
guard let _tap else { throw "Process tap unavailable" }
return _tap
}
}
@ObservationIgnored
private var currentFile: AVAudioFile?
@MainActor
func start() throws {
logger.debug(#function)
guard !isRecording else {
logger.warning("\(#function, privacy: .public) while already recording")
return
}
self.isRecording = true
let tap = try tap
if !tap.activated {
tap.activate()
if let errorMessage = tap.errorMessage {
logger.error("Tap activation error: \(errorMessage)")
self.isRecording = false
throw errorMessage
}
}
guard var streamDescription = tap.tapStreamDescription else {
logger.error("Tap stream description not available.")
self.isRecording = false
throw "Tap stream description not available."
}
guard let format = AVAudioFormat(streamDescription: &streamDescription) else {
logger.error("Failed to create AVAudioFormat from stream description.")
self.isRecording = false
throw "Failed to create AVAudioFormat."
}
logger.info("Using audio format: \(format, privacy: .public)")
let settings: [String: Any] = [
AVFormatIDKey: streamDescription.mFormatID,
AVSampleRateKey: format.sampleRate,
AVNumberOfChannelsKey: format.channelCount
]
try FileManager.default.createDirectory(at: fileURL.deletingLastPathComponent(), withIntermediateDirectories: true, attributes: nil)
do {
let file = try AVAudioFile(forWriting: fileURL, settings: settings, commonFormat: .pcmFormatFloat32, interleaved: format.isInterleaved)
self.currentFile = file
} catch {
logger.error("Failed to create AVAudioFile for writing: \(error, privacy: .public)")
self.isRecording = false
throw error
}
#if DEBUG
let systemModeActive: Bool
if case .systemAudio = tap.target {
systemModeActive = true
} else {
systemModeActive = false
}
print("DEBUG: About to call tap.run... (system mode? \(systemModeActive))")
#endif
try tap.run(on: queue) { [weak self] inNow, inInputData, inInputTime, outOutputData, inOutputTime in
guard let self else { return }
var localAudioLevel: Float = 0.0
do {
guard let currentFile = self.currentFile else {
DispatchQueue.main.async { if self.currentAudioLevel != 0.0 { self.currentAudioLevel = 0.0 } }
return
}
guard let buffer = AVAudioPCMBuffer(pcmFormat: format, bufferListNoCopy: inInputData, deallocator: nil) else {
print("ProcessTapRecorder: Failed to create PCM buffer")
DispatchQueue.main.async { if self.currentAudioLevel != 0.0 { self.currentAudioLevel = 0.0 } }
return
}
var rms: Float = 0.0
if let floatChannelData = buffer.floatChannelData, buffer.frameLength > 0 {
let channelData = floatChannelData[0]
let frameLength = Int(buffer.frameLength)
var sumOfSquares: Float = 0.0
for i in 0..<frameLength {
let sample = channelData[i]
sumOfSquares += sample * sample
}
rms = sqrt(sumOfSquares / Float(frameLength))
#if DEBUG
if case .systemAudio = (try? self.tap)?.target {
print("SYSTEM MODE: buffer.frameLength = \(frameLength), RMS = \(rms)")
if rms == 0.0 {
print("SYSTEM MODE: WARNING: Audio buffer is silent (RMS == 0.0)")
}
}
#endif
}
localAudioLevel = min(max(rms * 2.0, 0.0), 1.0)
if buffer.frameLength == 0 {
print("ProcessTapRecorder: Warning - received zero frames!")
}
try currentFile.write(from: buffer)
} catch {
self.logger.error("Buffer write error: \(error, privacy: .public)")
print("ProcessTapRecorder: Buffer write error:", error)
localAudioLevel = 0.0
}
DispatchQueue.main.async {
self.currentAudioLevel = localAudioLevel
}
} invalidationHandler: { [weak self] tap in
guard let self else { return }
DispatchQueue.main.async {
self.handleInvalidation()
}
}
print("ProcessTapRecorder: Recording started (isRecording set to true).")
}
@MainActor
func stop() {
logger.debug(#function)
guard isRecording else { return }
self.currentAudioLevel = 0.0
self.isRecording = false
guard let tapToInvalidate = try? self.tap else {
logger.warning("Tap unavailable during stop. Cleaning up recorder state.")
self.currentFile = nil
return
}
tapToInvalidate.invalidate()
self.currentFile = nil
}
@MainActor
private func handleInvalidation() {
logger.debug("Handling tap invalidation in recorder.")
if isRecording {
logger.info("Tap invalidated while recording. Stopping recording.")
self.currentFile = nil
self.isRecording = false
self.currentAudioLevel = 0.0
}
}
}