added timestamp and cvpixelbufferref to capturedframe

added actually encoding said frame sob added a get encodersettings function added configure compressionsession function added nserror check function added screampacket data schema
2026-03-11 13:26:18 +00:00 · 2026-01-03 20:52:27 +00:00
parent 7b3e260d57
commit 739b19bfb3
3 changed files with 134 additions and 10 deletions
--- a/CaptureEngine.swift
+++ b/CaptureEngine.swift
@@ -6,19 +6,22 @@
 //

 import Foundation
+import Cocoa
 import ScreenCaptureKit
 import VideoToolbox

 struct CapturedFrame {
 	static var invalid: CapturedFrame {
-		CapturedFrame(surface: nil, contentRect: .zero, contentScale: 0, scaleFactor: 0)
+		CapturedFrame(surface: nil, pixelBuffer: nil, contentRect: .zero, contentScale: 0, scaleFactor: 0, timestamp: .invalid)
 	}
 	
 	let surface: IOSurface?
+	let pixelBuffer: CVPixelBuffer?
 	let contentRect: CGRect
 	let contentScale: CGFloat
 	let scaleFactor: CGFloat
 	var size: CGSize { contentRect.size }
+	var timestamp: CMTime
 }

 class CaptureEngine: NSObject {
@@ -37,15 +40,16 @@ class CaptureEngine: NSObject {
 		var compressionSessionOut: VTCompressionSession?
 		let err = VTCompressionSessionCreate(
 			allocator: kCFAllocatorDefault,
-			width: 1600,
-			height: 900,
+			width: 3200,
+			height: 1800,
 			codecType: kCMVideoCodecType_H264,
 			encoderSpecification: videoEncoderSpec,
 			imageBufferAttributes: sourceImageBufferAttrs,
 			compressedDataAllocator: nil,
-			outputCallback: { outputCallbackRefCon, sourceFrameRefCon, status, infoFlags, samplebuffer in
-				print(status)
-			},
+			outputCallback: nil,
+//			outputCallback: { outputCallbackRefCon, sourceFrameRefCon, status, infoFlags, samplebuffer in
+//				print(status)
+//			},
 			refcon: nil,
 			compressionSessionOut: &compressionSessionOut
 		)
@@ -57,14 +61,23 @@ class CaptureEngine: NSObject {
 		return AsyncThrowingStream<CapturedFrame, Error> { continuation in
 			let streamOutput = StreamHandler(continuation: continuation)
 			self.streamOutput = streamOutput
-			streamOutput.frameBufferHandler = { continuation.yield($0) }
-			streamOutput.pcmBufferHandler = { print($0) }
 			
 			do {
 				streamOutput.frameBufferHandler = { frame in
-//					print("got frame \(frame.size) at \(frame.contentRect)")
+					VTCompressionSessionEncodeFrame(compressionSession,
+													imageBuffer: frame.pixelBuffer!,
+													presentationTimeStamp: frame.timestamp,
+													duration: .invalid,
+													frameProperties: nil,
+													infoFlagsOut: nil
+					) { status, infoFlags, sampleBuffer in
+						print()
+						
+					}
+//													outputHandler: self.outputHandler)
 					continuation.yield(frame)
 				}
+				streamOutput.pcmBufferHandler = { print($0) }
 				stream = SCStream(filter: filter, configuration: config, delegate: streamOutput)
 				
 				try stream?.addStreamOutput(streamOutput, type: .screen, sampleHandlerQueue: videoSampleBufferQueue)
@@ -77,6 +90,81 @@ class CaptureEngine: NSObject {
 		}
 	}
 	
+	func getEncoderSettings(session: VTCompressionSession) -> [CFString: Any]? {
+		var supportedPresetDictionaries: CFDictionary?
+		var encoderSettings: [CFString: Any]?
+		
+		_ = withUnsafeMutablePointer(to: &supportedPresetDictionaries) { valueOut in
+			if #available(macOS 26.0, *) {
+				VTSessionCopyProperty(session, key: kVTCompressionPropertyKey_SupportedPresetDictionaries, allocator: kCFAllocatorDefault, valueOut: valueOut)
+			} else {
+				// Fallback on earlier versions
+			}
+		}
+		
+		if let presetDictionaries = supportedPresetDictionaries as? [CFString: [CFString: Any]] {
+			encoderSettings = presetDictionaries
+		}
+		
+		return encoderSettings
+	}
+	
+	func configureVTCompressionSession(session: VTCompressionSession, expectedFrameRate: Float = 60) throws {
+//		var escapedContinuation: AsyncStream<(OSStatus, VTEncodeInfoFlags, CMSampleBuffer?, Int)>.Continuation!
+//		let compressedFrameSequence = AsyncStream<(OSStatus, VTEncodeInfoFlags, CMSampleBuffer?, Int)> { escapedContinuation = $0 }
+//		let outputContinuation = escapedContinuation!
+//		
+//		let compressionTask = Task {
+//			
+//		}
+		
+		var err: OSStatus = noErr
+		var variableBitRateMode = false
+		
+		let encoderSettings: [CFString: Any]?
+		encoderSettings = getEncoderSettings(session: session)
+		
+		if let encoderSettings {
+			if #available(macOS 26.0, *) {
+				if encoderSettings[kVTCompressionPropertyKey_VariableBitRate] != nil {
+					variableBitRateMode = true
+				}
+			} else {
+				// Fallback on earlier versions
+			}
+			
+			err = VTSessionSetProperties(session, propertyDictionary: encoderSettings as CFDictionary)
+			try NSError.check(err, "VTSessionSetProperties failed")
+//			err = VTSessionSetProperty(<#T##CM_NONNULL#>, key: <#T##CM_NONNULL#>, value: <#T##CM_NULLABLE#>)
+		}
+		
+		err = VTSessionSetProperty(session, key: kVTCompressionPropertyKey_RealTime, value: kCFBooleanTrue)
+		if err != noErr { print("failed to set to realtime \(err)") }
+		
+		err = VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ExpectedFrameRate, value: expectedFrameRate as CFNumber)
+		if err != noErr { print("failed to set to framerte \(err)") }
+
+		err = VTSessionSetProperty(session, key: kVTCompressionPropertyKey_ProfileLevel, value: kVTProfileLevel_H264_Main_AutoLevel)
+		if err != noErr { print("failed to set to profile level \(err)") }
+		
+		err = VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate, value: 10 as CFNumber)
+		if err != noErr { print("failed to set to framerte \(err)") }
+		
+		err = VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameInterval, value: 60 as CFNumber)
+		if err != noErr { print("failed to set to keyframe interval \(err)") }
+		
+		err = VTSessionSetProperty(session, key: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, value: 1 as CFNumber)
+		if err != noErr { print("failed to set to keyframe interval duratation \(err)") }
+		
+//		err = VTSessionSetProperty(session, key: kVTCompressionPropertyKey_SuggestedLookAheadFrameCount, value: 1 as CFNumber)
+//		if err != noErr { print("failed to set to lookahead \(err)") }
+		
+//		err = VTSessionSetProperty(session, key: kVTCompressionPropertyKey_SpatialAdaptiveQPLevel, value: 1 as CFNumber)
+//		if err != noErr { print("failed to set to framerte \(err)") }
+		
+//		VTCompressionSessionEncodeFrame(session, imageBuffer: <#T##CVImageBuffer#>, presentationTimeStamp: <#T##CMTime#>, duration: <#T##CMTime#>, frameProperties: <#T##CFDictionary?#>, infoFlagsOut: <#T##UnsafeMutablePointer<VTEncodeInfoFlags>?#>, outputHandler: <#T##VTCompressionOutputHandler##VTCompressionOutputHandler##(OSStatus, VTEncodeInfoFlags, CMSampleBuffer?) -> Void#>)
+	}
+	
 	func stopCapture() async {
 		do {
 			try await stream?.stopCapture()
@@ -142,9 +230,11 @@ class StreamHandler: NSObject, SCStreamOutput, SCStreamDelegate {
 			  let scaleFactor = attachments[.scaleFactor] as? CGFloat else { return nil }
 		
 		var frame = CapturedFrame(surface: surface,
+								  pixelBuffer: pixelBuffer,
 								  contentRect: contentRect,
 								  contentScale: contentScale,
-								  scaleFactor: scaleFactor)
+								  scaleFactor: scaleFactor,
+								  timestamp: CMSampleBufferGetPresentationTimeStamp(sampleBuffer))
 		return frame
 	}
 	
@@ -180,3 +270,14 @@ class StreamHandler: NSObject, SCStreamOutput, SCStreamDelegate {
 		continuation?.finish(throwing: error)
 	}
 }
+
+extension NSError {
+	static func check(_ status: OSStatus, _ message: String? = nil) throws {
+		guard status == noErr else {
+			if let message {
+				print("\(message), err: \(status)")
+			}
+			throw NSError(domain: NSOSStatusErrorDomain, code: Int(status))
+		}
+	}
+}