Last active
November 23, 2025 10:59
-
-
Save Matt54/f0baea72bb5e8acd81ae1e99493f71e7 to your computer and use it in GitHub Desktop.
PSVR2 Sense Controller Input Handling and Haptics in RealityKit
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| import ARKit | |
| import CoreHaptics | |
| import GameController | |
| import RealityKit | |
| import SwiftUI | |
| // MARK: - Immersive View | |
| struct ImmersiveSenseControllerInputsView: View { | |
| @State private var controllerManager = GameControllerManager() | |
| var body: some View { | |
| RealityView { content in | |
| content.add(controllerManager.rootEntity) | |
| await controllerManager.handleControllerSetup() | |
| } | |
| .task { | |
| let configuration = SpatialTrackingSession.Configuration(tracking: [.accessory]) | |
| let session = SpatialTrackingSession() | |
| await session.run(configuration) | |
| print("🛰️ SpatialTrackingSession running with [.accessory]") | |
| } | |
| } | |
| } | |
| // MARK: - ControllerState | |
| struct ControllerState { | |
| var controller: GCController | |
| var aimAnchor: AnchorEntity? | |
| var visualRoot: Entity? | |
| var inputState: InputState = .init() | |
| var hapticEngine: CHHapticEngine? | |
| var buttonTapHapticPlayer: CHHapticPatternPlayer? | |
| var triggerHapticPlayer: CHHapticPatternPlayer? | |
| struct InputState { | |
| var trigger = ButtonState() | |
| var thumbstickButton = ButtonState() | |
| var buttonA = ButtonState() | |
| var buttonB = ButtonState() | |
| var grip = ButtonState() | |
| var menu = ButtonState() | |
| var thumbstick = StickState() | |
| struct ButtonState { | |
| var pressure: Float = 0.0 | |
| var isPressed: Bool { return pressure > 0 } | |
| var sfSymbolName: String? | |
| } | |
| struct StickState { | |
| var x: Float = 0.0 | |
| var y: Float = 0.0 | |
| var isPressed: Bool = false | |
| var sfSymbolName: String? | |
| } | |
| } | |
| } | |
| // MARK: - GameControllerManager | |
| @Observable | |
| @MainActor | |
| final class GameControllerManager { | |
| var rootEntity = Entity() | |
| var controllers: [ObjectIdentifier: ControllerState] = [:] | |
| private var connectObserver: NSObjectProtocol? | |
| private var disconnectObserver: NSObjectProtocol? | |
| } | |
| // MARK: - Setup Controllers | |
| extension GameControllerManager { | |
| func handleControllerSetup() async { | |
| // Ensure we continue to receive controller events even if focus changes. | |
| GCController.shouldMonitorBackgroundEvents = true | |
| // Store observers to keep them alive | |
| connectObserver = NotificationCenter.default.addObserver( | |
| forName: .GCControllerDidConnect, | |
| object: nil, | |
| queue: .main | |
| ) { [weak self] notification in | |
| guard | |
| let self, | |
| let controller = notification.object as? GCController | |
| else { return } | |
| Task { @MainActor in | |
| await self.setupController(controller) | |
| } | |
| } | |
| disconnectObserver = NotificationCenter.default.addObserver( | |
| forName: .GCControllerDidDisconnect, | |
| object: nil, | |
| queue: .main | |
| ) { [weak self] notification in | |
| guard | |
| let self, | |
| let controller = notification.object as? GCController | |
| else { return } | |
| Task { @MainActor in | |
| await self.teardownController(controller) | |
| } | |
| } | |
| } | |
| private func setupController(_ controller: GCController) async { | |
| let key = ObjectIdentifier(controller) | |
| print("🎮 Connecting controller: \(controller.vendorName ?? "Unknown"), category: \(controller.productCategory)") | |
| // Add an anchor to the aim | |
| var anchor: AnchorEntity? | |
| if let source = try? await AnchoringComponent | |
| .AccessoryAnchoringSource(device: controller) | |
| { | |
| // options: ["aim", "grip", "grip_surface"] | |
| print("📍 Controller accessory locations: \(source.accessoryLocations)") | |
| // Use "aim" anchor for the axis indicator | |
| if let location = source.locationName(named: "aim") { | |
| let accessoryAnchor = AnchorEntity( | |
| .accessory(from: source, location: location), | |
| trackingMode: .predicted, | |
| physicsSimulation: .none | |
| ) | |
| rootEntity.addChild(accessoryAnchor) | |
| anchor = accessoryAnchor | |
| // Visual axis gizmo for the controller | |
| let axis = makeAxisIndicator() | |
| axis.position = [0, 0, -0.01] | |
| accessoryAnchor.addChild(axis) | |
| controllers[key]?.visualRoot = axis | |
| // Attach the status view | |
| let statusEntity = Entity() | |
| let statusView = ControllerInputStateView(manager: self, controllerID: key) | |
| let attachment = ViewAttachmentComponent(rootView: statusView) | |
| statusEntity.components.set(attachment) | |
| // Position with default offset initially | |
| statusEntity.position = [0.12, 0.08, 0] | |
| accessoryAnchor.addChild(statusEntity) | |
| // Wait for tracking to start, then update position based on chirality | |
| // This feels like a hack – there's probably a more appropriate strategy | |
| Task { @MainActor in | |
| while true { | |
| if let arkitAnchor = getAccessoryAnchor(entity: accessoryAnchor), | |
| arkitAnchor.trackingState != .untracked { | |
| let chirality = arkitAnchor.heldChirality ?? arkitAnchor.accessory.inherentChirality | |
| let xOffset: Float = chirality == .left ? 0.12 : (chirality == .right ? -0.12 : 0.12) | |
| statusEntity.position = [xOffset, 0.08, 0] | |
| break | |
| } | |
| try? await Task.sleep(nanoseconds: 50_000_000) // 50ms | |
| } | |
| } | |
| } | |
| } else { | |
| print("⚠️ Controller does not provide an accessory anchoring source (still using inputs).") | |
| } | |
| controllers[key] = ControllerState(controller: controller, aimAnchor: anchor) | |
| setupHaptics(for: controller, key: key) | |
| setupControllerInputHandlers(for: controller, key: key, anchor: anchor) | |
| } | |
| /// Get the ARKit accessory anchor from a RealityKit AnchorEntity | |
| func getAccessoryAnchor(entity: AnchorEntity) -> AccessoryAnchor? { | |
| if let accessoryAnchor = entity.components[ARKitAnchorComponent.self]?.anchor as? AccessoryAnchor { | |
| return accessoryAnchor | |
| } | |
| return nil | |
| } | |
| private func teardownController(_ controller: GCController) async { | |
| let key = ObjectIdentifier(controller) | |
| print("🛑 Controller disconnected: \(controller.vendorName ?? "Unknown")") | |
| // Remove anchor | |
| if let state = controllers[key] { | |
| if let anchor = state.aimAnchor { | |
| anchor.removeFromParent() | |
| } | |
| if let root = state.visualRoot { | |
| root.removeFromParent() | |
| } | |
| if let player = state.triggerHapticPlayer { | |
| try? player.stop(atTime: CHHapticTimeImmediate) | |
| } | |
| controllers[key]?.triggerHapticPlayer = nil | |
| if let engine = controllers[key]?.hapticEngine { | |
| try? await engine.stop() | |
| } | |
| controllers[key]?.hapticEngine = nil | |
| controllers[key]?.buttonTapHapticPlayer = nil | |
| } | |
| controllers.removeValue(forKey: key) | |
| } | |
| } | |
| // MARK: - Haptics | |
| extension GameControllerManager { | |
| private func setupHaptics(for controller: GCController, key: ObjectIdentifier) { | |
| guard let gamepad = controller.haptics else { | |
| print("ℹ️ No haptics for this controller.") | |
| return | |
| } | |
| let engine = gamepad.createEngine(withLocality: .default) | |
| do { | |
| try engine?.start() | |
| if let engine { | |
| controllers[key]?.hapticEngine = engine | |
| // Quick, light button tap haptic | |
| let tapPattern = try CHHapticPattern(events: [ | |
| CHHapticEvent( | |
| eventType: .hapticTransient, | |
| parameters: [ | |
| CHHapticEventParameter(parameterID: .hapticIntensity, value: 0.4), | |
| CHHapticEventParameter(parameterID: .hapticSharpness, value: 0.6) | |
| ], | |
| relativeTime: 0, | |
| duration: 0.025 // very short | |
| ) | |
| ], parameters: []) | |
| let player = try engine.makePlayer(with: tapPattern) | |
| controllers[key]?.buttonTapHapticPlayer = player | |
| } | |
| } catch { | |
| print("❌ Failed to setup controller haptics: \(error)") | |
| } | |
| } | |
| private func playTapHaptic(for key: ObjectIdentifier) { | |
| guard let player = controllers[key]?.buttonTapHapticPlayer else { return } | |
| do { | |
| try player.start(atTime: CHHapticTimeImmediate) | |
| } catch { | |
| print("❌ Failed to play tap haptic: \(error)") | |
| } | |
| } | |
| private func updateTriggerHaptic(for key: ObjectIdentifier, pressure: Float) { | |
| guard let engine = controllers[key]?.hapticEngine else { return } | |
| // Stop existing continuous haptic if pressure is 0 | |
| if pressure == 0 { | |
| if let existingPlayer = controllers[key]?.triggerHapticPlayer { | |
| try? existingPlayer.stop(atTime: CHHapticTimeImmediate) | |
| controllers[key]?.triggerHapticPlayer = nil | |
| } | |
| return | |
| } | |
| // Start continuous haptic if not already running | |
| // Use a long duration so it runs continuously until stopped | |
| guard controllers[key]?.triggerHapticPlayer == nil else { return } | |
| do { | |
| let continuousPattern = try CHHapticPattern(events: [ | |
| CHHapticEvent( | |
| eventType: .hapticContinuous, | |
| parameters: [ | |
| CHHapticEventParameter(parameterID: .hapticIntensity, value: 0.7), | |
| CHHapticEventParameter(parameterID: .hapticSharpness, value: 0.5) | |
| ], | |
| relativeTime: 0, | |
| duration: 60.0 // Very long duration | |
| ) | |
| ], parameters: []) | |
| let player = try engine.makePlayer(with: continuousPattern) | |
| try player.start(atTime: CHHapticTimeImmediate) | |
| controllers[key]?.triggerHapticPlayer = player | |
| } catch { | |
| print("❌ Failed to start continuous trigger haptic: \(error)") | |
| } | |
| } | |
| } | |
| // MARK: - Input Handling | |
| extension GameControllerManager { | |
| /// Helper method to safely update input state for a controller | |
| private func updateInputState(for key: ObjectIdentifier, update: (inout ControllerState.InputState) -> Void) { | |
| guard var controller = controllers[key] else { return } | |
| update(&controller.inputState) | |
| controllers[key] = controller | |
| } | |
| private func setupControllerInputHandlers( | |
| for controller: GCController, | |
| key: ObjectIdentifier, | |
| anchor: AnchorEntity? | |
| ) { | |
| let input = controller.input | |
| // Initialize SF Symbols (so our UI knows what to display for each) | |
| updateInputState(for: key) { state in | |
| state.trigger.sfSymbolName = input.buttons[.trigger]?.sfSymbolsName | |
| state.thumbstickButton.sfSymbolName = input.buttons[.thumbstickButton]?.sfSymbolsName | |
| state.buttonA.sfSymbolName = input.buttons[.a]?.sfSymbolsName | |
| state.buttonB.sfSymbolName = input.buttons[.b]?.sfSymbolsName | |
| state.grip.sfSymbolName = input.buttons[.grip]?.sfSymbolsName | |
| state.menu.sfSymbolName = input.buttons[.menu]?.sfSymbolsName | |
| state.thumbstick.sfSymbolName = input.dpads[.thumbstick]?.sfSymbolsName | |
| } | |
| // Log all available buttons with their identifying information | |
| print("🎮 Available buttons in input:") | |
| input.buttons.forEach { button in | |
| let localizedName = button.localizedName ?? "Unknown" | |
| let sfSymbol = button.sfSymbolsName ?? "none" | |
| let aliases = button.aliases.isEmpty ? "none" : button.aliases.joined(separator: ", ") | |
| print(" - localizedName: \(localizedName)") | |
| print(" sfSymbolsName: \(sfSymbol)") | |
| print(" aliases: \(aliases)") | |
| } | |
| input.buttons[.trigger]?.pressedInput.valueDidChangeHandler = { _, _, pressure in | |
| Task { @MainActor in | |
| self.updateInputState(for: key) { | |
| $0.trigger.pressure = pressure | |
| } | |
| self.updateTriggerHaptic(for: key, pressure: pressure) | |
| } | |
| } | |
| // Secondary action: thumbstick button | |
| input.buttons[.thumbstickButton]?.pressedInput.pressedDidChangeHandler = { _, _, pressed in | |
| Task { @MainActor in | |
| self.updateInputState(for: key) { $0.thumbstickButton.pressure = pressed ? 1.0 : 0 } | |
| if pressed { | |
| self.playTapHaptic(for: key) | |
| } | |
| } | |
| } | |
| // Face buttons | |
| input.buttons[.a]?.pressedInput.pressedDidChangeHandler = { _, _, pressed in | |
| Task { @MainActor in | |
| self.updateInputState(for: key) { $0.buttonA.pressure = pressed ? 1.0 : 0 } | |
| if pressed { self.playTapHaptic(for: key) } | |
| } | |
| } | |
| input.buttons[.b]?.pressedInput.pressedDidChangeHandler = { _, _, pressed in | |
| Task { @MainActor in | |
| self.updateInputState(for: key) { $0.buttonB.pressure = pressed ? 1.0 : 0 } | |
| if pressed { self.playTapHaptic(for: key) } | |
| } | |
| } | |
| // Grip button | |
| input.buttons[.grip]?.pressedInput.pressedDidChangeHandler = { _, _, pressed in | |
| Task { @MainActor in | |
| self.updateInputState(for: key) { $0.grip.pressure = pressed ? 1.0 : 0 } | |
| if pressed { self.playTapHaptic(for: key) } | |
| } | |
| } | |
| // Options button | |
| input.buttons[.menu]?.pressedInput.pressedDidChangeHandler = { _, _, pressed in | |
| Task { @MainActor in | |
| self.updateInputState(for: key) { $0.menu.pressure = pressed ? 1.0 : 0 } | |
| if pressed { self.playTapHaptic(for: key) } | |
| } | |
| } | |
| // Use thumbstick axes | |
| if let thumbstick = input.dpads[.thumbstick] { | |
| thumbstick.xyAxes.valueDidChangeHandler = { element, axisInput, value in | |
| Task { @MainActor in | |
| self.updateInputState(for: key) { | |
| $0.thumbstick.x = value.x | |
| $0.thumbstick.y = value.y | |
| } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| // MARK: - Axis Indicator | |
| extension GameControllerManager { | |
| private func makeAxisIndicator() -> Entity { | |
| let length: Float = 0.02 | |
| let thickness: Float = 0.002 | |
| let originSize: Float = 0.004 | |
| // Origin cube (white) | |
| let originCube = ModelEntity( | |
| mesh: .generateBox(size: .init(repeating: originSize)), | |
| materials: [SimpleMaterial(color: .white, isMetallic: false)] | |
| ) | |
| let xAxis = ModelEntity( | |
| mesh: .generateBox(size: [length, thickness, thickness]), | |
| materials: [SimpleMaterial(color: .systemRed, isMetallic: false)] | |
| ) | |
| xAxis.position = [-length / 2, 0, 0] | |
| let yAxis = ModelEntity( | |
| mesh: .generateBox(size: [thickness, length, thickness]), | |
| materials: [SimpleMaterial(color: .systemGreen, isMetallic: false)] | |
| ) | |
| yAxis.position = [0, length / 2, 0] | |
| let zAxis = ModelEntity( | |
| mesh: .generateBox(size: [thickness, thickness, length]), | |
| materials: [SimpleMaterial(color: .systemBlue, isMetallic: false)] | |
| ) | |
| zAxis.position = [0, 0, -length / 2] | |
| let root = Entity() | |
| root.addChild(originCube) | |
| root.addChild(xAxis) | |
| root.addChild(yAxis) | |
| root.addChild(zAxis) | |
| return root | |
| } | |
| } | |
| // MARK: - Controller Input State View | |
| private struct ControllerInputStateView: View { | |
| var manager: GameControllerManager | |
| let controllerID: ObjectIdentifier | |
| var body: some View { | |
| if let state = manager.controllers[controllerID]?.inputState { | |
| VStack(alignment: .leading, spacing: 12) { | |
| Text(manager.controllers[controllerID]?.controller.vendorName ?? "Controller Input") | |
| .font(.caption) | |
| .foregroundStyle(.secondary) | |
| HStack { | |
| ButtonStatusView(label: "Trig", state: state.trigger) | |
| ButtonStatusView(label: "Grip", state: state.grip) | |
| } | |
| HStack { | |
| ButtonStatusView(label: "A", state: state.buttonA) | |
| ButtonStatusView(label: "B", state: state.buttonB) | |
| } | |
| HStack { | |
| ButtonStatusView(label: "Menu", state: state.menu) | |
| ButtonStatusView(label: "Stick", state: state.thumbstickButton) | |
| } | |
| VStack(alignment: .leading, spacing: 4) { | |
| Text("Thumbstick") | |
| .font(.caption2) | |
| .foregroundStyle(.secondary) | |
| HStack { | |
| ThumbstickView(x: state.thumbstick.x, y: state.thumbstick.y) | |
| VStack(alignment: .leading) { | |
| Text("X: \(state.thumbstick.x, specifier: "%.2f")") | |
| Text("Y: \(state.thumbstick.y, specifier: "%.2f")") | |
| } | |
| .font(.caption2) | |
| .monospacedDigit() | |
| } | |
| } | |
| } | |
| .padding() | |
| .glassBackgroundEffect() | |
| .frame(width: 220) | |
| .allowsHitTesting(false) | |
| } | |
| } | |
| private struct ButtonStatusView: View { | |
| let label: String | |
| let state: ControllerState.InputState.ButtonState | |
| var body: some View { | |
| ZStack { | |
| // Pressure Indicating outer stroke | |
| if state.pressure > 0 { | |
| Circle() | |
| .trim(from: 0, to: CGFloat(state.pressure)) | |
| .stroke( | |
| .green, | |
| style: StrokeStyle(lineWidth: 4, lineCap: .round) | |
| ) | |
| .rotationEffect(.degrees(-90)) | |
| .padding(2) | |
| } | |
| VStack { | |
| if let symbol = state.sfSymbolName { | |
| Image(systemName: symbol) | |
| .font(.title2) | |
| } else { | |
| Text(label) | |
| .font(.caption.bold()) | |
| } | |
| } | |
| } | |
| .frame(width: 44, height: 44) | |
| .background(state.isPressed ? .green.opacity(0.5) : .clear) | |
| .clipShape(Circle()) | |
| .overlay( | |
| Circle().stroke(.primary.opacity(0.2), lineWidth: 1) | |
| ) | |
| } | |
| } | |
| private struct ThumbstickView: View { | |
| let x: Float | |
| let y: Float | |
| var body: some View { | |
| ZStack { | |
| // Grid background | |
| RoundedRectangle(cornerRadius: 8) | |
| .fill(.black.opacity(0.2)) | |
| .frame(width: 60, height: 60) | |
| // Center axes | |
| Path { path in | |
| path.move(to: CGPoint(x: 30, y: 0)) | |
| path.addLine(to: CGPoint(x: 30, y: 60)) | |
| path.move(to: CGPoint(x: 0, y: 30)) | |
| path.addLine(to: CGPoint(x: 60, y: 30)) | |
| } | |
| .stroke(.white.opacity(0.2), lineWidth: 1) | |
| // Thumbstick position indicator | |
| Circle() | |
| .fill(.white) | |
| .frame(width: 8, height: 8) | |
| .offset( | |
| x: CGFloat(x) * 25, | |
| y: CGFloat(-y) * 25 | |
| ) | |
| } | |
| .frame(width: 60, height: 60) | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment