Skip to content

Commit 63be86b

Browse files
jack-arturojack-arturo
committed
fix: Update iPhone hand tracking landmark mapping and debug logs
Revised iPhone hand landmark keys to match Vision framework abbreviations and updated all related calculations to use new keys. Added debug logging for tracking source changes, incoming iPhone messages, and landmark keys. Improved LiDAR depth normalization for MediaPipe compatibility and enhanced error handling and message logging in the WebSocket connection.
1 parent 57cd33a commit 63be86b

3 files changed

Lines changed: 116 additions & 60 deletions

File tree

‎src/components/GraphCanvas.tsx‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,13 @@ export function GraphCanvas({
147147
const gestureState = source === 'iphone' ? iphoneState : mediapipeState
148148
const gesturesActive = source === 'iphone' ? iphoneConnected : mediapipeActive
149149

150+
// Debug log when source changes
151+
useEffect(() => {
152+
console.log(`🎯 Tracking source: ${source}, enabled: ${gestureControlEnabled}, active: ${gesturesActive}`)
153+
console.log(` iPhone: connected=${iphoneConnected}, hands=${iphoneState.handsDetected}`)
154+
console.log(` MediaPipe: active=${mediapipeActive}, hands=${mediapipeState.handsDetected}`)
155+
}, [source, gestureControlEnabled, gesturesActive, iphoneConnected, mediapipeActive, iphoneState.handsDetected, mediapipeState.handsDetected])
156+
150157
useEffect(() => {
151158
onTrackingInfoChange?.({
152159
source,
@@ -568,9 +575,9 @@ function Scene({
568575
}
569576

570577
// Decay smoothed values
571-
smoothed.translateZ *= 0.9
572-
smoothed.rotateX *= 0.9
573-
smoothed.rotateY *= 0.9
578+
smoothed.translateZ *= 0.9
579+
smoothed.rotateX *= 0.9
580+
smoothed.rotateY *= 0.9
574581

575582
// Gentle recenter: slowly pull cloud back toward origin
576583
group.position.x *= (1 - RECENTER_STRENGTH)
@@ -652,14 +659,14 @@ function Scene({
652659

653660
{/* LOD Labels - only for selected/hovered/nearby nodes */}
654661
{displayConfig.showLabels && (
655-
<LODLabels
656-
nodes={layoutNodes}
657-
selectedNode={selectedNode}
658-
hoveredNode={hoveredNode}
659-
searchTerm={searchTerm}
662+
<LODLabels
663+
nodes={layoutNodes}
664+
selectedNode={selectedNode}
665+
hoveredNode={hoveredNode}
666+
searchTerm={searchTerm}
660667
labelFadeDistance={displayConfig.labelFadeDistance}
661-
matchingIds={matchingIds}
662-
/>
668+
matchingIds={matchingIds}
669+
/>
663670
)}
664671

665672
{/* Expanded Node View - shows when a node is selected via hand */}

‎src/components/Hand2DOverlay.tsx‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -583,19 +583,19 @@ function LaserBeam({ ray, stableRay, color, isGripped, hasHit = false }: LaserBe
583583
originX = (1 - ray.origin.x) * 100
584584
originY = ray.origin.y * 100
585585

586-
const centerX = 50
587-
const centerY = 50
588-
const handDeviationX = (originX - 50) * LASER_DEVIATION_SCALE
589-
const handDeviationY = (originY - 50) * LASER_DEVIATION_SCALE
590-
const targetX = centerX - handDeviationX * (1 - LASER_CENTER_BIAS)
591-
const targetY = centerY - handDeviationY * (1 - LASER_CENTER_BIAS)
592-
593-
const toCenterX = targetX - originX
594-
const toCenterY = targetY - originY
595-
const dist = Math.sqrt(toCenterX * toCenterX + toCenterY * toCenterY)
596-
const normX = dist > 0 ? toCenterX / dist : 0
597-
const normY = dist > 0 ? toCenterY / dist : 0
598-
const laserLength = dist
586+
const centerX = 50
587+
const centerY = 50
588+
const handDeviationX = (originX - 50) * LASER_DEVIATION_SCALE
589+
const handDeviationY = (originY - 50) * LASER_DEVIATION_SCALE
590+
const targetX = centerX - handDeviationX * (1 - LASER_CENTER_BIAS)
591+
const targetY = centerY - handDeviationY * (1 - LASER_CENTER_BIAS)
592+
593+
const toCenterX = targetX - originX
594+
const toCenterY = targetY - originY
595+
const dist = Math.sqrt(toCenterX * toCenterX + toCenterY * toCenterY)
596+
const normX = dist > 0 ? toCenterX / dist : 0
597+
const normY = dist > 0 ? toCenterY / dist : 0
598+
const laserLength = dist
599599

600600
endX = originX + normX * laserLength
601601
endY = originY + normY * laserLength

‎src/hooks/useIPhoneHandTracking.ts‎

Lines changed: 86 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -12,28 +12,36 @@ import type { GestureState, HandLandmarks, PinchRay } from './useHandGestures'
1212
import type { NormalizedLandmarkList } from '@mediapipe/hands'
1313

1414
// iPhone landmark names to MediaPipe indices
15+
// Vision framework uses abbreviated keys: VNHLK + finger letter + joint
16+
// W=wrist, T=thumb, I=index, M=middle, R=ring, P=pinky (little)
1517
const LANDMARK_MAP: Record<string, number> = {
16-
'VNHLKJWRIST': 0,
17-
'VNHLKJTHUMBCMC': 1,
18-
'VNHLKJTHUMBMP': 2,
19-
'VNHLKJTHUMBIP': 3,
20-
'VNHLKJTHUMBTIP': 4,
21-
'VNHLKJINDEXMCP': 5,
22-
'VNHLKJINDEXPIP': 6,
23-
'VNHLKJINDEXDIP': 7,
24-
'VNHLKJINDEXTIP': 8,
25-
'VNHLKJMIDDLEMCP': 9,
26-
'VNHLKJMIDDLEPIP': 10,
27-
'VNHLKJMIDDLEDIP': 11,
28-
'VNHLKJMIDDLETIP': 12,
29-
'VNHLKJRINGMCP': 13,
30-
'VNHLKJRINGPIP': 14,
31-
'VNHLKJRINGDIP': 15,
32-
'VNHLKJRINGTIP': 16,
33-
'VNHLKJLITTLEMCP': 17,
34-
'VNHLKJLITTLEPIP': 18,
35-
'VNHLKJLITTLEDIP': 19,
36-
'VNHLKJLITTLETIP': 20,
18+
// Wrist
19+
'VNHLKWRI': 0,
20+
// Thumb (T)
21+
'VNHLKTCMC': 1,
22+
'VNHLKTMP': 2,
23+
'VNHLKTIP': 3,
24+
'VNHLKTTIP': 4,
25+
// Index (I)
26+
'VNHLKIMCP': 5,
27+
'VNHLKIPIP': 6,
28+
'VNHLKIDIP': 7,
29+
'VNHLKITIP': 8,
30+
// Middle (M)
31+
'VNHLKMMCP': 9,
32+
'VNHLKMPIP': 10,
33+
'VNHLKMDIP': 11,
34+
'VNHLKMTIP': 12,
35+
// Ring (R)
36+
'VNHLKRMCP': 13,
37+
'VNHLKRPIP': 14,
38+
'VNHLKRDIP': 15,
39+
'VNHLKRTIP': 16,
40+
// Little/Pinky (P)
41+
'VNHLKPMCP': 17,
42+
'VNHLKPPIP': 18,
43+
'VNHLKPDIP': 19,
44+
'VNHLKPTIP': 20,
3745
}
3846

3947
interface IPhoneLandmark {
@@ -86,8 +94,8 @@ function distance3D(a: IPhoneLandmark, b: IPhoneLandmark): number {
8694

8795
// Calculate pinch strength from iPhone landmarks
8896
function calculatePinchStrength(landmarks: Record<string, IPhoneLandmark>): number {
89-
const thumbTip = landmarks['VNHLKJTHUMBTIP']
90-
const indexTip = landmarks['VNHLKJINDEXTIP']
97+
const thumbTip = landmarks['VNHLKTTIP']
98+
const indexTip = landmarks['VNHLKITIP']
9199
if (!thumbTip || !indexTip) return 0
92100

93101
const dist = distance3D(thumbTip, indexTip)
@@ -97,10 +105,10 @@ function calculatePinchStrength(landmarks: Record<string, IPhoneLandmark>): numb
97105

98106
// Closed fist strength from fingertip-to-wrist distances
99107
function calculateGrabStrength(landmarks: Record<string, IPhoneLandmark>): number {
100-
const wrist = landmarks['VNHLKJWRIST']
108+
const wrist = landmarks['VNHLKWRI']
101109
if (!wrist) return 0
102110

103-
const tips = ['VNHLKJTHUMBTIP', 'VNHLKJINDEXTIP', 'VNHLKJMIDDLETIP', 'VNHLKJRINGTIP', 'VNHLKJLITTLETIP']
111+
const tips = ['VNHLKTTIP', 'VNHLKITIP', 'VNHLKMTIP', 'VNHLKRTIP', 'VNHLKPTIP']
104112
.map((k) => landmarks[k])
105113
.filter(Boolean) as IPhoneLandmark[]
106114
if (tips.length < 3) return 0
@@ -110,29 +118,33 @@ function calculateGrabStrength(landmarks: Record<string, IPhoneLandmark>): numbe
110118
return Math.max(0, Math.min(1, 1 - (avg - 0.08) / 0.17))
111119
}
112120

113-
// Calculate pinch ray from iPhone landmarks (with REAL depth!)
121+
// Calculate pinch ray from iPhone landmarks (with normalized depth)
114122
function calculatePinchRay(landmarks: Record<string, IPhoneLandmark>, hasLiDAR: boolean): PinchRay {
115-
const thumbTip = landmarks['VNHLKJTHUMBTIP']
116-
const indexTip = landmarks['VNHLKJINDEXTIP']
117-
const wrist = landmarks['VNHLKJWRIST']
123+
const thumbTip = landmarks['VNHLKTTIP']
124+
const indexTip = landmarks['VNHLKITIP']
125+
const wrist = landmarks['VNHLKWRI']
118126

119127
if (!thumbTip || !indexTip || !wrist) {
120128
return { origin: { x: 0.5, y: 0.5, z: 0 }, direction: { x: 0, y: 0, z: -1 }, isValid: false, strength: 0 }
121129
}
122130

131+
// Normalize depths for consistent scaling
132+
const thumbZ = normalizeLiDARDepth(thumbTip.z, hasLiDAR)
133+
const indexZ = normalizeLiDARDepth(indexTip.z, hasLiDAR)
134+
const wristZ = normalizeLiDARDepth(wrist.z, hasLiDAR)
135+
123136
// Origin: midpoint between thumb and index tips
124137
const origin = {
125138
x: (thumbTip.x + indexTip.x) / 2,
126139
y: (thumbTip.y + indexTip.y) / 2,
127-
// Use REAL depth from LiDAR if available!
128-
z: hasLiDAR ? (thumbTip.z + indexTip.z) / 2 : 0,
140+
z: (thumbZ + indexZ) / 2,
129141
}
130142

131143
// Direction: from wrist through pinch point
132144
const rawDir = {
133145
x: origin.x - wrist.x,
134146
y: origin.y - wrist.y,
135-
z: hasLiDAR ? (origin.z - wrist.z) : -0.5, // Default forward if no LiDAR
147+
z: origin.z - wristZ || -0.1, // Small default forward if no difference
136148
}
137149

138150
const length = Math.sqrt(rawDir.x * rawDir.x + rawDir.y * rawDir.y + rawDir.z * rawDir.z)
@@ -146,8 +158,20 @@ function calculatePinchRay(landmarks: Record<string, IPhoneLandmark>, hasLiDAR:
146158
return { origin, direction, isValid, strength }
147159
}
148160

161+
// Normalize LiDAR depth (meters) to MediaPipe-like relative depth
162+
// LiDAR: 0.3m (close) to 3.0m (far) -> MediaPipe-like: 0.1 to -0.3
163+
// Reference: ~1.0m is "neutral" -> 0
164+
function normalizeLiDARDepth(depthMeters: number, hasLiDAR: boolean): number {
165+
if (!hasLiDAR || depthMeters === 0) return 0
166+
167+
// Invert and scale: closer = positive, farther = negative
168+
// At 1.0m -> 0, at 0.5m -> 0.1, at 2.0m -> -0.2
169+
const normalized = (1.0 - depthMeters) * 0.2
170+
return Math.max(-0.5, Math.min(0.5, normalized))
171+
}
172+
149173
// Convert iPhone landmarks to MediaPipe-compatible format
150-
function convertToMediaPipeLandmarks(landmarks: Record<string, IPhoneLandmark>): NormalizedLandmarkList {
174+
function convertToMediaPipeLandmarks(landmarks: Record<string, IPhoneLandmark>, hasLiDAR: boolean = false): NormalizedLandmarkList {
151175
const result: NormalizedLandmarkList = []
152176

153177
// Initialize all 21 landmarks with defaults
@@ -162,7 +186,8 @@ function convertToMediaPipeLandmarks(landmarks: Record<string, IPhoneLandmark>):
162186
result[idx] = {
163187
x: lm.x,
164188
y: lm.y,
165-
z: lm.z,
189+
// Normalize LiDAR depth to MediaPipe-like values
190+
z: normalizeLiDARDepth(lm.z, hasLiDAR),
166191
visibility: 1,
167192
}
168193
}
@@ -213,6 +238,8 @@ export function useIPhoneHandTracking(options: UseIPhoneHandTrackingOptions = {}
213238
const frameCountRef = useRef(0)
214239
const lastFpsTimeRef = useRef(Date.now())
215240
const reconnectTimeoutRef = useRef<number>()
241+
const messageCountRef = useRef(0)
242+
const hasLoggedLandmarksRef = useRef(false)
216243

217244
// Process incoming hand data
218245
const processMessage = useCallback((data: IPhoneMessage) => {
@@ -235,7 +262,16 @@ export function useIPhoneHandTracking(options: UseIPhoneHandTrackingOptions = {}
235262

236263
// Process each hand
237264
for (const hand of data.hands) {
238-
const landmarks = convertToMediaPipeLandmarks(hand.landmarks)
265+
// Debug: log the actual landmark keys from iPhone (once per connection)
266+
if (!hasLoggedLandmarksRef.current && Object.keys(hand.landmarks).length > 0) {
267+
hasLoggedLandmarksRef.current = true
268+
console.log('📍 iPhone landmark keys:', Object.keys(hand.landmarks))
269+
console.log('📍 Expected keys:', Object.keys(LANDMARK_MAP))
270+
const sampleEntry = Object.entries(hand.landmarks)[0]
271+
console.log('📍 Sample landmark:', sampleEntry?.[0], '→', sampleEntry?.[1])
272+
}
273+
274+
const landmarks = convertToMediaPipeLandmarks(hand.landmarks, hand.hasLiDARDepth)
239275
const handData: HandLandmarks = {
240276
landmarks,
241277
worldLandmarks: landmarks,
@@ -322,22 +358,35 @@ export function useIPhoneHandTracking(options: UseIPhoneHandTrackingOptions = {}
322358
wsRef.current = ws
323359

324360
ws.onopen = () => {
325-
console.log('📱 Connected to iPhone hand tracking')
361+
console.log('📱 Connected to iPhone hand tracking bridge')
326362
setIsConnected(true)
363+
messageCountRef.current = 0
364+
hasLoggedLandmarksRef.current = false
327365
}
328366

329367
ws.onmessage = (event) => {
330368
try {
331369
const data = JSON.parse(event.data) as IPhoneMessage
370+
messageCountRef.current++
371+
372+
// Log first few messages and then periodically
373+
if (messageCountRef.current <= 3 || messageCountRef.current % 100 === 0) {
374+
console.log(`📨 Message #${messageCountRef.current}:`, data.type, data.hands?.length || 0, 'hands')
375+
}
376+
332377
if (data.type === 'hand_tracking') {
333378
processMessage(data)
334379
} else if (data.type === 'bridge_status') {
380+
console.log('📡 Bridge status:', data)
335381
if (typeof data.phoneConnected === 'boolean') setPhoneConnected(data.phoneConnected)
336382
if (Array.isArray(data.ips)) setBridgeIps(data.ips)
337383
if (typeof data.phonePort === 'number') setPhonePort(data.phonePort)
384+
} else {
385+
// Debug: log unexpected message types
386+
console.log('📨 Unknown message type:', data.type, data)
338387
}
339388
} catch (e) {
340-
console.error('Parse error:', e)
389+
console.error('Parse error:', e, event.data)
341390
}
342391
}
343392

0 commit comments

Comments
 (0)