Skip to content

Instantly share code, notes, and snippets.

@Hirosaji

Hirosaji/.block

Last active December 29, 2023 13:14
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save Hirosaji/00d3c9ceba65ef7ddf0f2890968132dc to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save Hirosaji/00d3c9ceba65ef7ddf0f2890968132dc to your computer and use it in GitHub Desktop.
Download ZIP
three-vrm - VRM motion capture by PoseNet
Raw
.block
# three-vrm - VRM motion capture by PoseNet
license: mit
Raw
README.md
Raw
index.html
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<title>three-vrm example</title>
<meta
name="viewport"
content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no"
/>
<style>
body {
margin: 0;
}
canvas {
display: block;
}
#motion {
position: absolute;
bottom: 0;
right: 0;
color: white;
}
#webacamCanvas { transform: rotateY(180deg); }
</style>
</head>
<body>
<div id="vrm"></div>
<div id="motion">
<div id="loading-indicator">PoseNet model is loading.</div>
<canvas id="webacamCanvas" width="480" height="320"></canvas>
<video id="video" width="480" height="320" style="display:none;" autoplay playsinline>Video stream not
available.</video>
</div>
<script src="https://unpkg.com/[email protected]/build/three.js"></script>
<script src="https://unpkg.com/[email protected]/examples/js/loaders/GLTFLoader.js"></script>
<script src="https://unpkg.com/[email protected]/examples/js/controls/OrbitControls.js"></script>
<script src="./three-vrm.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs"></script>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/posenet"></script>
<script>
//===================================
// motion capture process
let poseStore = {};
const webacamCanvas = document.getElementById("webacamCanvas");
const webcamCtx = webacamCanvas.getContext("2d");
const video = document.getElementById('video');
// display camera movie Canvas detected parts
function detectAndDraw(net) {
webcamCtx.drawImage(video, 0, 0, 480, 320);
net.estimateSinglePose(video, {
flipHorizontal: false
})
.then(function(pose) {
drawKeypoints(pose);
});
}
// draw detected parts by PoseNet
function drawKeypoints(pose) {
pose.keypoints.forEach(keypoint => {
if (keypoint.score > 0.4) {
poseStore[keypoint.part] = {
x: 480/2 - keypoint.position.x,
y: 320/2 - keypoint.position.y
};
webcamCtx.beginPath();
webcamCtx.fillStyle = "rgb(255, 255, 0)"; // 黄色
webcamCtx.arc(
keypoint.position.x,
keypoint.position.y,
5,
(10 * Math.PI) / 180,
(80 * Math.PI) / 180,
true
);
webcamCtx.fill();
webcamCtx.fillText(
keypoint.part,
keypoint.position.x,
keypoint.position.y + 10
);
}
});
}
// get camera movie
navigator.mediaDevices.getUserMedia({ audio: false, video: true })
.then(function (mediaStream) {
// set video tag srcObject
video.srcObject = mediaStream;
video.onloadedmetadata = function (e) {
video.play();
};
return posenet.load();
})
.then(function (net) {
var loadingIndicator = document.getElementById("loading-indicator");
loadingIndicator.style.display = 'none';
setInterval(function () { detectAndDraw(net); }, 100);
});
//===================================
// renderer
const renderer = new THREE.WebGLRenderer();
renderer.setSize( window.innerWidth, window.innerHeight );
renderer.setPixelRatio( window.devicePixelRatio );
document.getElementById('vrm').appendChild( renderer.domElement );
// camera
const camera = new THREE.PerspectiveCamera( 30.0, window.innerWidth / window.innerHeight, 0.1, 20.0 );
camera.position.set( 0.0, 1.0, 5.0 );
// camera controls
const controls = new THREE.OrbitControls( camera, renderer.domElement );
controls.screenSpacePanning = true;
controls.target.set( 0.0, 1.0, 0.0 );
controls.update();
// scene
const scene = new THREE.Scene();
// light
const light = new THREE.DirectionalLight( 0xffffff );
light.position.set( 1.0, 1.0, 1.0 ).normalize();
scene.add( light );
// gltf and vrm
let currentVrm = undefined;
const loader = new THREE.GLTFLoader();
loader.crossOrigin = 'anonymous';
loader.load(
"./three-vrm-girl.vrm",
( gltf ) => {
console.log(gltf)
THREE.VRM.from( gltf ).then( ( vrm ) => {
scene.add( vrm.scene );
currentVrm = vrm;
vrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.Hips ).rotation.y = Math.PI;
console.log( vrm );
} );
},
( progress ) => console.log( 'Loading model...', 100.0 * ( progress.loaded / progress.total ), '%' ),
( error ) => console.error( error )
);
// helpers
const gridHelper = new THREE.GridHelper( 10, 10 );
scene.add( gridHelper );
const axesHelper = new THREE.AxesHelper( 5 );
scene.add( axesHelper );
// animate
const clock = new THREE.Clock();
let angleStore = {};
// X axis
function getAngleFromX(pos2, pos1) {
return Math.atan2(pos2.y - pos1.y, pos2.x - pos1.x);
}
function animate() {
requestAnimationFrame( animate );
const deltaTime = clock.getDelta();
if ( currentVrm ) {
if (poseStore) {
if (poseStore.leftShoulder && poseStore.rightShoulder) {
// spine & shoulder
let angle = getAngleFromX(poseStore.rightShoulder, poseStore.leftShoulder);
if (angle !== null) {
angle = angle * -1;
angleStore.Spine = angle;
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.Spine ).rotation.z = angle;
}
}
if (poseStore.leftEye && poseStore.rightEye) {
// neck $ eyes
let angle = getAngleFromX(poseStore.rightEye, poseStore.leftEye);
if (angle !== null) {
angle = angle * -1;
angleStore.Neck = angle;
angle = angle - (angleStore.Spine || 0);
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.Neck ).rotation.z = angle;
}
}
if (poseStore.leftShoulder && poseStore.leftElbow) {
// arms
let angle = getAngleFromX(poseStore.leftElbow, poseStore.leftShoulder);
if (angle !== null) {
angle = Math.PI - angle;
angleStore.RightUpperArm = angle;
angle = angle - (angleStore.Spine || 0);
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.RightUpperArm ).rotation.z = angle;
}
}
if (poseStore.leftWrist && poseStore.leftElbow) {
// arms
let angle = getAngleFromX(poseStore.leftWrist, poseStore.leftElbow);
if (angle !== null) {
angle = Math.PI - angle;
angleStore.RightLowerArm = angle;
angle = angle - (angleStore.RightUpperArm || 0);
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.RightLowerArm ).rotation.z = angle;
}
}
if (poseStore.rightShoulder && poseStore.rightElbow) {
// arms
let angle = getAngleFromX(poseStore.rightElbow, poseStore.rightShoulder);
if (angle !== null) {
angle = angle * -1;
angleStore.LeftUpperArm = angle;
angle = angle - (angleStore.Spine || 0);
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.LeftUpperArm ).rotation.z = angle;
}
}
if (poseStore.rightWrist && poseStore.rightElbow) {
// arms
let angle = getAngleFromX(poseStore.rightWrist, poseStore.rightElbow);
if (angle !== null) {
angle = angle * -1;
angleStore.LeftLowerArm = angle;
angle = angle - (angleStore.LeftUpperArm || 0);
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.LeftLowerArm ).rotation.z = angle;
}
}
}
// update vrm
currentVrm.update( deltaTime );
}
renderer.render( scene, camera );
}
animate();
</script>
</body>
</html>
Raw
three-vrm-girl.vrm
This file has been truncated, but you can view the full file.
View raw
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment