Skip to content

Commit 7340240

Browse files
VirtualStarryVirtualStarry
committed
feat(hand&body): 支持人体&手势检测
1 parent 702e2eb commit 7340240

14 files changed

+1148
-0
lines changed

Diff for: ‎miniprogram/app.json

+2
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@
7474
"pages/ar/osd-ar/osd-ar",
7575
"pages/ar/face-detect/face-detect",
7676
"pages/ar/photo-face-detect/photo-face-detect",
77+
"pages/ar/hand-detect/hand-detect",
78+
"pages/ar/photo-hand-detect/photo-hand-detect",
7779
"pages/ar/body-detect/body-detect",
7880
"pages/ar/photo-body-detect/photo-body-detect",
7981
"pages/page/set-navigation-bar-title/set-navigation-bar-title",

Diff for: ‎miniprogram/packageAPI/pages/ar/hand-detect/behavior.js

+190
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
import {
2+
createScopedThreejs
3+
} from './threejs-miniprogram'
4+
import {
5+
registerGLTFLoader
6+
} from '../loaders/gltf-loader'
7+
8+
const info = wx.getSystemInfoSync()
9+
10+
export default function getBehavior() {
11+
return Behavior({
12+
data: {
13+
width: 1,
14+
height: 1,
15+
fps: 0,
16+
memory: 0,
17+
cpu: 0,
18+
},
19+
methods: {
20+
onReady() {
21+
wx.createSelectorQuery()
22+
.select('#webgl')
23+
.node()
24+
.exec(res => {
25+
this.canvas = res[0].node
26+
27+
const info = wx.getSystemInfoSync()
28+
const pixelRatio = info.pixelRatio
29+
const calcSize = (width, height) => {
30+
console.log(`canvas size: width = ${width} , height = ${height}`)
31+
this.canvas.width = width * pixelRatio / 2
32+
this.canvas.height = height * pixelRatio / 2
33+
this.setData({
34+
width,
35+
height,
36+
})
37+
}
38+
calcSize(info.windowWidth, info.windowHeight * 0.8)
39+
40+
this.initVK()
41+
})
42+
},
43+
onUnload() {
44+
if (this._texture) {
45+
this._texture.dispose()
46+
this._texture = null
47+
}
48+
if (this.renderer) {
49+
this.renderer.dispose()
50+
this.renderer = null
51+
}
52+
if (this.scene) {
53+
this.scene.dispose()
54+
this.scene = null
55+
}
56+
if (this.camera) this.camera = null
57+
if (this.model) this.model = null
58+
if (this._insertModel) this._insertModel = null
59+
if (this._insertModels) this._insertModels = null
60+
if (this.planeBox) this.planeBox = null
61+
if (this.mixers) {
62+
this.mixers.forEach(mixer => mixer.uncacheRoot(mixer.getRoot()))
63+
this.mixers = null
64+
}
65+
if (this.clock) this.clock = null
66+
67+
if (this.THREE) this.THREE = null
68+
if (this._tempTexture && this._tempTexture.gl) {
69+
this._tempTexture.gl.deleteTexture(this._tempTexture)
70+
this._tempTexture = null
71+
}
72+
if (this._fb && this._fb.gl) {
73+
this._fb.gl.deleteFramebuffer(this._fb)
74+
this._fb = null
75+
}
76+
if (this._program && this._program.gl) {
77+
this._program.gl.deleteProgram(this._program)
78+
this._program = null
79+
}
80+
if (this.canvas) this.canvas = null
81+
if (this.gl) this.gl = null
82+
if (this.session) this.session = null
83+
if (this.anchor2DList) this.anchor2DList = []
84+
},
85+
initVK() {
86+
// 初始化 threejs
87+
this.initTHREE()
88+
89+
// 自定义初始化
90+
if (this.init) this.init()
91+
92+
console.log('this.gl', this.gl)
93+
94+
const session = this.session = wx.createVKSession({
95+
track: {
96+
plane: {
97+
mode: 3
98+
},
99+
hand: {
100+
mode: 1
101+
}
102+
},
103+
version: 'v1',
104+
gl: this.gl
105+
})
106+
session.start(err => {
107+
if (err) return console.error('VK error: ', err)
108+
console.log('@@@@@@@@ VKSession.version', session.version)
109+
110+
const canvas = this.canvas
111+
112+
const calcSize = (width, height, pixelRatio) => {
113+
console.log(`canvas size: width = ${width} , height = ${height}`)
114+
this.canvas.width = width * pixelRatio / 2
115+
this.canvas.height = height * pixelRatio / 2
116+
this.setData({
117+
width,
118+
height,
119+
})
120+
}
121+
122+
session.on('resize', () => {
123+
const info = wx.getSystemInfoSync()
124+
calcSize(info.windowWidth, info.windowHeight * 0.8, info.pixelRatio)
125+
})
126+
127+
session.on('addAnchors', anchors => {
128+
this.data.anchor2DList = anchors.map(anchor => ({
129+
points: anchor.points,
130+
origin: anchor.origin,
131+
size: anchor.size
132+
}))
133+
})
134+
135+
session.on('updateAnchors', anchors => {
136+
this.data.anchor2DList = []
137+
// 摄像头实时检测人手的时候 updateAnchors 会在每帧触发，所以性能要求更高，用 gl 画
138+
this.data.anchor2DList = this.data.anchor2DList.concat(anchors.map(anchor => ({
139+
points: anchor.points,
140+
origin: anchor.origin,
141+
size: anchor.size
142+
})))
143+
})
144+
145+
session.on('removeAnchors', anchors => {
146+
this.data.anchor2DList = []
147+
})
148+
149+
// 逐帧渲染
150+
const onFrame = timestamp => {
151+
// let start = Date.now()
152+
const frame = session.getVKFrame(canvas.width, canvas.height)
153+
if (frame) {
154+
this.render(frame)
155+
}
156+
157+
session.requestAnimationFrame(onFrame)
158+
}
159+
session.requestAnimationFrame(onFrame)
160+
})
161+
},
162+
initTHREE() {
163+
const THREE = this.THREE = createScopedThreejs(this.canvas)
164+
registerGLTFLoader(THREE)
165+
166+
// 相机
167+
this.camera = new THREE.Camera()
168+
169+
// 场景
170+
const scene = this.scene = new THREE.Scene()
171+
172+
// 光源
173+
const light1 = new THREE.HemisphereLight(0xffffff, 0x444444) // 半球光
174+
light1.position.set(0, 0.2, 0)
175+
scene.add(light1)
176+
const light2 = new THREE.DirectionalLight(0xffffff) // 平行光
177+
light2.position.set(0, 0.2, 0.1)
178+
scene.add(light2)
179+
180+
// 渲染层
181+
const renderer = this.renderer = new THREE.WebGLRenderer({
182+
antialias: true,
183+
alpha: true
184+
})
185+
renderer.gammaOutput = true
186+
renderer.gammaFactor = 2.2
187+
},
188+
},
189+
})
190+
}

0 commit comments

Comments
 (0)