gesture

This device consists of four SG92R micro servos, one MG996R servo, one MG995R servo, and an Arduino.
‍
The machine learning part uses Vision OSC connected to Wekinator. The design logic is that each gesture corresponds to a circle, and the rotation angle of the gesture controls the actual rotation of the small circle. When all four gestures are rotated to the specified angle, the iris structure will rotate and open, and the internal push rod will push out the locked item.

/visionOSC to wekinator
//VisionOSC connect demo created by Revive (Weiqi sun)
import oscP5.*;
import netP5.*;
NetAddress wekinatorAddress;
OscP5 oscP5;
int HAND_N_PART = 21;
int MAX_DET = 32;
class KeyPt {
float x;
float y;
float score;
KeyPt(float _x, float _y, float _score) {
x = _x;
y = _y;
score = _score;
}
}
class PtsDetection {
float score;
KeyPt[] points;
PtsDetection(int n) {
points = new KeyPt[n];
}
}
PtsDetection[] hands;
int nHand = 0;
int vidW;
int vidH;
void setup() {
size(640, 480);
oscP5 = new OscP5(this, 59183);
wekinatorAddress = new NetAddress("127.0.0.1", 6448); // Change to your Wekinator
address and port
hands = new PtsDetection[MAX_DET];
}
void draw() {
background(0);
fill(0, 255, 255);
drawPtsDetection(hands, nHand, 5);
fill(255);
}
void drawPtsDetection(PtsDetection[] dets, int nDet, int rad) {
pushStyle();
noStroke();
for (int i = 0; i < nDet; i++) {
for (int j = 0; j < dets[i].points.length; j++) {
if (dets[i].points[j] != null) {
circle(dets[i].points[j].x, dets[i].points[j].y, rad);
}
}
}
popStyle();
}
int readPtsDetection(OscMessage msg, int nParts, PtsDetection[] out) {
vidW = msg.get(0).intValue();
vidH = msg.get(1).intValue();
int nDet = msg.get(2).intValue();
int n = nParts * 3 + 1;
for (int i = 0; i < nDet; i++) {
PtsDetection det = new PtsDetection(nParts);
det.score = msg.get(3 + i * n).floatValue();
for (int j = 0; j < nParts; j++) {
float x = msg.get(3 + i * n + 1 + j * 3).floatValue();
float y = msg.get(3 + i * n + 1 + j * 3 + 1).floatValue();
float score = msg.get(3 + i * n + 1 + j * 3 + 2).floatValue();
det.points[j] = new KeyPt(x, y, score);
}
out[i] = det;
}
return nDet;
}
void oscEvent(OscMessage msg) {
if (msg.addrPattern().equals("/hands/arr")) {
nHand = readPtsDetection(msg, HAND_N_PART, hands);
// Create a new OSC message for sending relative positions
OscMessage wekinatorMsg = new OscMessage("/wek/inputs");
// Add relative positions of points to the message
for (int i = 0; i < nHand; i++) {
for (int j = 0; j < HAND_N_PART; j++) {
if (hands[i].points[j] != null) {
float relativeX = hands[i].points[j].x / vidW; // Calculate relative X position
float relativeY = hands[i].points[j].y / vidH; // Calculate relative Y position
wekinatorMsg.add(relativeX);
wekinatorMsg.add(relativeY);
}
}
}
// Send the OSC message to Wekinator
oscP5.send(wekinatorMsg, wekinatorAddress);
}
}

//Arduino
#include "PCA9685.h"
#include <Wire.h>
ServoDriver servo;
// Define constants for servo pins
const int servoR1Pin = 1;
const int servoR2Pin = 2;
const int servoR3Pin = 3;
const int servoR4Pin = 4;
const int servoDoorPin = 5;
const int servoR6Pin = 6;
// Define angle range for servoR1, servoR2, servoR3, and servoR4
const int minValidAngle = 20; // Minimum valid angle
const int maxValidAngle = 180; // Maximum valid angle
// Variables to track the current angles
int currentAngleR1 = 90;
int currentAngleR2 = 90;
int currentAngleR3 = 90;
int currentAngleR4 = 90;
int currentAngleDoor = 90;
int currentAngleR6 = 90;
void setup() {
// Join I2C bus (I2Cdev library doesn't do this automatically)
Wire.begin();
Serial.begin(9600);
servo.init(0x7f);
}
void loop() {
static bool servoDoorMoved = false;
if (Serial.available() >= 4) {
char classification = Serial.read();
int angle = 0;
for (int i = 0; i < 3; i++) {
char digit = Serial.read();
if (digit >= '0' && digit <= '9') {
angle = angle * 10 + (digit - '0');
} else {
Serial.println("Error");
return;
}
}
if (classification >= '1' && classification <= '5' && angle >= 0 && angle <= 180) {
switch (classification) {
case '1':
servo.setAngle(servoR1Pin, angle);
break;
case '2':
servo.setAngle(servoR2Pin, angle);
break;
case '3':
servo.setAngle(servoR3Pin, angle);
break;
case '4':
servo.setAngle(servoR4Pin, angle);
break;
case '5':
if ( !servoDoorMoved) {
// Check if servoR1, servoR2, servoR3, and servoR4 are within the valid angle range
if (currentAngleR1 >= minValidAngle && currentAngleR1 <= maxValidAngle &&
currentAngleR2 >= minValidAngle && currentAngleR2 <= maxValidAngle &&
currentAngleR3 >= minValidAngle && currentAngleR3 <= maxValidAngle &&
currentAngleR4 >= minValidAngle && currentAngleR4 <= maxValidAngle) {
servo.setAngle(servoDoorPin, 180);
delay(5000);
servo.setAngle(servoR6Pin, 179);
delay (2000);
servo.setAngle(servoR6Pin, 90);
servoDoorMoved = true;
Serial.println("classifiers " + String(classification) + "，angle：" + String(angle));
} else {
Serial.println("Error: servoR1, servoR2, servoR3, and servoR4 angles not in the
valid range.");
}
} else {
// Handle other servo control cases here
}
}}else {
Serial.println("Error");
}
}
}

import processing.serial.*;
import oscP5.*;
import netP5.*;
int receivedInteger;
// Arduino communication
Serial arduino;
OscP5 oscP5;
// These variables will be synchronized with the Arduino and they should be the same on the
Arduino side.
public int classificationSignal;
public int angle;
int outputInterval = 1000; // Output interval in milliseconds
int lastOutputTime = 0;
void setup() {
size(600, 300);
// Start the communication with Wekinator. Listen on port 12000, return messages on port
6448
oscP5 = new OscP5(this, 12000);
String arduinoPort = "/dev/cu.usbmodem141201"; // Adjust this as necessary
arduino = new Serial(this, arduinoPort, 9600);
}
void draw() {
background(0);
text("Forwarding Wekinator messages to Arduino...", 50, 30);
}
void sendArduinoData(int classificationSignal, int angle) {
classificationSignal = constrain(classificationSignal, 0, 9);
String formattedAngle = String.format("%03d", angle);
String formattedData = classificationSignal + formattedAngle;
arduino.write(formattedData); // Write your message
println("Received OSC Signal: Classification=" + classificationSignal + ", Angle=" +
formattedAngle);
}
// Receive OSC messages from Wekinator
void oscEvent(OscMessage msg) {
if (msg.checkAddrPattern("/wek/outputs")) {
float value1 = msg.get(0).floatValue();
float value2 = msg.get(1).floatValue();
classificationSignal = int(value1);
angle = int(value2);
String formattedAngle = String.format("%03d", angle); // Format angle as a 3-digit string
println("Received OSC Signal: Classification=" + classificationSignal + ", Angle=" +
formattedAngle);
int currentTime = millis();
if (currentTime - lastOutputTime >= outputInterval) {
sendArduinoData(classificationSignal, angle);
lastOutputTime = currentTime;
}
}
}