CTS (Camera Tracking System)

Application notes
Compact all in one solution to add vision to your project.

Copyright ThinkSmallThings 2013

Revision History
Date
Version
01/24/2013 V1.0

Author
Claudiu G.

Description
Initial release

References
[1] Online documentation: http://thinksmallthings.wordpress.com
[2] Processing: http://www.processing.org

Index
1 Introduction ............................................................................................... 5
2 Streaming .................................................................................................. 6
2.1 Image Data format ................................................................................... 6
2.2 Using Processing ..................................................................................... 6
2.3 YUV422 .................................................................................................... 7
2.4 YUV211 .................................................................................................... 8
2.5 B&W ......................................................................................................... 9
3 Tracking ................................................................................................... 10
3.1 Data format ............................................................................................. 10
3.2 Fast Tracking .......................................................................................... 10
3.3 Tracking multiple blobs ........................................................................... 11
4 Appendix.................................................................................................. 13

Copyright/Licensing/Disclaimer Info
CTS hardware Copyright 2013 Claudiu Georgescu
CTS Client PC Software Copyright 2013 Claudiu Georgescu
All material in this document is Copyright 2013 by Claudiu Georgescu
Both CTS and CTS Client PC software are licensed under the GNU General
Public License. If interested in an alternative licensing scheme, please contact
Claudiu Georgescu @ http://thinksmallthings.wordpress.com.
The CTS solution is distributed in the hope that it will be useful. However, no
warranties, either expressed or implied, are made regarding the operation,
use or results of this system. The CTS should not be used in any life-critical
applications.

1 Introduction
This document covers details on how to access and use CTS module within
an application. For convenience PROCESSING was used in most of the
examples within this document.

2 Streaming
One of the CTS features is the ability to capture and stream data to a client
over serial link.

2.1 Image Data format

Details about data format are presented in CTS Module, User Manual
document, however, for quick reference see bellow:

Pic 2.1: YUV422

Pic 2.2: YUV211

Pic 2.3: B & W

2.2 Using Processing

The purpose of this document is not to explain how PROCESSING is working
but rather to explain how data generated by CTS module can be used from
other applications.
Each of the following sections of this document will use the bellow section the
way is presented bellow, as such, will only be presented here and not in every
chapter.
import processing.serial.*;
String myString = null;
Serial myPort;
PFont f;

void setup(){
println(Serial.list());
myPort = new Serial(this, "/dev/cu.usbserial-A800GKF2", 115200);
size(320, 240);
delay(500);
delay(50);
// CTS: Set image resolution to QVGA
myPort.write("r0");
delay(50);
// CTS: Set image orientation
myPort.write("fn");
delay(50);
// CTS: Set image brightness
myPort.write("b2");
delay(50);
// CTS: Set image contrast
myPort.write("c4");
delay(50);
// CTS: Set image saturation
myPort.write("s4");
delay(50);
// CTS: Set image gamma
myPort.write("g5");
delay(50);
// CTS: Set image white balance
myPort.write("a1");
delay(50);
// CTS: Set image sharpness
myPort.write("Sd");
delay(50);
// CTS: Set learning box width and height
myPort.write("Lb281e");
delay(50);
// CTS: initiate training for color class 0
myPort.write("Lc0");
delay(50);
}
int unsignedByte( int val ) {
return ( val < 0 ? 256+val : val );
}

NOTE: The part in red need to be changed to match existing configuration as

is specific to OS and existing hardware

2.3 YUV422
Bellow is the code to capture and display a frame in YUV422 format:
void draw() {
int x = 0;
int y = 0;
float R, G, B;
int
int
int
int

Y = 0;
Y1 = 0;
U = 0;
V = 0;

myPort.write("FY4");
while (myPort.available() > 0){

byte[] inBuffer = new byte[2*H*W+4];

// Read image from serial port
myPort.readBytes(inBuffer);
if ((inBuffer != null) && (inBuffer[0] == 0x46) && (inBuffer[1] == 0)){
for (y=0;y<H;y++){
for (x=0;x<W/2;x++){
noStroke();
colorMode(RGB, 255);
Y
V
Y1
U

// Prepare YUV values

= inBuffer[y*2*W+x*4+2]
= inBuffer[y*2*W+x*4+2+1]
= inBuffer[y*2*W+x*4+2+2]
= inBuffer[y*2*W+x*4+2+3]

// Calculate RGB from YUV,

B = 1.164*(Y - 16) + 2.018*(U
G = 1.164*(Y - 16) - 0.813*(V
R = 1.164*(Y - 16) + 1.596*(V

&
&
&
&

0xFF;
0xFF;
0xFF;
0xFF;
pixel n
- 128);
- 128) - 0.391*(U - 128);
- 128);

fill(R, G, B);
rect(x*2, y, 1, 1);
// Calculate RGB from YUV, pixel n + 1
B= 1.164*(Y1 - 16) + 2.018*(U - 128);
G = 1.164*(Y1 - 16) - 0.813*(V - 128) - 0.391*(U - 128);
R = 1.164*(Y1 - 16) + 1.596*(V - 128);
fill(R, G, B);
rect(x*2+1, y, 1, 1);
}
}
}
myPort.clear();
}
}

2.4 YUV211
Bellow is the code to capture and display a frame in YUV211 format:
void draw
int pos
int x =
int y =

() {
= 0;
0;
0;

float R = 0;
float G = 0;
float B = 0;
int
int
int
int

Y = 0;
Y1 = 0;
U = 0;
V = 0;

myPort.write("FY2");
while (myPort.available() > 0){
byte[] inBuffer = new byte[H*W+4];
// Read image from serial port
myPort.readBytes(inBuffer);
if ((inBuffer != null) && (inBuffer[0] == 0x46) && (inBuffer[1] == 0)){
for (y=0;y<H;y++){
for (x=0;x<W/2;x++){
noStroke();
colorMode(RGB, 255);
// Prepare YUV values
Y = inBuffer[y*W+x*2+3]
U = (inBuffer[y*W+x*2+3]
Y1= inBuffer[y*W+x*2+3+1]
V = (inBuffer[y*W+x*2+3+1]

&
&
&
&

0xF0;
0x0F) << 4;
0xF0;
0x0F) << 4;

// Calculate RGB from YUV, pixel n

B = 1.164*(Y - 16) + 2.018*(U - 128);

G = 1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128);

R = 1.164*(Y - 16) + 1.596*(V - 128);
fill(R, G, B);
rect(2*x, y, 1, 1);
// Calculate RGB from YUV, pixel n + 1
B= 1.164*(Y1 - 16) + 2.018*(U - 128);
G = 1.164*(Y1 - 16) - 0.813*(V - 128) - 0.391*(U - 128);
R = 1.164*(Y1 - 16) + 1.596*(V - 128);
fill(R, G, B);
rect(x*2+1, y, 1, 1);
}
}
}
myPort.clear();
}
}

2.5 B&W
Bellow is the code to capture and display a frame in B&W format:
void draw() {
int pos = 0;
int x = 0;
int y = 0;
int pixel = 0;
int Y = 0;
int Y1 = 0;
myPort.write("FYb");
while (myPort.available () > 0) {
byte[] inBuffer = new byte[H*(W/2)+5];
// Read image from serial port
myPort.readBytes(inBuffer);
if ((inBuffer != null) && (inBuffer[0] == 0x46) && (inBuffer[1] == 0)) {
for (y=0;y<H;y++) {
for (x=0;x<W/2;x++) {
noStroke();
colorMode(RGB, 255);
pixel = inBuffer[y*(W/2)+x+3] & 0xFF;
// calculate pixel values
Y = pixel & 0xF0;
Y1 =(pixel & 0x0F) << 4;
// Display pixel n
fill(Y);
rect(2*x, y, 1, 1);
// Display pixel n + 1
fill(Y1);
rect(x*2+1, y, 1, 1);
}
}
}
myPort.clear();
}
}

3 Tracking
While in Tracking mode, CTS may return tracking information and image data.
For image data, same code as in Streaming section (Chapter 2) and will not
be discussed here.
As a rule, the image data if enabled will always be streamed first and then the
tracking information (data format is explained in details in CTS module User
Manual document).

3.1 Data format

For reference:
- Tracking, multiple blobs:
Byte1:
Byte2:
Byte3:
...
Byte(4+((0 to n)*15)+0):
Byte(4+((0 to n)*15)+1):
Byte(4+((0 to n)*15)+2):
Byte(4+((0 to n)*15)+3):
Byte(4+((0 to n)*15)+4):
Byte(4+((0 to n)*15)+5):
Byte(4+((0 to n)*15)+6):
Byte(4+((0 to n)*15)+7):
Byte(4+((0 to n)*15)+8):
Byte(4+((0 to n)*15)+9):
Byte(4+((0 to n)*15)+10):
Byte(4+((0 to n)*15)+11):
Byte(4+((0 to n)*15)+12):
Byte(4+((0 to n)*15)+13):
Byte(4+((0 to n)*15)+14):
...
Byte(4+n*15+15):
Byte(4+n*15+16):

Byte1:
Byte2:
Byte3:
Byte4:
Byte5:
Byte6:
Byte7:
Byte8:
Byte9:

Byte1:
Byte2:
Byte3:
Byte4:
Byte5:

Start byte (0x42)

Start byte (0x00)
Number of blobs detected(n)
Blob ID
COG X MSB
COG X LSB
COG Y MSB
COG Y LSB
COG Y MSB
Weight in percentage
Top X MSB
Top X LSB
Top Y MSB
Top Y LSB
Bottom X MSB
Bottom X LSB
Bottom Y MSB
Bottom Y LSB
End byte (0x00)
End byte (0xFF)

Tracking, single blob:

Start byte (0x42)
Start byte (0x00)
COG X MSB
COG X LSB
COG Y MSB
COG Y LSB
Weight (in percentage)
End byte (0x00)
End byte (0xFF)

Tracking, no blob:
Start byte (0x42)
Start byte (0x00)
0x00
End byte (0x00)
End byte (0xFF)

3.2 Fast Tracking

In this mode, CTS will only track one color and will return minimal information
about detected blob. To read and process this information, see code sample
bellow using PROCESSING:

void draw() {
int pos = 0;
int x = 0;
int y = 0;
int pixel = 0;
int Y = 0;
int Y1 = 0;
myPort.write("TF0");
while (myPort.available () > 0) {
byte[] preBuffer = new byte[3];
myPort.readBytes(preBuffer);
println(">>"+preBuffer[2]);
if ((preBuffer[0] == 66) && (preBuffer[1] == 0x00)
byte[] blobBuffer = new byte[7];
myPort.readBytes(blobBuffer);

&& (preBuffer[2] == 0x01)) {

int weight= unsignedByte(blobBuffer[4] / 5);

int cogx = unsignedByte(blobBuffer[0] << 8) + unsignedByte(blobBuffer[1]);
int cogy = unsignedByte(blobBuffer[2] << 8) + unsignedByte(blobBuffer[3]);
noFill();
strokeWeight(2);
stroke (0, 0, 0);
rect(topX - 2, topY - 1, botX topX + 1, botY topY + 1);
fill (0, 0, 0);
rect (cogx - 3, cogy - 3, 6, 6);
}
myPort.clear();
}
}

Or using C:
if (savailable()){
c = sgetc();
if (c == 0x42){
c = sgetc();
c = sgetc();
if (c == 0x01){
i = 0;
for(i=0;i<7;i++){
buf[i] = sgetc();
}
// retrieve COG X, COG Y and
cogX
= ((buf[0] & 0x07) <<
cogY
= ((buf[2] & 0x07) <<
weight = (buf[4] & 0x7f);
} else {
// reset values if no target
cogX = 0;
cogY = 0;
weight = 0;
}
// Your code

weight if target detected

8 ) | buf[1];
8 ) | buf[3];
detected

}
}

3.3 Tracking multiple blobs

When tracking multiple blobs (TC/Tc or TS/Ts modes) CTS will return a larger
set of data as result. To read and process this information, see code sample
bellow using PROCESSING:
void draw() {

int pos = 0;
int x = 0;
int y = 0;
int pixel = 0;
int Y = 0;
int Y1 = 0;
myPort.write("TCt");
while (myPort.available () > 0) {
byte[] preBuffer = new byte[3];
myPort.readBytes(preBuffer);
if ((preBuffer[0] == 0x42) && (preBuffer[1] == 0x00)) {
byte counter = preBuffer[2];
for (x = 0; x < counter; x++) {
byte[] blobBuffer = new byte[14];
myPort.readBytes(blobBuffer);
int
int
int
int
int
int
int
int

id
=
weight=
cogx =
cogy =
topX =
topY =
botX =
botY =

blobBuffer[0];
unsignedByte(blobBuffer[5]);
unsignedByte(blobBuffer[1] << 8) + unsignedByte(blobBuffer[2]);
unsignedByte(blobBuffer[3] << 8) + unsignedByte(blobBuffer[4]);
unsignedByte(blobBuffer[6] << 8) + unsignedByte(blobBuffer[7]);
unsignedByte(blobBuffer[8] << 8) + unsignedByte(blobBuffer[9]);
unsignedByte(blobBuffer[10] << 8) + unsignedByte(blobBuffer[11]);
unsignedByte(blobBuffer[12] << 8) + unsignedByte(blobBuffer[13]);

noFill();
strokeWeight(2);
stroke (0, 0, 0);
rect((topX-2) * m, (topY-1) * m, (botX - topX+1) * m, (botY - topY+1) * m);
fill (0, 0, 0);
rect ((cogx - 3)* m, (cogy -3)*m, 6*m, 6*m);
}
}
myPort.clear();
}
}

4 Appendix
Code for streaming YUV211 frame and displaying tracking boxes for multiple
blobs:
import processing.serial.*;
String myString = null;
Serial myPort;
PFont f;
void setup() {
println(Serial.list());
myPort = new Serial(this, "/dev/cu.usbserial-A800GKF2", 921600);
size(W * m, H * m+20);
delay(500);
delay(50);
myPort.write("fn");
delay(50);
myPort.write("b2");
delay(50);
myPort.write("c4");
delay(50);
myPort.write("s4");
delay(50);
myPort.write("g5");
delay(50);
myPort.write("a1");
delay(50);
myPort.write("Sd");
delay(50);
}
void draw() {
int pos = 0;
int x = 0;
int y = 0;
float R = 0;
float G = 0;
float B = 0;
int
int
int
int

Y = 0;
Y1 = 0;
U = 0;
V = 0;

myPort.write("TCy");
while (myPort.available () > 0) {
byte[] inBuffer = new byte[H*W+4];
myPort.readBytes(inBuffer);
if ((inBuffer != null) && (inBuffer[1] == 0)) {
for (y=0;y<H;y++) {
for (x=0;x<W/2;x++) {
noStroke();
colorMode(RGB, 255);
Y = inBuffer[y*W+x*2+3]
& 0xF0;
U = (inBuffer[y*W+x*2+3]
& 0x0F) << 4;
Y1= inBuffer[y*W+x*2+3+1] & 0xF0;
V = (inBuffer[y*W+x*2+3+1] & 0x0F) << 4;
B = 1.164*(Y - 16) + 2.018*(U - 128);
G = 1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128);
R = 1.164*(Y - 16) + 1.596*(V - 128);
fill(R, G, B);
rect(2*x, y, 1, 1);
B= 1.164*(Y1 - 16) + 2.018*(U - 128);
G = 1.164*(Y1 - 16) - 0.813*(V - 128) - 0.391*(U - 128);
R = 1.164*(Y1 - 16) + 1.596*(V - 128);
fill(R, G, B);
rect(x*2+1, y, 1, 1);
}

}
}
byte[] preBuffer = new byte[3];
myPort.readBytes(preBuffer);
if ((preBuffer[0] == 0x42) && (preBuffer[1] == 0x00)) {
byte counter = preBuffer[2];
for (x = 0; x < counter; x++) {
byte[] blobBuffer = new byte[14];
myPort.readBytes(blobBuffer);
int
int
int
int
int
int
int
int

id
=
weight=
cogx =
cogy =
topX =
topY =
botX =
botY =

blobBuffer[0];
unsignedByte(blobBuffer[5]);
unsignedByte(blobBuffer[1] << 8) + unsignedByte(blobBuffer[2]);
unsignedByte(blobBuffer[3] << 8) + unsignedByte(blobBuffer[4]);
unsignedByte(blobBuffer[6] << 8) + unsignedByte(blobBuffer[7]);
unsignedByte(blobBuffer[8] << 8) + unsignedByte(blobBuffer[9]);
unsignedByte(blobBuffer[10] << 8) + unsignedByte(blobBuffer[11]);
unsignedByte(blobBuffer[12] << 8) + unsignedByte(blobBuffer[13]);

noFill();
strokeWeight(2);
stroke (0, 0, 0);
rect((topX-2) * m, (topY-1) * m, (botX - topX+1) * m, (botY - topY+1) * m);
fill (0, 0, 0);
rect ((cogx - 3)* m, (cogy -3)*m, 6*m, 6*m);
}
}
myPort.clear();
}
}
int unsignedByte( int val ) {
return ( val < 0 ? 256+val : val );
}