OpenCV reliably detect finger tips in webcam video

edit

I am developing an application that tracks fingers on a desktop/laptop keyboard for touch typing exercises. For demo purposes, I am detecting skin by thresholding the HSV hue value, which is working fairly well, and gives me a good image to work with:

From here, I am grabbing a contour that is fairly accurate (although sometimes it is segmented, not sure how to deal with that yet), I get a convex hull of this threshold which is as accurate as the contour allows, and then detect convexity defects to try and find fingers and edges.

The problem I have is that I need to be able to identify at least the pinky, ring, middle, and index fingers (thumb doesn't matter so much), but my results often times only find one or two fingers on each hand.

Here is my relevant code:

void FingerMap::findFingers( const cv::Mat& mask, Rect leftRegion, Rect rightRegion )
{
    cv::Mat grey_eql, grey_eql_smooth;

    cv::blur( mask, grey_eql_smooth, cv::Size(3, 3) );

    cv::Mat left, right;

    left = grey_eql_smooth.clone()( (const cv::Rect&)leftRegion );
    right = grey_eql_smooth.clone()( (const cv::Rect&)rightRegion );

    findFingersHand( "left-hand", left, fingers[0], leftRegion.topleft() );
    findFingersHand( "right-hand", right, fingers[1], rightRegion.topleft() );

    cv::imshow( "foreground-smoothed", grey_eql_smooth );
    cv::imshow( "fingerContour", copy );
}

void FingerMap::findFingersHand( const std::string& label, cv::Mat& maskPortion, std::vector<cv::Point2f>& fingerPositions, cv::Point offset )
{
    cv::Mat thresh;
    std::vector< std::vector< cv::Point > > contours;
    std::vector< cv::Vec4i > hierarchy;

    cv::threshold( maskPortion, thresh, 150.0, 255, cv::THRESH_BINARY /*| CV_THRESH_OTSU*/ );

    cv::findContours(
        thresh,
        contours,
        hierarchy,
        /*CV_RETR_CCOMP*/ CV_RETR_EXTERNAL,
        CV_CHAIN_APPROX_SIMPLE,
        offset
    );

    int count=0;
    std::vector<cv::Point2f> possibleFingers;
    double largestArea[2] = { 500.0, 500.0 };
    int largestAreaIndex[2] = { -1, -1 };
    for(unsigned int c=0; c < contours.size(); c++)
    {
        if ( contours[c].size() < 3 ) continue;
        double area = cv::contourArea( contours[c] );
        if ( area > largestArea[0] )
        {
            largestArea[1] = largestArea[0];
            largestAreaIndex[1] = largestAreaIndex[0];
            largestArea[0] = area;
            largestAreaIndex[0] = (int)c;
        }
        else if ( area > largestArea[1] )
        {
            largestArea[1] = area;
            largestAreaIndex[1] = (int)c;
        }
    }

    for(unsigned int id=0; id < 2; id++)
    {
        if ( largestAreaIndex[id] == -1 )
        {
            continue;
        }
        std::vector<int> hull;
        std::vector<cv::Vec4i> defects;
        cv::convexHull( cv::Mat( contours[ largestAreaIndex[id] ] ), hull, false );

        cv::convexityDefects( contours[ largestAreaIndex[id] ], hull, defects );
        for(unsigned int i=1; i < (defects.size()-1); i++)
        {
            if ( ( defects[i][3] / 256.0f ) < 10.0f ) continue; // ?
            possibleFingers.push_back( contours[ largestAreaIndex[id] ][ defects[i][0] ] );
            possibleFingers.push_back( contours[ largestAreaIndex[id] ][ defects[i][1] ] );
        }
    }

    // Find grouped possible fingers
    int maskHeight = maskPortion.size().height - 20;
    if ( possibleFingers.size() > 0 )
    {
        fingerPositions.clear();
        unsigned int fingerStart = 0;
        for(unsigned int pf=1; pf < possibleFingers.size(); pf++)
        {
            if ( ( possibleFingers[pf-1].y >= maskHeight ) || ( possibleFingers[pf].y >= maskHeight ) ) continue;
            float d = distance( possibleFingers[pf-1], possibleFingers[pf] );
            if ( d > 15.0f )
            {
                cv::Point fingerPoint = possibleFingers[fingerStart];
                float numPoints = 1.0f;
                for(unsigned int j=fingerStart+1; j < pf; j++)
                {
                    add( fingerPoint, possibleFingers[j] );
                    numPoints++;
                }
                divide( fingerPoint, numPoints );
                fingerPoint.y += 15;

                fingerPositions.push_back( fingerPoint );

                fingerStart = pf;
            }
        }
    }
}

Any ideas for improvements?