Reading and processing video frames with Java

The class VideoSource loads a video file and offers access to the individual frames.

To use this class you need to have JMF installed on your machine. The video format must be supported by your Java/JMF platform. Best is to use no codec at all; "raw" video.

The usage is as follows:

VideoSource vs = new VideoSource("file://c:\test.avi");
vs.initialize();
...
int frameIndex = 12345; // any frame
BufferedImage frame = vs.getFrame(frameIndex);

package popscan;

import java.awt.Graphics2D;

import java.awt.Image;

import java.awt.image.BufferedImage;

import java.net.URL;

import javax.media.Buffer;

import javax.media.ControllerEvent;

import javax.media.ControllerListener;

import javax.media.Manager;

import javax.media.Player;

import javax.media.PrefetchCompleteEvent;

import javax.media.RealizeCompleteEvent;

import javax.media.control.FrameGrabbingControl;

import javax.media.control.FramePositioningControl;

import javax.media.format.VideoFormat;

import javax.media.util.BufferToImage;

public class VideoSource implements ControllerListener {

public static final int NOT_READY = 1;

public static final int READY = 2;

public static final int ERROR = 3;

Player _player;

String _videoFilename;

FramePositioningControl _framePositioningControl;

FrameGrabbingControl _frameGrabbingControl;

private int _state;

// the filename must contain protocol,

//for example file://c:\\test.avi

public VideoSource(String videoFilename) {

_videoFilename = videoFilename;

_state = NOT_READY;

}

/*

* Create Player object and start realizing it

*/

public void initialize() {

try {

_player = Manager.createPlayer(new URL(_videoFilename));

_player.addControllerListener(this);

// realize call will launch a chain of events,

// see controllerUpdate()

_player.realize();

} catch (Exception e) {

System.out.println("Could not create VideoSource!");

e.printStackTrace();

setState(ERROR);

return;

}

}

/*

* Returns the current state

*/

public int getState() {

return _state;

}

/*

* Returns the number of frames for current video if

* the VideoSource is ready, in any other case returns -1.

*/

public int getFrameCount() {

if (getState()!=READY) {

return  -1;

}

return  _framePositioningControl.

mapTimeToFrame(_player.getDuration());

}

/*

* Returns the video frame from given index as BufferedImage.

* If VideoSource is not ready or index is out of bounds,

*  returns null.

*/

public BufferedImage getFrame(int index) {

if (getState()!=READY||index<0||index>getFrameCount()) {

return null;

}

_framePositioningControl.seek(index);

Buffer buffer = _frameGrabbingControl.grabFrame();

Image img = new BufferToImage((VideoFormat)buffer.

getFormat()).createImage(buffer);

// image creation may also fail!

if (img!=null) {

BufferedImage bi = new BufferedImage(img.getWidth(null),

img.getHeight(null), BufferedImage.TYPE_INT_ARGB);

Graphics2D g = bi.createGraphics();

g.drawImage(img, 0,0, null);

return bi;

}

return null;

}

// callback for ControllerListener

public void controllerUpdate(ControllerEvent event) {

if (event instanceof RealizeCompleteEvent) {

_player.prefetch();

} else if (event instanceof PrefetchCompleteEvent) {

// get controls

_framePositioningControl = (FramePositioningControl)_player.

getControl("javax.media.control.FramePositioningControl");

if (_framePositioningControl==null) {

System.out.println("Error: FramePositioningControl!");

setState(ERROR);

return;

}

_frameGrabbingControl = (FrameGrabbingControl)_player.

getControl("javax.media.control.FrameGrabbingControl");

if (_frameGrabbingControl==null) {

System.out.println("Error: FrameGrabbingControl!");

setState(ERROR);

return;

}

setState(READY);

}

}

// for setting the state internally

private void setState(int nextState) {

_state = nextState;

}

}

And here is a tester class to demonstrate how the VideoSource works.

It creates an AWT Frame and draws the video frames one by one as fast as possible to the Frame's Graphics surface.

First it draws the frames from the first frame to the last frame and when the loop ends, the frames are drawn from the last frame to first frame. This is to show that you really can access the individual frames.

In this case the frames to be drawn are next to each other, so if you are accessing the frames randomly, you may encounter some performance issues.

package popscan;

import java.awt.Frame;

import java.awt.image.BufferedImage;

public class TestFrame {

public static void main(String[] args) {

VideoSource videoSource = new VideoSource(args[0]);

videoSource.initialize();

while (videoSource.getState()==VideoSource.NOT_READY) {

try { Thread.sleep(100); } catch (Exception e) { }

}

if (videoSource.getState()==VideoSource.ERROR) {

System.out.println("Error while initing"+args[0]);

return;

}

Frame frame = new Frame();

frame.setVisible(true);

int frameCount = videoSource.getFrameCount();

// forward

for (int frameIndex=0;frameIndex<frameCount;frameIndex++) {

BufferedImage i = videoSource.getFrame(frameIndex);

// do what ever you need to do your frame

drawFrame(frame,i,frameIndex);

}

// backwards

for (int frameIndex=frameCount;frameIndex>=0;frameIndex--) {

BufferedImage i = videoSource.getFrame(frameIndex);

// do what ever you need to do your frame

drawFrame(frame,i,frameIndex);

}

System.exit(0);

}

public static void drawFrame(Frame frame, BufferedImage image,

int index) {

if (image!=null) {

frame.setSize(image.getWidth(), image.getWidth());

frame.getGraphics().drawImage(image, 0, 0, null);

System.out.println("Image at index: "+index);

} else {

System.out.println("null image");

}

}

}

Capturing webcam image with Java Media Framework

I have used Java Media Framework (JMF) quite a lot in the past, and recently I strarted to experiment again with some of my old algorithms. Having changed my computers during these years, I noticed that the old code did not work correctly. Especially, setting the webcam video capture format did not respond. This is the problem I noticed many people were still having, based on the question on the stackoverflow etc.

After some hours of debugging, I found out that the format needs to be set before the javax.media.Player is started. After the player has been started, the FormatControl.setFormat() has no effect.

Here is also a full source code minimal webcam image grabber. Please, let me know if you have problems with it. Please note that the initialization of the video capture device may take some seconds. If you are grabbing images as fast as possible, the first image captures will fail (the images returned have dimension 1x1) until the video capture device is ready.

Use class like this:

    FrameGrabber grabber = new FrameGrabber();
    if (grabber.init()==true) {
        BufferedImage frame = grabber.grab();
    }

And the class source code:

import java.util.Vector;
import java.awt.Graphics2D;
import java.awt.Image;
import java.awt.image.BufferedImage;
import javax.media.*;
import javax.media.control.FormatControl;
import javax.media.control.FrameGrabbingControl;
import javax.media.format.VideoFormat;
import javax.media.format.YUVFormat;
import javax.media.util.BufferToImage;

//
// source code by tejopa / popscan.blogspot.com, 2012
//

public class FrameGrabber {
 
    CaptureDeviceInfo    _cdi;
    Player               _player;
    FrameGrabbingControl _frameGrabber;
    FormatControl        _formatControl;

    public FrameGrabber() { }
  
    public boolean init() {
     Vector devices = CaptureDeviceManager.getDeviceList(new YUVFormat());
     if (devices.size()<1) {
      System.out.println("No capture devices available!");
     } else {
      _cdi = (CaptureDeviceInfo)devices.elementAt(0);
     }
     try {
            _player = Manager.createRealizedPlayer(_cdi.getLocator());
         _frameGrabber = (FrameGrabbingControl)_player.getControl("javax.media.control.FrameGrabbingControl");
         _formatControl = (FormatControl)_player.getControl("javax.media.control.FormatControl");
         _formatControl.setFormat(_cdi.getFormats()[0]);
         _player.start();
         return true;
     } catch (Exception e) {
      e.printStackTrace();
  }
     return false;
    }
    
    public BufferedImage grab() {
        if (_frameGrabber==null) {
            System.out.println("FrameGrabber is not initialized!");
            return null;
        }
     // Grab a frame from the capture device
        Buffer buf = _frameGrabber.grabFrame();
        // check that buf actually exists!
        if (buf!=null) {
            Image img = new BufferToImage((VideoFormat)buf.getFormat()).createImage(buf);
            // image creation may also fail!
            if (img!=null) {
         BufferedImage bi = new BufferedImage(img.getWidth(null), img.getHeight(null), BufferedImage.TYPE_INT_ARGB);
  Graphics2D g = bi.createGraphics();
  g.drawImage(img, 0,0, null);
  return bi;
            }
        } 
        // grab failed, still return image, but make it empty and small
        BufferedImage bi = new BufferedImage(1,1, BufferedImage.TYPE_INT_ARGB);
        return bi;
    }
}