ITHCWY: Robert Ellison's Blog

Get an email when your security camera sees something new (Apps Script + Cloud Vision)

Get an email when your security camera sees something new (Apps Script + Cloud Vision)

Nest (previously DropCam) can email you when it detects activity but that gets boring quickly. How about an email only when it sees something totally new?

The script below downloads a frame from a web cam and then calls the Google Cloud Vision API to label features. It keeps a record of everything that has previously been seen and only sends an email when a new feature is detected. You could easily tweak this to email on a specific feature (i.e. every time your dog is spotted), or to count the number of times a feature appears. I'm using a Nest cam but any security camera that has a publicly visible image download URL will work.

var OAuthCreds = {
  "type""service_account",
  //...
};

var SendEmailTo = '';
var MonitorImageUrl = '';



function main() {
  var timestamp = Date.now().toString();
  var scriptProperties = PropertiesService.getScriptProperties();
  var currentProps = scriptProperties.getProperties();
  
  Logger.log('Grabbing a frame');
  var url = MonitorImageUrl + '&cb=' + timestamp;
  var response = UrlFetchApp.fetch(url);
  var image = response.getBlob();
  image.setName('image.jpg');
  var bytes = image.getBytes();
  var encodedImage = Utilities.base64EncodeWebSafe(bytes);
  
  Logger.log('Calling cloud vision');
  var service = getService();
  if (service.hasAccess()) {
    
    var request = {
      "requests":[
        {
          "image":{
            "content": encodedImage
          },
          "features":[
            {
              "type""LABEL_DETECTION",
              "maxResults":50
            }
          ]
        }
      ]
    }
    
    var annotateUrl = 'https://vision.googleapis.com/v1/images:annotate';
    var annotateResponse = UrlFetchApp.fetch(annotateUrl, {
      "headers": {
        Authorization: 'Bearer ' + service.getAccessToken()
      },
      "method" : "post",
      "contentType" : "application/json",
      "payload" : JSON.stringify(request, null, 2)
    });
    var json = JSON.parse(annotateResponse.getContentText());
    
    var anythingNew = false;
    var newText = '';
    
    for (var l = 0; l < json.responses[0].labelAnnotations.length; l++) {
      var description = json.responses[0].labelAnnotations[l].description;
      var score = json.responses[0].labelAnnotations[l].score;
      
      if (!(description in currentProps)) {
        Logger.log('Found new feature: ' + description);
        scriptProperties.setProperty(description, score);
        anythingNew = true;
        newText += 'Found: ' + description + ' (score: ' + score + ')\r\n';
      }
    }
    
    if (anythingNew) {
      MailApp.sendEmail(SendEmailTo, 'Found something new on the webcam ' + new Date(), newText, { 
        attachments: [image] 
      });
    }
    
  } else {
    Logger.log(service.getLastError());
  }
}

// modified from https://github.com/googlesamples/apps-script-oauth2/blob/master/samples/GoogleServiceAccount.gs#L50 below...
function getService() {
  return OAuth2.createService('CloudVision')
      // Set the endpoint URL.
      .setTokenUrl(OAuthCreds.token_uri)

      // Set the private key and issuer.
      .setPrivateKey(OAuthCreds.private_key)
      .setIssuer(OAuthCreds.client_email)

      // Set the name of the user to impersonate. This will only work for
      // Google Apps for Work/EDU accounts whose admin has setup domain-wide
      // delegation:
      // https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority
     // .setSubject(USER_EMAIL)

      // Set the property store where authorized tokens should be persisted.
      .setPropertyStore(PropertiesService.getScriptProperties())

      // Set the scope. This must match one of the scopes configured during the
      // setup of domain-wide delegation.
      .setScope('https://www.googleapis.com/auth/cloud-platform');
}

function reset() {
  var service = getService();
  service.reset();
}

There is a bit of setup to get this working. Create a new Apps Script project in Google Drive and paste the code above in. You'll need to provide you own values for the three variables at the top.

OAuthCreds is the contents of the JSON format private key file for a Google Developer Console project. Go to the console, create a new project and enable the Cloud Vision API. You'll also need to enable billing (more on this below) - a trial account will work fine for this. Once the API is enabled create a service account under Credentials and download the JSON file. Just paste the contents of this into the script.

That's the hard part over. Now enter the URL of the image to monitor (see this post for instructions on finding this for a Nest / DropCam device) as MonitorImageUrl and your email address for SendEmailTo.

One last thing - follow the instructions here to reference the OAuth2 for Apps Script library.

Once this is all done run the script (the main() function) and authorize it. You should get an email with a picture attached and a list of the labels detected together with a confidence score from 0 to 1. If this doesn't happen check the logs (under the View menu).

You can now schedule the script to run repeatedly (Resources -> Current project's triggers). You get up to 1,000 units a month for free so once an hour should be safe. If you need more frequent updates check the Cloud Vision pricing guide for details.

After a few runs you should only get an email when something new is detected. If you're seeing too many wild guesses then add a filter on the score to exclude low confidence features.

Enjoy, and leave a comment if you have problems (or modify this in interesting ways).

(Previously)

Comments

Terese

Pretty! This has been an incredibly wonderful article. Many thanks for providing this information.

Paul Farrell

This is a great article but it is a bit out of date. I think it wont work anymore since DropCam was moved over to Google/NEST.

Maybe it can be updated?

Robert Ellison

Paul - it still works for me. See the link in the post above for instructions on getting a public image URL for a Nest cam. This hasn't changed with the transition from DropCam to Nest. If that doesn't help let me know where you're getting stuck.

Add Comment

All comments are moderated to weed out spam. Email address is optional and is only used to display your Gravatar.