a digital scan of a 35mm film image of a processing sketch running on an LCD
Skip to Content

Street Scraping

Ever wanted to download Google Street View panoramas programmatically? If you said "yes!", then this is the code for you.

I reverse-engineered their URL query scheme using the Live HTTP Headers plug-in for Firefox. From there, I just parsed through the XML returned to download each jpeg tile and then stitch them together in order.

I used a variation of this code in combination with a few other programs to parse a GPS log, download the street view panorama for each position in the log, and then stitch each frame together into a video documenting my trip on the M5 bus.

  1. import processing.core.*;
  2. import processing.net.*;
  3. import processing.xml.*;
  4.  
  5. // get a hold of this for the xml library
  6. // better way to do this part?
  7. PApplet main = this;
  8.  
  9. void setup() {
  10.    // we usually end up with 3328 x 1664 pixel images
  11.    // let's undersample considerably for a more sane window size
  12.   size(832, 416);
  13.  
  14.   // your google maps api key goes here
  15.   // sign up: http://code.google.com/apis/maps/signup.html
  16.   String apiKey = "REPLACE ME WITH YOUR VERY OWN API KEY";
  17.  
  18.   // pick a location
  19.   // takes an address or a lat / lon
  20.   String location = "721 broadway, new york, ny";
  21.  
  22.   // generate the panorama
  23.   Panorama pano = new Panorama(location, apiKey);
  24.  
  25.   // show the image
  26.   image(pano.fullPano, 0, 0, width, height);
  27.  
  28.   // save the full-res image to the sketch folder
  29.   pano.fullPano.save(location + ".jpg");
  30.  
  31.   // skip the draw loop
  32.   noLoop();
  33. }
  34.  
  35.  
  36. class Panorama {
  37.   float lat;
  38.   float lon;
  39.   String panoId;  
  40.   int imageWidth;
  41.   int imageHeight;
  42.   int tileWidth;
  43.   int tileHeight;
  44.   int xTileCount;
  45.   int yTileCount;  
  46.   PImage[][] tiles;
  47.   PImage fullPano;
  48.   String apiKey;
  49.  
  50.   // constructor
  51.   Panorama(String address, String _apiKey) {
  52.     apiKey = _apiKey;
  53.     String[] location = geocode(address);
  54.     lon = float(location[0]);
  55.     lat = float(location[1]);
  56.  
  57.     fetchInfo(lat, lon);
  58.     buildImage();
  59.   }
  60.  
  61.  
  62.   void fetchInfo(float lat, float lon) {
  63.     String url = "http://maps.google.com/cbk?output=xml&ll=" + lat + "," + lon;
  64.  
  65.     XMLElement response = new XMLElement(main, url);
  66.  
  67.     XMLElement kid = response.getChild(0);
  68.     imageWidth = kid.getIntAttribute("image_width");
  69.     imageHeight = kid.getIntAttribute("image_height");
  70.     tileWidth = kid.getIntAttribute("tile_width");    
  71.     tileHeight = kid.getIntAttribute("tile_height");
  72.     panoId = kid.getStringAttribute("pano_id");
  73.  
  74.     xTileCount = imageWidth / tileWidth;
  75.     yTileCount = imageHeight / tileHeight;
  76.  
  77.     println("Panorama ID: " + panoId);
  78.     println("X Tile Count: " + xTileCount);
  79.     println("Y Tile Count: " + yTileCount);
  80.   }
  81.  
  82.   void buildImage() {
  83.     fullPano = createImage(imageWidth, imageHeight, RGB);
  84.     fullPano.loadPixels();
  85.  
  86.     for(int xPos = 0; xPos <= xTileCount; xPos++) {
  87.       for(int yPos = 0; yPos <= yTileCount; yPos++) {
  88.         // &.jpg fools processing into handling it
  89.         String imageUrl = "http://cbk0.google.com/cbk?output=tile&panoid=" + panoId + "&zoom=3&x=" + xPos + "&y=" + yPos + "&.jpg";
  90.         fullPano.set(xPos * tileWidth, yPos * tileHeight, loadImage(imageUrl));
  91.         println("Loaded tile " + xPos + "::" + yPos);      
  92.       }
  93.     }
  94.  
  95.     fullPano.updatePixels();        
  96.   }
  97.  
  98.   String[] geocode(String address) {
  99.     String cleanAddress =  address.replace(' ', '+');
  100.     String url = "http://maps.google.com/maps/geo?q=" + cleanAddress + "&output=xml&oe=utf8&sensor=false&key=" + apiKey;
  101.     println(url);
  102.     XMLElement location = new XMLElement(main, url);
  103.     XMLElement kid = location.getChild("Response/Placemark/Point/coordinates");
  104.     String rawCoordinates = kid.getContent();
  105.     String[] latlon = shorten(split(rawCoordinates, ',')); // ditch the 0
  106.  
  107.     println(latlon);
  108.  
  109.     return latlon;
  110.   }
  111. }

October 7 2009 at 4 AM

Add Your Comment