Friday, October 11, 2013

MongoDB Real-Time Analytics Part-2


Here is another schema sample for storing Daily,Hourly and Minutes data 

{
    "_id" : "daily", ---> should be unique date for the day
    "article" : {
        "HT97" : 100,
        "HT98" : 100,
        "HT99" : 100
    },
    "daily" : 300,
    "hourly" : {
        "15" : 200,
        "16" : 100
    },
    "minutely" : {
        "957" : 100,
        "958" : 100,
        "960" : 100
    }
}



package com.realtime;

 import java.text.SimpleDateFormat;
import java.util.Date;

import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBObject;
import com.mongodb.Mongo;

public class test {

public static void main(String[] args) {
   
    DBCollection collection = getCollection();
    for(int i=0;i<100;i++){
    Date now = new Date();
    String format  = new SimpleDateFormat("yyyy-MM-dd-HH-mm").format(now);
    Request1 r = new Request1(format,"HT"+i);
    upsert(r.getArticleId(),r,collection);
    System.out.println("upsert request: HT"+i);
    }
         
}
public static void upsert(String id, Request1 request,
            DBCollection collection) {
        BasicDBObject query = new BasicDBObject();
         query.put("_id", "monthly");
        DBObject dbo = createUpsert(request);
        collection.update(query, dbo, true, false);

    }

/**
* increment the stats
* @param request
* @return
*/
    public static DBObject createUpsert(Request1 request) {
        DBObject upsert = new BasicDBObject();
        DBObject inc = new BasicDBObject();
         // Count total hits
        inc.put("daily", 1);
        inc.put("minutely."+getMinutes(request),1); 
        inc.put("hourly."+getHour(request), 1);
        inc.put("article."+request.getArticleId(),1);

        upsert.put("$inc", inc);
        return upsert;
    }
    /**
     * get the current minute in day
     * @param r
     * @return
     */
    public static int getMinutes(Request1 r ){
    String format = r.getTime();
    String sMinutes = format.substring(format.lastIndexOf("-")+1);
    int minutes= Integer.parseInt(sMinutes);
    
    String sHour = getHour(r);
    int hour = Integer.parseInt(sHour);
    
    minutes = hour * 60 + minutes;
     return minutes;
    }
    /**
     * get current hour in a day
     * @param r
     * @return
     */
public static String getHour(Request1 r){
String format = r.getTime();
format = format.substring(format.lastIndexOf("-")-2,format.length()-3);
 
return format;
}
    public static DBCollection getCollection() {
     
        Mongo mongo;
        try {
            mongo = new Mongo();
        } catch (Exception e) {
            throw new RuntimeException(e);
        }

        DB db = mongo.getDB("analytics");
        return db.getCollection("metrics");
    }
}

Create ElasticSearch cluster on single machine

I wanted to figure out how to create a multi-node ElasticSearch cluster on single machine. So i followed these instructions First i did...