jmquintana79
10/3/2012 - 12:43 PM

map-reduce in with MongoDB with PyMongo for Parse timing calls

Map-reduce in with MongoDB with PyMongo for Parse timing calls

'''
Sample object:

{
    "url": "/1/functions/Klass3", 
    "_id": {
        "$oid": "506c082701d5410ec7000002"
    }, 
    "log": [
        {
            "duration": 30.353700876235962, 
            "params": "{\"sender\": \"abcde12345\", \"receivers\": [{\"amount\": \"3.34\", \"id\": \"abcde12345\"}, {\"amount\": \"3.33\", \"id\": \"9485hfguhj\"}, {\"amount\": \"3.33\", \"id\": \"93kdujr87\"}]}", 
            "sent": {
                "$date": 1349286055012
            }
        }, 
        {
            "duration": 0.5051431655883789, 
            "params": "{\"sender\": \"abcde12345\", \"receivers\": [{\"amount\": \"3.34\", \"id\": \"abcde12345\"}, {\"amount\": \"3.33\", \"id\": \"9485hfguhj\"}, {\"amount\": \"3.33\", \"id\": \"93kdujr87\"}]}", 
            "sent": {
                "$date": 1349286212207
            }
        }, 
        {
            "duration": 0.5687680244445801, 
            "params": "{\"sender\": \"abcde12345\", \"receivers\": [{\"amount\": \"3.34\", \"id\": \"abcde12345\"}, {\"amount\": \"3.33\", \"id\": \"9485hfguhj\"}, {\"amount\": \"3.33\", \"id\": \"93kdujr87\"}]}", 
            "sent": {
                "$date": 1349286393610
            }
        }
    ]
}



Sample results:
{
    "counts": {
        "input": 4, 
        "reduce": 4, 
        "emit": 21, 
        "output": 4
    }, 
    "timeMillis": 19, 
    "ok": 1.0, 
    "results": [
        {
            "_id": "/1/classes/Klass1", 
            "value": {
                "count": 8.0, 
                "total": 5.25708794593811, 
                "avg": 0.6571359932422638
            }
        }, 
        {
            "_id": "/1/classes/Klass2", 
            "value": {
                "count": 6.0, 
                "total": 1.4316394329071045, 
                "avg": 0.23860657215118408
            }
        }, 
        {
            "_id": "/1/functions/Klass3", 
            "value": {
                "count": 3.0, 
                "total": 31.42761206626892, 
                "avg": 10.475870688756308
            }
        }, 
        {
            "_id": "/1/login/", 
            "value": {
                "count": 4.0, 
                "total": 1.687263011932373, 
                "avg": 0.42181575298309326
            }
        }
    ]
}

'''


from pymongo import Connection
from bson.code import Code
from bson.son import SON

conn = Connection(); 
db = conn.******; 
db.authenticate("**", "***"); 
coll = db.******


mapf = Code('''
	function() { 
		var self = this; 
		self.log.forEach(function(l){ 
			emit(self.url, {
				total: l.duration, 
				count: 0, 
				avg: 0
			}); 
		}); 
	} ''')

reducef = Code('''
	function(key, values) { 
		var r = {
			count: 0, 
			total: 0, 
			avg: 0
		}; 

		values.forEach(function(item) { 
			r.count += 1;
			r.total += item.total; 
		});

		return r; 
	} ''')

finalisef = Code('''
	function(key, value) {
		if(value.count > 0) { 
			value.avg = value.total / value.count; 
		} 

		return value; 
	}''')

coll.map_reduce(map=mapf, reduce=reducef,  out=SON([('inline',1)]), finalize=finalisef)