python datastore + webhdf

5 files changed, 58 insertions, 31 deletions

diff --git a/ebus/datastore.py b/ebus/datastore.py
index 88dcdc9..930b4fb 100644
--- a/ebus/datastore.py
+++ b/ebus/datastore.py
@@ -40,8 +40,12 @@ class Datastore(object):
                         if not name in self.files:
                                 path = os.path.join(self.basepath, name+".hdf")
                                 print "open {0}".format(path)
-                                file = tables.openFile(path, "a", title = "eBus Datastore")
-                                self.files[name] = file
+				if os.path.exists(path):
+					self.files[name] = tables.openFile(path, "a", title = "eBus Datastore")
+				elif klass:
+					self.files[name] = tables.openFile(path, "w", title = "eBus Datastore")
+				else:
+					raise Exception("No such sensor {0}".format(name))
 
                         if not name in self.tables:
                                 t = None
@@ -49,6 +53,7 @@ class Datastore(object):
                                         t = self.files[name].getNode("/"+name)
                                 except tables.NoSuchNodeError,e:
                                         if not klass: raise e
+
                                         t = self.files[name].createTable("/",
                                                         name,
                                                         klass,
@@ -56,17 +61,27 @@ class Datastore(object):
                                                         filters=tables.Filters(complevel=1),
                                                         createparents=True)
                                         t.cols.timestamp.createCSIndex()
+					self.files[name].close()
+
+					self.files[name] = tables.openFile(path, "a", title = "eBus Datastore")
+					t = self.files[name].getNode("/"+name)
+
                                 self.tables[name] = t
 
                         return self.tables[name]
 
         def addValue(self, name, ts, value, klass,flush=False):
                 t = self.getTable(name, klass)
+		if klass != None:
+		    assert klass.columns['value'].type == t.cols.value.type, "Type check failed"
+
                 with self.fileLock:
                         t.row['timestamp'] = ts
                         t.row['value'] = value
                         t.row.append()
-                        if flush: t.flush()
+                        if flush: 
+				t.flush()
+				t.flushRowsToIndex()
 
         def addValueInt(self, name, ts, value): self.addValue(name, ts, value, ValueInt)
         def addValueString(self, name, ts, value): self.addValue(name, ts, value, ValueString)
diff --git a/ebus/webhdf/__init__.py b/ebus/webhdf/__init__.py
index af13f87..c1f34dd 100644
--- a/ebus/webhdf/__init__.py
+++ b/ebus/webhdf/__init__.py
@@ -13,7 +13,7 @@ import bottle
 
 import ebus.datastore
 
-datastore = ebus.datastore.Datastore("testhdffiles")
+datastore = ebus.datastore.Datastore("hdf-data")
 
 app = bottle.Bottle("ebus")
 
@@ -50,6 +50,8 @@ def sensor_data_put(name,timestamp=None):
             klass = ebus.datastore.ValueFloat
         elif type == "string":
             klass = ebus.datastore.ValueString
+	elif type == "":
+	    klass = None
         else:
             return {'error':'INVALID_TYPE', msg:'Type {0} is invalid'.format(type)}
 
@@ -58,38 +60,42 @@ def sensor_data_put(name,timestamp=None):
 	logging.info(msg)
         return {'error':None,'msg':msg}
     except Exception,e:
-        return {'error':e,'msg':e}
+	logging.error("Error: " + "{0} name={1} value={2} type={3}".format(e, name, value, type))
+        return {'error':str(e),'msg':str(e)}
 
 
 
 @app.route('/sensor/:name/:startdate/:enddate')
 def sensor_name_start_end(name,startdate,enddate):
+    SAMPLING_STEPSIZE=500
     try:
         startdate, enddate = int(startdate), int(enddate)
 	logging.info("/sensor/ start={0} end={1}".format(startdate, enddate))
         table=datastore.getTable(name)
         with datastore:
-	    i = table.where("(timestamp >= startdate) & (timestamp <= enddate)",step=100)
+	    i = table.where("(timestamp >= startdate) & (timestamp <= enddate)",
+			    condvars={'startdate':startdate,'enddate':enddate,'timestamp':table.cols.timestamp},
+			    step=SAMPLING_STEPSIZE)
 	    timestamps = []
 	    try:
 	        for x in range(20):
 		    i.next()
 		    timestamps.append(i['timestamp'])
-	    except:
-		pass
+	    except: pass
+
 	    if len(timestamps) > 10:
-		    diff = map(lambda (x1,x2): (x2-x1)/100, zip(timestamps[:-1], timestamps[1:]))
+		    diff = map(lambda (x1,x2): (x2-x1)/SAMPLING_STEPSIZE, zip(timestamps[:-1], timestamps[1:]))
 		    diff_avg = numpy.average(diff)
 		    time_period = enddate - startdate
 		    samples = time_period / diff_avg
 		    step = numpy.ceil(samples / 400.0)
 
-		    data = [(x['timestamp']*1000, x['value']) 
+		    data = [(x['timestamp'], x['value']) 
 			    for x in table.where("(timestamp >= startdate) & (timestamp <= enddate)", step=step)]
 		    logging.info("diff={0} samples={1} step={2} len={3} ({4})".format(diff_avg, samples, step, len(data),name))
 	    else:
-	            logging.info("No data found ({0})".format(name))
-		    data = []
+		    # Not enough data, sampling approach useless, deliver all data
+		    data = [(x['timestamp'], x['value']) for x in table.where("(timestamp >= startdate) & (timestamp <= enddate)")]
             return {'sensor':name, 'error':None,'data':data}
     except Exception,e:
 	logging.error("Error: " + str(e) + str(type(e)))
@@ -107,7 +113,7 @@ def sensor_avg_start(name, startdate, period=60*15): #15min
             f_group = range(startdate, enddate, period)
             data = map(lambda (group_id, grouped_rows): (group_id, numpy.average([row['value'] for row in grouped_rows])),
                     itertools.groupby(sel_rows, lambda t: (t['timestamp']/period)*period))
-            data = map(lambda (timestamp,value): (timestamp*1000, value), data)
+            data = map(lambda (timestamp,value): (timestamp, value), data)
             return {'sensor':name, 'error':None,'data':data}
     except Exception,e:
             return {'sensor':name, 'error':str(e), 'data':None}
diff --git a/ebus/webhdf/static/css/stylesheet.css b/ebus/webhdf/static/css/stylesheet.css
index edc0c9b..cfaa2fe 100644
--- a/ebus/webhdf/static/css/stylesheet.css
+++ b/ebus/webhdf/static/css/stylesheet.css
@@ -1,22 +1,25 @@
 body {
+	margin: 0;
 	font-family:sans;
 	text-align:center;
+	width:100%;
 }
 
 #ebusgraph {
-    	margin:auto;
-        width:100%;
+	margin: auto;
+        width: 90%;
         height: 70%;
 }
 
 #overview {
-	width: 100%;
-    	margin:auto;
-	height:100px;
+    	margin: 2% auto 0% auto;
+	width: 90%;
+	height: 100px;
 }
 
 #options {
-	width:800px;
+	height: 10%;
+	width: 800px;
 	margin:auto;
 	text-align:left;
 }
diff --git a/ebus/webhdf/static/index.html b/ebus/webhdf/static/index.html
index 04ecc15..53f103e 100644
--- a/ebus/webhdf/static/index.html
+++ b/ebus/webhdf/static/index.html
@@ -8,9 +8,9 @@
   </head>
   <body>
     <div id="ebusgraph"></div>
+    <div id="overview"></div>
     <div id="options">
 	<div id="sensorpicker"></div>
     </div>
-    <div id="overview"></div>
   </body>
 </html>
diff --git a/ebus/webhdf/static/src/ebus.js b/ebus/webhdf/static/src/ebus.js
index ce37557..5c4fa16 100644
--- a/ebus/webhdf/static/src/ebus.js
+++ b/ebus/webhdf/static/src/ebus.js
@@ -11,7 +11,7 @@ d.now = new Date().getTime();
 function int(n) { return Math.round(n); };
 
 $(document).ready(function(){
-    var from = d.now - 5*d.day;
+    var from = d.now - 1*d.day;
     var fromOverview = d.now - 30*d.day;
     var to = d.now;
     var datasetDetail = []
@@ -23,7 +23,7 @@ $(document).ready(function(){
                       {"sensorname":"heizkreisregler10.betriebsdatenRegler1.kesselTemperatur","show":true, "color":"#283074"},
 		      {"sensorname":"heizkreisregler9.solarDaten.tempWarmwasserSolar","show":false, "color":"#f0ff4c"},
 		      {"sensorname":"feuerungsautomat1.betriebsdatenRegler1.aussenTemperatur","show":false, "color":"#84b500"},
-		      {"sensorname":"de.wettermichel.temperature","show":true, "color":"#24f590"},
+	//	      {"sensorname":"de.wettermichel.temperature","show":true, "color":"#24f590"},
 		      {"sensorname":"heizkreisregler10.betriebsdatenRegler1.boilerTemperatur","show":true, "color":"#48b4ff"}];
 
     var pickSensorConfig = function(sensorname) {
@@ -40,7 +40,7 @@ $(document).ready(function(){
 	    plotDetail = $.plot($("#ebusgraph"), 
 				datasetDetail, 
 				{ 
-				    xaxis: { mode: "time", min: from, max:to },
+				    xaxis: { mode: "time", min: tzFix(from), max:tzFix(to) },
 				    yaxis: { min: -16, max: 100 },
 				    legend: { show : true}
 				});
@@ -54,7 +54,7 @@ $(document).ready(function(){
 				      lines: { show: true, lineWidth: 1 },
 				      shadowSize: 0
 				  },
-				  xaxis: { mode: "time", min: fromOverview, max: d.now },
+				  xaxis: { mode: "time", min: tzFix(fromOverview), max: tzFix(d.now) },
 				  yaxis: { ticks: [], min: -26, max: 100, autoscaleMargin: 0.1 },
 				  legend: { show: false },
 		   		  selection: { mode: "x" }
@@ -73,15 +73,16 @@ $(document).ready(function(){
 	unplotSensorDetail(sensorname);
 	unplotSensorOverview(sensorname);
     };
-    var tzFix = function(d) {
-        return d; //return d - new Date().getTimezoneOffset() * 60 * 1000;
-    }
+    var tzFix = function(d) { return d - new Date().getTimezoneOffset() * 60 * 1000; }
+    var tzUnfix = function(d) { return d + new Date().getTimezoneOffset() * 60 * 1000; }
 
     var plotSensorDetail = function(sensorConfig) {
 	$.getJSON("sensor/"+escape(sensorConfig.sensorname)+"/"+int(from/1000)+"/"+int(to/1000),
 		  function(response) {
 		      if (!response.error) {
-              response.data = response.data.map(function(d) { return [ tzFix(d[0]), d[1] ]; });
+	                  response.data = response.data.map(function(d) {
+				  return [ tzFix(d[0]*1000), d[1] ]; 
+			  });
 			  datasetDetail.push({'data':response['data'],
 					'label':sensorConfig.sensorname,
 					'color':sensorConfig.color});
@@ -106,7 +107,9 @@ $(document).ready(function(){
 	$.getJSON("avg/"+escape(sensorConfig.sensorname)+"/"+int(fromOverview/1000),
 		  function(response) {
 		      if (!response.error) {
-              response.data = response.data.map(function(d) { return [ tzFix(d[0]), d[1] ]; });
+                          response.data = response.data.map(function(d) { 
+				  return [ tzFix(d[0]*1000), d[1] ]; 
+			  });
 			  datasetOverview.push({'data':response['data'],
 						'label':sensorConfig.sensorname,
 						'color':sensorConfig.color});
@@ -136,8 +139,8 @@ $(document).ready(function(){
 	    plotOverview.setSelection({xaxis: {'from': from, 'to': to}}, true);
 	    return;
 	} else {
-	    from = range_from;
-	    to = range_to;
+	    from = tzUnfix( range_from );
+	    to = tzUnfix( range_to );
 	}
 	sensors = [];
 	for (elem in datasetOverview) {