From 604e92559787ed7e7b590321f6a1ff8fc515e06d Mon Sep 17 00:00:00 2001 From: Ebus-at-dockstar Date: Thu, 7 Mar 2013 14:20:42 +0100 Subject: python datastore + webhdf --- ebus/datastore.py | 21 ++++++++++++++++++--- ebus/webhdf/__init__.py | 26 ++++++++++++++++---------- ebus/webhdf/static/css/stylesheet.css | 15 +++++++++------ ebus/webhdf/static/index.html | 2 +- ebus/webhdf/static/src/ebus.js | 25 ++++++++++++++----------- 5 files changed, 58 insertions(+), 31 deletions(-) (limited to 'ebus') diff --git a/ebus/datastore.py b/ebus/datastore.py index 88dcdc9..930b4fb 100644 --- a/ebus/datastore.py +++ b/ebus/datastore.py @@ -40,8 +40,12 @@ class Datastore(object): if not name in self.files: path = os.path.join(self.basepath, name+".hdf") print "open {0}".format(path) - file = tables.openFile(path, "a", title = "eBus Datastore") - self.files[name] = file + if os.path.exists(path): + self.files[name] = tables.openFile(path, "a", title = "eBus Datastore") + elif klass: + self.files[name] = tables.openFile(path, "w", title = "eBus Datastore") + else: + raise Exception("No such sensor {0}".format(name)) if not name in self.tables: t = None @@ -49,6 +53,7 @@ class Datastore(object): t = self.files[name].getNode("/"+name) except tables.NoSuchNodeError,e: if not klass: raise e + t = self.files[name].createTable("/", name, klass, @@ -56,17 +61,27 @@ class Datastore(object): filters=tables.Filters(complevel=1), createparents=True) t.cols.timestamp.createCSIndex() + self.files[name].close() + + self.files[name] = tables.openFile(path, "a", title = "eBus Datastore") + t = self.files[name].getNode("/"+name) + self.tables[name] = t return self.tables[name] def addValue(self, name, ts, value, klass,flush=False): t = self.getTable(name, klass) + if klass != None: + assert klass.columns['value'].type == t.cols.value.type, "Type check failed" + with self.fileLock: t.row['timestamp'] = ts t.row['value'] = value t.row.append() - if flush: t.flush() + if flush: + t.flush() + t.flushRowsToIndex() def addValueInt(self, name, ts, value): self.addValue(name, ts, value, ValueInt) def addValueString(self, name, ts, value): self.addValue(name, ts, value, ValueString) diff --git a/ebus/webhdf/__init__.py b/ebus/webhdf/__init__.py index af13f87..c1f34dd 100644 --- a/ebus/webhdf/__init__.py +++ b/ebus/webhdf/__init__.py @@ -13,7 +13,7 @@ import bottle import ebus.datastore -datastore = ebus.datastore.Datastore("testhdffiles") +datastore = ebus.datastore.Datastore("hdf-data") app = bottle.Bottle("ebus") @@ -50,6 +50,8 @@ def sensor_data_put(name,timestamp=None): klass = ebus.datastore.ValueFloat elif type == "string": klass = ebus.datastore.ValueString + elif type == "": + klass = None else: return {'error':'INVALID_TYPE', msg:'Type {0} is invalid'.format(type)} @@ -58,38 +60,42 @@ def sensor_data_put(name,timestamp=None): logging.info(msg) return {'error':None,'msg':msg} except Exception,e: - return {'error':e,'msg':e} + logging.error("Error: " + "{0} name={1} value={2} type={3}".format(e, name, value, type)) + return {'error':str(e),'msg':str(e)} @app.route('/sensor/:name/:startdate/:enddate') def sensor_name_start_end(name,startdate,enddate): + SAMPLING_STEPSIZE=500 try: startdate, enddate = int(startdate), int(enddate) logging.info("/sensor/ start={0} end={1}".format(startdate, enddate)) table=datastore.getTable(name) with datastore: - i = table.where("(timestamp >= startdate) & (timestamp <= enddate)",step=100) + i = table.where("(timestamp >= startdate) & (timestamp <= enddate)", + condvars={'startdate':startdate,'enddate':enddate,'timestamp':table.cols.timestamp}, + step=SAMPLING_STEPSIZE) timestamps = [] try: for x in range(20): i.next() timestamps.append(i['timestamp']) - except: - pass + except: pass + if len(timestamps) > 10: - diff = map(lambda (x1,x2): (x2-x1)/100, zip(timestamps[:-1], timestamps[1:])) + diff = map(lambda (x1,x2): (x2-x1)/SAMPLING_STEPSIZE, zip(timestamps[:-1], timestamps[1:])) diff_avg = numpy.average(diff) time_period = enddate - startdate samples = time_period / diff_avg step = numpy.ceil(samples / 400.0) - data = [(x['timestamp']*1000, x['value']) + data = [(x['timestamp'], x['value']) for x in table.where("(timestamp >= startdate) & (timestamp <= enddate)", step=step)] logging.info("diff={0} samples={1} step={2} len={3} ({4})".format(diff_avg, samples, step, len(data),name)) else: - logging.info("No data found ({0})".format(name)) - data = [] + # Not enough data, sampling approach useless, deliver all data + data = [(x['timestamp'], x['value']) for x in table.where("(timestamp >= startdate) & (timestamp <= enddate)")] return {'sensor':name, 'error':None,'data':data} except Exception,e: logging.error("Error: " + str(e) + str(type(e))) @@ -107,7 +113,7 @@ def sensor_avg_start(name, startdate, period=60*15): #15min f_group = range(startdate, enddate, period) data = map(lambda (group_id, grouped_rows): (group_id, numpy.average([row['value'] for row in grouped_rows])), itertools.groupby(sel_rows, lambda t: (t['timestamp']/period)*period)) - data = map(lambda (timestamp,value): (timestamp*1000, value), data) + data = map(lambda (timestamp,value): (timestamp, value), data) return {'sensor':name, 'error':None,'data':data} except Exception,e: return {'sensor':name, 'error':str(e), 'data':None} diff --git a/ebus/webhdf/static/css/stylesheet.css b/ebus/webhdf/static/css/stylesheet.css index edc0c9b..cfaa2fe 100644 --- a/ebus/webhdf/static/css/stylesheet.css +++ b/ebus/webhdf/static/css/stylesheet.css @@ -1,22 +1,25 @@ body { + margin: 0; font-family:sans; text-align:center; + width:100%; } #ebusgraph { - margin:auto; - width:100%; + margin: auto; + width: 90%; height: 70%; } #overview { - width: 100%; - margin:auto; - height:100px; + margin: 2% auto 0% auto; + width: 90%; + height: 100px; } #options { - width:800px; + height: 10%; + width: 800px; margin:auto; text-align:left; } diff --git a/ebus/webhdf/static/index.html b/ebus/webhdf/static/index.html index 04ecc15..53f103e 100644 --- a/ebus/webhdf/static/index.html +++ b/ebus/webhdf/static/index.html @@ -8,9 +8,9 @@
+
-
diff --git a/ebus/webhdf/static/src/ebus.js b/ebus/webhdf/static/src/ebus.js index ce37557..5c4fa16 100644 --- a/ebus/webhdf/static/src/ebus.js +++ b/ebus/webhdf/static/src/ebus.js @@ -11,7 +11,7 @@ d.now = new Date().getTime(); function int(n) { return Math.round(n); }; $(document).ready(function(){ - var from = d.now - 5*d.day; + var from = d.now - 1*d.day; var fromOverview = d.now - 30*d.day; var to = d.now; var datasetDetail = [] @@ -23,7 +23,7 @@ $(document).ready(function(){ {"sensorname":"heizkreisregler10.betriebsdatenRegler1.kesselTemperatur","show":true, "color":"#283074"}, {"sensorname":"heizkreisregler9.solarDaten.tempWarmwasserSolar","show":false, "color":"#f0ff4c"}, {"sensorname":"feuerungsautomat1.betriebsdatenRegler1.aussenTemperatur","show":false, "color":"#84b500"}, - {"sensorname":"de.wettermichel.temperature","show":true, "color":"#24f590"}, + // {"sensorname":"de.wettermichel.temperature","show":true, "color":"#24f590"}, {"sensorname":"heizkreisregler10.betriebsdatenRegler1.boilerTemperatur","show":true, "color":"#48b4ff"}]; var pickSensorConfig = function(sensorname) { @@ -40,7 +40,7 @@ $(document).ready(function(){ plotDetail = $.plot($("#ebusgraph"), datasetDetail, { - xaxis: { mode: "time", min: from, max:to }, + xaxis: { mode: "time", min: tzFix(from), max:tzFix(to) }, yaxis: { min: -16, max: 100 }, legend: { show : true} }); @@ -54,7 +54,7 @@ $(document).ready(function(){ lines: { show: true, lineWidth: 1 }, shadowSize: 0 }, - xaxis: { mode: "time", min: fromOverview, max: d.now }, + xaxis: { mode: "time", min: tzFix(fromOverview), max: tzFix(d.now) }, yaxis: { ticks: [], min: -26, max: 100, autoscaleMargin: 0.1 }, legend: { show: false }, selection: { mode: "x" } @@ -73,15 +73,16 @@ $(document).ready(function(){ unplotSensorDetail(sensorname); unplotSensorOverview(sensorname); }; - var tzFix = function(d) { - return d; //return d - new Date().getTimezoneOffset() * 60 * 1000; - } + var tzFix = function(d) { return d - new Date().getTimezoneOffset() * 60 * 1000; } + var tzUnfix = function(d) { return d + new Date().getTimezoneOffset() * 60 * 1000; } var plotSensorDetail = function(sensorConfig) { $.getJSON("sensor/"+escape(sensorConfig.sensorname)+"/"+int(from/1000)+"/"+int(to/1000), function(response) { if (!response.error) { - response.data = response.data.map(function(d) { return [ tzFix(d[0]), d[1] ]; }); + response.data = response.data.map(function(d) { + return [ tzFix(d[0]*1000), d[1] ]; + }); datasetDetail.push({'data':response['data'], 'label':sensorConfig.sensorname, 'color':sensorConfig.color}); @@ -106,7 +107,9 @@ $(document).ready(function(){ $.getJSON("avg/"+escape(sensorConfig.sensorname)+"/"+int(fromOverview/1000), function(response) { if (!response.error) { - response.data = response.data.map(function(d) { return [ tzFix(d[0]), d[1] ]; }); + response.data = response.data.map(function(d) { + return [ tzFix(d[0]*1000), d[1] ]; + }); datasetOverview.push({'data':response['data'], 'label':sensorConfig.sensorname, 'color':sensorConfig.color}); @@ -136,8 +139,8 @@ $(document).ready(function(){ plotOverview.setSelection({xaxis: {'from': from, 'to': to}}, true); return; } else { - from = range_from; - to = range_to; + from = tzUnfix( range_from ); + to = tzUnfix( range_to ); } sensors = []; for (elem in datasetOverview) { -- cgit v1.2.1