From 773093acabfb84b88c37bb0342d6a8d4a6f8267a Mon Sep 17 00:00:00 2001 From: Ebus-at-dockstar Date: Sat, 26 Jul 2014 19:57:28 +0200 Subject: move ebus-datastore to heap --- heap/ebus-datastore/testfiles/bsdtest.py | 49 ++++++++++++++++++++ heap/ebus-datastore/testfiles/bsdtest_query.py | 22 +++++++++ heap/ebus-datastore/testfiles/datastore-test.py | 21 +++++++++ heap/ebus-datastore/testfiles/importtest.py | 61 +++++++++++++++++++++++++ heap/ebus-datastore/testfiles/test.py | 14 ++++++ 5 files changed, 167 insertions(+) create mode 100644 heap/ebus-datastore/testfiles/bsdtest.py create mode 100644 heap/ebus-datastore/testfiles/bsdtest_query.py create mode 100644 heap/ebus-datastore/testfiles/datastore-test.py create mode 100644 heap/ebus-datastore/testfiles/importtest.py create mode 100644 heap/ebus-datastore/testfiles/test.py (limited to 'heap/ebus-datastore/testfiles') diff --git a/heap/ebus-datastore/testfiles/bsdtest.py b/heap/ebus-datastore/testfiles/bsdtest.py new file mode 100644 index 0000000..b6587d0 --- /dev/null +++ b/heap/ebus-datastore/testfiles/bsdtest.py @@ -0,0 +1,49 @@ +import bsddb +import psycopg2 +from subprocess import Popen,PIPE +import time +import sys + +db = bsddb.btopen("test.btdbm") + +conn = psycopg2.connect("dbname=ebus user=ebus") +cur = conn.cursor() +sql = """SELECT * from sensor""" +cur.execute(sql) +sensors = cur.fetchall() +sensors = map(lambda a: (a[0], a[1]), sensors) +sensors = dict(sensors) +cur.close() +conn.close() + +pg_dump = Popen(["psql","-A","-t"], stdin=PIPE, stdout=PIPE) +pg_dump.stdin.write("COPY value TO stdout WITH DELIMITER '|' NULL AS '';\n") +pg_dump.stdin.close() + +c=0 +try: + while not pg_dump.stdout.closed: + line = pg_dump.stdout.readline() + c += 1 + if line == "": + continue + (oid, s_timestamp, sensor_id, type, value_float, value_int, value_string) = line.split("|") + name = sensors[int(sensor_id)] + timestamp = time.strptime(s_timestamp.split(".")[0], "%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timestamp) + if type == "int": + value= value_int + elif type == "float": + value = value_float + else: + continue + + db["%s:%d"%(name,timestamp)] = value + if c%10000==0: + print "%d: %s:%s=%s"%(c,name,s_timestamp,value) +except KeyboardInterrupt: + db.close() + +print "done" + + diff --git a/heap/ebus-datastore/testfiles/bsdtest_query.py b/heap/ebus-datastore/testfiles/bsdtest_query.py new file mode 100644 index 0000000..39bd3f1 --- /dev/null +++ b/heap/ebus-datastore/testfiles/bsdtest_query.py @@ -0,0 +1,22 @@ +import bsddb +import psycopg2 +from subprocess import Popen,PIPE +import time +import sys +import numpy as np + +db = bsddb.btopen("test.btdbm","r") + +c=0 +a=[] +print db.set_location("heizkreisregler10.betriebsdatenRegler1.kesselTemperatur") +while c<50000: + k,v = db.next() + a.append(float(v)) + c+=1 + +print "count=", len(a) +print "avg=", np.average(a) +print "done" + + diff --git a/heap/ebus-datastore/testfiles/datastore-test.py b/heap/ebus-datastore/testfiles/datastore-test.py new file mode 100644 index 0000000..8fd6847 --- /dev/null +++ b/heap/ebus-datastore/testfiles/datastore-test.py @@ -0,0 +1,21 @@ +import time +import struct +from ebus.datastore import Datastore + +d = Datastore("datastore-data", readonly=True) + +""" +config = d.get_config("heizkreisregler1.datumZeit.minuten") +for (key, value) in d.get_range("heizkreisregler1.datumZeit.minuten", + 1321056000000, + 1325100000000): + print time.strftime("%F %H:%M:%S", time.gmtime(key/1000)), \ + struct.unpack(config['format'].encode('latin-1'), value) +""" + +data = d.get_range_n("heizkreisregler1.datumZeit.minuten", + 1321056000000, 1325100000000,number=100) +print len(list(data)) + +d.close() + diff --git a/heap/ebus-datastore/testfiles/importtest.py b/heap/ebus-datastore/testfiles/importtest.py new file mode 100644 index 0000000..68cb989 --- /dev/null +++ b/heap/ebus-datastore/testfiles/importtest.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +import sys +import time +import psycopg2 +from datastore import Datastore +from subprocess import Popen,PIPE + +conn = psycopg2.connect("dbname=ebus user=ebus") +cur = conn.cursor() +sql = """SELECT * from sensor""" +cur.execute(sql) +sensors = cur.fetchall() +sensors = map(lambda a: (a[0], a[1]), sensors) +sensors = dict(sensors) +cur.close() +conn.close() + +d = Datastore("datastore-data") + +pg_dump = Popen(["psql","-A","-t"], stdin=PIPE, stdout=PIPE) +pg_dump.stdin.write("COPY value TO stdout WITH DELIMITER '|' NULL AS '';\n") +#pg_dump.stdin.write("SELECT * FROM value WHERE timestamp > current_timestamp - interval '30 days';\n\\q\\n") +pg_dump.stdin.close() + + +c=0 +while not pg_dump.stdout.closed: + line = pg_dump.stdout.readline() + c += 1 + if line == "": break + (oid, timestamp, sensor_id, type, value_float, value_int, value_string) = line.split("|") + name = sensors[int(sensor_id)] + + """ if not name in ("heizkreisregler9.solarDaten.tempKollektor", + "heizkreisregler10.betriebsdatenRegler1.kesselTemperatur", + "heizkreisregler9.solarDaten.tempWarmwasserSolar", + "feuerungsautomat1.betriebsdatenRegler1.aussenTemperatur", + "heizkreisregler10.betriebsdatenRegler1.boilerTemperatur"): + continue """ + + if c % 10000 == 0: print timestamp + + timestamp = time.strptime(timestamp.split(".")[0], "%Y-%m-%d %H:%M:%S") + timestamp = time.mktime(timestamp) + timestamp = timestamp * 1000 + timestamp = int(timestamp) + + if type == "int": + d.addValueInt(name, timestamp, int(value_int)) + elif type == "float": + d.addValueFloat(name, timestamp, float(value_float)) + elif type == "string": + d.addValueString(name, timestamp, value_string) + else: + print "invalid type %s" % type + # print "{name} {type} {timestamp} {value}".format(name=name, type=type, timestamp=timestamp, value=value) + + +d.close() + +# vim: autoindent tabstop=4 shiftwidth=4 expandtab softtabstop=4 filetype=python diff --git a/heap/ebus-datastore/testfiles/test.py b/heap/ebus-datastore/testfiles/test.py new file mode 100644 index 0000000..506439c --- /dev/null +++ b/heap/ebus-datastore/testfiles/test.py @@ -0,0 +1,14 @@ +import time +import ebus.datastore + +d = ebus.datastore.Datastore("datastore-data",readonly=True) + +start = long(time.time()-60*60*5)*1000 +end = long(time.time())*1000 +start = 1364030373029 +end = 1364116773029 +data = d.get_range_n("heizkreisregler9.solarDaten.tempKollektor", + start, + end, + 200) +print len(list(data)) -- cgit v1.2.1