1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#
import os
from threading import Lock
import tables
import logging
class ValueFloat(tables.IsDescription):
timestamp = tables.Time32Col() #index on Time64 is broken on pytables 2.4
value = tables.Float32Col()
class ValueInt(tables.IsDescription):
timestamp = tables.Time32Col()
value = tables.Int32Col()
class ValueString(tables.IsDescription):
timestamp = tables.Time32Col()
value = tables.StringCol(20)
class Datastore(object):
def __init__(self,basepath):
self.basepath = basepath
self.files = {}
self.fileLock = Lock()
self.tables = {}
# list all table in a dictionary { path.dot.name : object }
def listTables(self):
return dict(map(lambda e: (e.title, e), self.files.root))
def __enter__(self):
self.fileLock.__enter__()
def __exit__(self, *a):
self.fileLock.__exit__(*a)
def getTable(self, name, klass=None):
name = name.replace(".","_")
with self.fileLock:
if not name in self.files:
path = os.path.join(self.basepath, name+".hdf")
print "open {0}".format(path)
if os.path.exists(path):
self.files[name] = tables.openFile(path, "a", title = "eBus Datastore")
elif klass:
self.files[name] = tables.openFile(path, "w", title = "eBus Datastore")
else:
raise Exception("No such sensor {0}".format(name))
if not name in self.tables:
t = None
try:
t = self.files[name].getNode("/"+name)
except tables.NoSuchNodeError,e:
if not klass: raise e
t = self.files[name].createTable("/",
name,
klass,
'Description ' + name,
filters=tables.Filters(complevel=1),
createparents=True)
t.cols.timestamp.createCSIndex()
self.tables[name] = t
return self.tables[name]
def addValue(self, name, ts, value, klass):
t = self.getTable(name, klass)
if klass != None:
assert klass.columns['value'].type == t.cols.value.type, "Type check failed"
with self.fileLock:
t.row['timestamp'] = ts
t.row['value'] = value
t.row.append()
def addValueInt(self, name, ts, value): self.addValue(name, ts, value, ValueInt)
def addValueString(self, name, ts, value): self.addValue(name, ts, value, ValueString)
def addValueFloat(self, name, ts, value): self.addValue(name, ts, value, ValueFloat)
def flush(self):
with self.fileLock:
for file in self.files.values():
logging.debug("flush file: %s" % file.filename)
file.flush()
def flush_table(self,name,table,close=False,index=False):
if close or table.cols.timestamp.index.nelements == 0:
logging.debug("close table: %s" % table.name)
name = name.replace(".","_")
#close file after first insert
#to force a index build on re-open
self.tables.pop(name).close()
self.files.pop(name).close()
elif 'row' in table.__dict__:
logging.debug("flush buffer {name}".format(name=name))
table.row._flushBufferedRows()
if index and table.indexed and table.autoIndex:
logging.debug("index {name}".format(name=name))
# Flush any unindexed row
rowsadded = table.flushRowsToIndex(_lastrow=True)
if table._dirtyindexes:
# Finally, re-index any dirty column
table.reIndexDirty()
def close(self):
with self.fileLock:
for file in self.files.values():
file.close()
|
