From 7c8e064216cc672557cbf3892ff090490505e408 Mon Sep 17 00:00:00 2001 From: Yves Fischer Date: Thu, 18 Apr 2013 15:42:54 +0200 Subject: leveldb: size restricted range query --- datastore-leveldb/src/main.cpp | 15 +++-- datastore-leveldb/src/web.cpp | 128 ++++++++++++++++++++++++++++++----------- datastore-leveldb/src/web.h | 10 ++-- 3 files changed, 110 insertions(+), 43 deletions(-) (limited to 'datastore-leveldb/src') diff --git a/datastore-leveldb/src/main.cpp b/datastore-leveldb/src/main.cpp index 3b0dd98..3000a25 100644 --- a/datastore-leveldb/src/main.cpp +++ b/datastore-leveldb/src/main.cpp @@ -19,10 +19,12 @@ struct mg_context *ctx; int begin_request_handler(struct mg_connection *conn) { boost::cmatch match; - const char *url = mg_get_request_info(conn)->uri; + const struct mg_request_info *request_info = mg_get_request_info(conn); for (auto item = web_handler.begin(); item != web_handler.end(); ++item) { - if (boost::regex_match(url, match, (*item).first)) { + if (boost::regex_match(request_info->uri, match, (*item).first)) { + std::cout << request_info->request_method << " " + << request_info->uri << std::endl; (*item).second(match, conn); return 1; } @@ -52,12 +54,15 @@ int main(int argc, char **argv) { // Routing web_handler.push_front(std::make_pair( - boost::regex("/api/value/([a-zA-Z0-9]+)/([0-9]+)"), + web_handle_api_value_R, web_handle_api_value)); web_handler.push_front(std::make_pair( - boost::regex("/api/range/([a-zA-Z0-9]+)/([0-9]+)/([0-9]+)"), + web_handle_api_range_R, web_handle_api_range)); - + web_handler.push_front(std::make_pair( + web_handle_api_range_size_R, + web_handle_api_range_size)); + // Signals: handle C-c struct sigaction action; diff --git a/datastore-leveldb/src/web.cpp b/datastore-leveldb/src/web.cpp index 4898451..2adf8eb 100644 --- a/datastore-leveldb/src/web.cpp +++ b/datastore-leveldb/src/web.cpp @@ -1,8 +1,12 @@ #include "web.h" #include "db.h" +#include +#include #include #include +static const leveldb::Comparator *cmp = leveldb::BytewiseComparator(); + static inline std::string make_key(uint64_t timestamp) { std::stringstream key; key << "ts-"; @@ -10,9 +14,17 @@ static inline std::string make_key(uint64_t timestamp) { return key.str(); } -void web_handle_api_value(const boost::cmatch &match, struct mg_connection *conn) { -// const struct mg_request_info *request_info = mg_get_request_info(conn); +static inline bool parse_key(leveldb::Slice &&key, uint64_t *value) { + if (key.size() != 20+3) + return false; + *value = std::stoul(key.data()+3); + return true; +} + +const boost::regex web_handle_api_value_R( + "/api/value/([a-zA-Z0-9]+)/([0-9]+)"); +void web_handle_api_value(const boost::cmatch &match, struct mg_connection *conn) { std::string sensor(match[1].str()); uint64_t timestamp = std::stoul(match[2].str()); char buf[1024]; @@ -26,14 +38,39 @@ void web_handle_api_value(const boost::cmatch &match, struct mg_connection *conn return; } - std::cout << "sensor=" << sensor << " key=" << make_key(timestamp) << std::endl; db->Put(leveldb::WriteOptions(), make_key(timestamp), value); mg_printf(conn, "HTTP/1.1 200 Value received\r\n\r\n"); } -void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn) { - static const leveldb::Comparator *cmp = leveldb::BytewiseComparator(); +static inline void print_json_tuple(struct mg_connection *conn, + std::ostringstream &outbuf, + leveldb::Slice &&key, + leveldb::Slice &&value) { + + size_t key_size = key.size(); + if (key_size != 20+3) { + std::cerr << "invalid key" << std::endl; + return; + } + + unsigned int offset = 3; // "ts-" + // skip zeros in timestamp + while (offset < key_size-1 and *(key.data()+offset) == '0') + offset++; + + outbuf << '['; + outbuf.write(key.data()+offset, key_size-offset); + outbuf << ','; + outbuf.write(value.data(), value.size()); + outbuf << ']'; + mg_write(conn, outbuf.str().c_str(), outbuf.tellp()); +} + + +const boost::regex web_handle_api_range_R( + "/api/range/([a-zA-Z0-9]+)/([0-9]+)/([0-9]+)"); +void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn) { std::string sensor(match[1].str()); uint64_t start = std::stoul(match[2].str()); uint64_t end = std::stoul(match[3].str()); @@ -42,17 +79,16 @@ void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn leveldb::DB *db = getDB(sensor); if (db == nullptr) { - mg_printf(conn, "HTTP/1.1 500 Internal Error\r\n\r\n"); + mg_printf(conn, "HTTP/1.1 500 Internal Error\r\n" + "\r\n"); return; } mg_printf(conn, - "HTTP/1.1 200 Value received\r\n" + "HTTP/1.1 200 OK\r\n" "Content-Type: application/json; encoding=UTF-8\r\n" "\r\n"); - std::cout << "sensor=" << sensor << " start=" << start << " end=" << end << std::endl; - std::ostringstream out; out << "{'sensor':'" << sensor << "', 'error':null, 'data':["; mg_write(conn, out.str().c_str(), out.str().size()); @@ -63,33 +99,59 @@ void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn for (it->Seek(key_start); it->Valid() && cmp->Compare(it->key(), key_end) < 0; it->Next()) { - const char *key = it->key().data(); - size_t key_size = it->key().size(); - if (key_size != 20+3) { - std::cerr << "invalid key" << std::endl; - return; - } - - unsigned int offset = 3; // "ts-" - // skip zeros in timestamp - while (offset < key_size-1 and *(key+offset) == '0') - offset++; - - if (first) - first = false; - else - outbuf << ','; - - outbuf << '['; - outbuf.write(key+offset, key_size-offset); - outbuf << ','; - outbuf.write(it->value().data(), it->value().size()); - outbuf << ']'; - - mg_write(conn, outbuf.str().c_str(), outbuf.tellp()); + if (first) first = false; + else outbuf << ","; + print_json_tuple(conn, outbuf, it->key(), it->value()); outbuf.seekp(0); } mg_printf(conn, "]}\r\n"); delete it; } +const boost::regex web_handle_api_range_size_R( + "/api/range/([a-zA-Z0-9]+)/([0-9]+)/([0-9]+)/([0-9]+)"); +void web_handle_api_range_size(const boost::cmatch &match, struct mg_connection *conn) { + std::string sensor(match[1].str()); + uint64_t start = std::stoul(match[2].str()); + uint64_t end = std::stoul(match[3].str()); + uint64_t size = std::stoul(match[4].str()); + + leveldb::DB *db = getDB(sensor); + if (db == nullptr) { + mg_printf(conn, "HTTP/1.1 500 Internal Error\r\n" + "\r\n"); + return; + } + + mg_printf(conn, + "HTTP/1.1 200 OK\r\n" + "Content-Type: application/json; encoding=UTF-8\r\n" + "\r\n"); + + uint64_t step = std::max((uint64_t)1, (end-start) / size); + + std::ostringstream out; + out << "{'sensor':'" << sensor << "', " + << "'error':null, " + << "'size':" << size << ", " + << "'step':" << step << ", " + << "'data':["; + mg_write(conn, out.str().c_str(), out.str().size()); + + bool first = true; + uint64_t actual_size = 0; + std::ostringstream outbuf; + leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions()); + for (uint64_t key = start; key <= end; key += step) { + it->Seek(make_key(key)); + + if (first) first = false; + else outbuf << ","; + + print_json_tuple(conn, outbuf, it->key(), it->value()); + outbuf.seekp(0); + actual_size++; + } + mg_printf(conn, "], 'actual_size':%ld}\r\n", actual_size); + delete it; +} diff --git a/datastore-leveldb/src/web.h b/datastore-leveldb/src/web.h index 1360d3d..acb38f5 100644 --- a/datastore-leveldb/src/web.h +++ b/datastore-leveldb/src/web.h @@ -4,15 +4,15 @@ extern "C" { #include "mongoose.h" } -#ifdef __clang__ -#include -#warning "Using clang" -#define BOOST_NO_CXX11_NUMERIC_LIMITS -#endif #include +extern const boost::regex web_handle_api_value_R; void web_handle_api_value(const boost::cmatch &match, struct mg_connection *conn); +extern const boost::regex web_handle_api_range_R; void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn); +extern const boost::regex web_handle_api_range_size_R; +void web_handle_api_range_size(const boost::cmatch &match, struct mg_connection *conn); + #endif/*HAVE_WEB_H*/ -- cgit v1.2.1