leveldb: size restricted range query

3 files changed, 110 insertions, 43 deletions

diff --git a/datastore-leveldb/src/main.cpp b/datastore-leveldb/src/main.cpp
index 3b0dd98..3000a25 100644
--- a/datastore-leveldb/src/main.cpp
+++ b/datastore-leveldb/src/main.cpp
@@ -19,10 +19,12 @@ struct mg_context *ctx;
 int begin_request_handler(struct mg_connection *conn) {
     boost::cmatch match;
 
-    const char *url = mg_get_request_info(conn)->uri;
+    const struct mg_request_info *request_info = mg_get_request_info(conn);
 
     for (auto item = web_handler.begin(); item != web_handler.end(); ++item) {
-        if (boost::regex_match(url, match, (*item).first)) {
+        if (boost::regex_match(request_info->uri, match, (*item).first)) {
+            std::cout << request_info->request_method << " "
+                      << request_info->uri << std::endl;
             (*item).second(match, conn);
             return 1;
         }
@@ -52,12 +54,15 @@ int main(int argc, char **argv) {
 
     // Routing
     web_handler.push_front(std::make_pair(
-                boost::regex("/api/value/([a-zA-Z0-9]+)/([0-9]+)"),
+                web_handle_api_value_R,
                 web_handle_api_value));
     web_handler.push_front(std::make_pair(
-                boost::regex("/api/range/([a-zA-Z0-9]+)/([0-9]+)/([0-9]+)"),
+                web_handle_api_range_R,
                 web_handle_api_range));
-
+    web_handler.push_front(std::make_pair(
+                web_handle_api_range_size_R,
+                web_handle_api_range_size));
+    
 
     // Signals: handle C-c
     struct sigaction action;
diff --git a/datastore-leveldb/src/web.cpp b/datastore-leveldb/src/web.cpp
index 4898451..2adf8eb 100644
--- a/datastore-leveldb/src/web.cpp
+++ b/datastore-leveldb/src/web.cpp
@@ -1,8 +1,12 @@
 #include "web.h"
 #include "db.h"
+#include <algorithm>
+#include <vector>
 #include <leveldb/comparator.h>
 #include <iomanip>
 
+static const leveldb::Comparator *cmp = leveldb::BytewiseComparator();
+
 static inline std::string make_key(uint64_t timestamp) {
     std::stringstream key;
     key << "ts-";
@@ -10,9 +14,17 @@ static inline std::string make_key(uint64_t timestamp) {
     return key.str();
 }
 
-void web_handle_api_value(const boost::cmatch &match, struct mg_connection *conn) {
-//    const struct mg_request_info *request_info = mg_get_request_info(conn);
+static inline bool parse_key(leveldb::Slice &&key, uint64_t *value) {
+    if (key.size() != 20+3) 
+        return false;
+    *value = std::stoul(key.data()+3);
+    return true;
+}
 
+
+const boost::regex web_handle_api_value_R(
+    "/api/value/([a-zA-Z0-9]+)/([0-9]+)");
+void web_handle_api_value(const boost::cmatch &match, struct mg_connection *conn) {
     std::string sensor(match[1].str());
     uint64_t timestamp = std::stoul(match[2].str());
     char buf[1024];
@@ -26,14 +38,39 @@ void web_handle_api_value(const boost::cmatch &match, struct mg_connection *conn
         return;
     }
 
-    std::cout << "sensor=" << sensor << " key=" << make_key(timestamp) << std::endl;
     db->Put(leveldb::WriteOptions(), make_key(timestamp), value);
     mg_printf(conn, "HTTP/1.1 200 Value received\r\n\r\n");
 }
 
-void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn) {
-    static const leveldb::Comparator *cmp = leveldb::BytewiseComparator();
+static inline void print_json_tuple(struct mg_connection *conn,
+        std::ostringstream &outbuf,
+        leveldb::Slice &&key,
+        leveldb::Slice &&value) {
+
+    size_t key_size =  key.size();
+    if (key_size != 20+3) {
+        std::cerr << "invalid key" << std::endl;
+        return;
+    }
+
+    unsigned int offset = 3; // "ts-"
+    // skip zeros in timestamp
+    while (offset < key_size-1 and *(key.data()+offset) == '0')
+        offset++;
+
+    outbuf << '[';
+    outbuf.write(key.data()+offset, key_size-offset);
+    outbuf << ',';
+    outbuf.write(value.data(), value.size());
+    outbuf << ']';
 
+    mg_write(conn, outbuf.str().c_str(), outbuf.tellp());
+}
+
+
+const boost::regex web_handle_api_range_R(
+    "/api/range/([a-zA-Z0-9]+)/([0-9]+)/([0-9]+)");
+void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn) {
     std::string sensor(match[1].str());
     uint64_t start = std::stoul(match[2].str());
     uint64_t end = std::stoul(match[3].str());
@@ -42,17 +79,16 @@ void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn
 
     leveldb::DB *db = getDB(sensor);
     if (db == nullptr) {
-        mg_printf(conn, "HTTP/1.1 500 Internal Error\r\n\r\n");
+        mg_printf(conn, "HTTP/1.1 500 Internal Error\r\n"
+                        "\r\n");
         return;
     }
 
     mg_printf(conn,
-            "HTTP/1.1 200 Value received\r\n"
+            "HTTP/1.1 200 OK\r\n"
             "Content-Type: application/json; encoding=UTF-8\r\n"
             "\r\n");
 
-    std::cout << "sensor=" << sensor << " start=" << start << " end=" << end << std::endl;
-
     std::ostringstream out;
     out << "{'sensor':'" << sensor << "', 'error':null, 'data':[";
     mg_write(conn, out.str().c_str(), out.str().size());
@@ -63,33 +99,59 @@ void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn
     for (it->Seek(key_start); 
             it->Valid() && cmp->Compare(it->key(), key_end) < 0;
             it->Next()) {
-        const char *key = it->key().data();
-        size_t key_size =  it->key().size();
-        if (key_size != 20+3) {
-            std::cerr << "invalid key" << std::endl;
-            return;
-        }
-
-        unsigned int offset = 3; // "ts-"
-        // skip zeros in timestamp
-        while (offset < key_size-1 and *(key+offset) == '0')
-            offset++;
-
-        if (first)
-            first = false;
-        else
-            outbuf << ',';
-
-        outbuf << '[';
-        outbuf.write(key+offset, key_size-offset);
-        outbuf << ',';
-        outbuf.write(it->value().data(), it->value().size());
-        outbuf << ']';
-
-        mg_write(conn, outbuf.str().c_str(), outbuf.tellp());
+        if (first) first = false;
+        else outbuf << ",";
+        print_json_tuple(conn, outbuf, it->key(), it->value());
         outbuf.seekp(0);
     }
     mg_printf(conn, "]}\r\n");
     delete it;
 }
 
+const boost::regex web_handle_api_range_size_R(
+    "/api/range/([a-zA-Z0-9]+)/([0-9]+)/([0-9]+)/([0-9]+)");
+void web_handle_api_range_size(const boost::cmatch &match, struct mg_connection *conn) {
+    std::string sensor(match[1].str());
+    uint64_t start = std::stoul(match[2].str());
+    uint64_t end = std::stoul(match[3].str());
+    uint64_t size = std::stoul(match[4].str());
+
+    leveldb::DB *db = getDB(sensor);
+    if (db == nullptr) {
+        mg_printf(conn, "HTTP/1.1 500 Internal Error\r\n"
+                        "\r\n");
+        return;
+    }
+
+    mg_printf(conn,
+            "HTTP/1.1 200 OK\r\n"
+            "Content-Type: application/json; encoding=UTF-8\r\n"
+            "\r\n");
+
+    uint64_t step = std::max((uint64_t)1, (end-start) / size);
+
+    std::ostringstream out;
+    out << "{'sensor':'" << sensor << "', "
+        << "'error':null, "
+        << "'size':" << size << ", "
+        << "'step':" << step << ", "
+        << "'data':[";
+    mg_write(conn, out.str().c_str(), out.str().size());
+
+    bool first = true;
+    uint64_t actual_size = 0;
+    std::ostringstream outbuf;
+    leveldb::Iterator* it = db->NewIterator(leveldb::ReadOptions());
+    for (uint64_t key = start; key <= end; key += step) {
+        it->Seek(make_key(key));
+
+        if (first) first = false;
+        else outbuf << ",";
+
+        print_json_tuple(conn, outbuf, it->key(), it->value());
+        outbuf.seekp(0);
+        actual_size++;
+    }
+    mg_printf(conn, "], 'actual_size':%ld}\r\n", actual_size);
+    delete it;
+}
diff --git a/datastore-leveldb/src/web.h b/datastore-leveldb/src/web.h
index 1360d3d..acb38f5 100644
--- a/datastore-leveldb/src/web.h
+++ b/datastore-leveldb/src/web.h
@@ -4,15 +4,15 @@
 extern "C" {
 #include "mongoose.h"
 }
-#ifdef __clang__
-#include <cstddef>
-#warning "Using clang" 
-#define BOOST_NO_CXX11_NUMERIC_LIMITS 
-#endif
 #include <boost/regex.hpp>
 
+extern const boost::regex web_handle_api_value_R;
 void web_handle_api_value(const boost::cmatch &match, struct mg_connection *conn);
 
+extern const boost::regex web_handle_api_range_R;
 void web_handle_api_range(const boost::cmatch &match, struct mg_connection *conn);
 
+extern const boost::regex web_handle_api_range_size_R;
+void web_handle_api_range_size(const boost::cmatch &match, struct mg_connection *conn);
+
 #endif/*HAVE_WEB_H*/