diff options
| -rwxr-xr-x | fuzzer1.py | 2 | ||||
| -rw-r--r-- | pyinflux/client/__init__.py | 16 | ||||
| -rw-r--r-- | pyinflux/parser/__init__.py | 155 | ||||
| -rw-r--r-- | pyinflux/test/__init__.py | 2 | ||||
| -rw-r--r-- | pyinflux/test/test_client.py | 36 | ||||
| -rw-r--r-- | pyinflux/test/test_parser.py | 169 | ||||
| -rw-r--r-- | setup.py | 3 | ||||
| -rwxr-xr-x | test.py | 7 |
8 files changed, 261 insertions, 129 deletions
@@ -14,7 +14,7 @@ value_generator = itertools.count() def test(number): for value in value_generator: - if value + 1 % 500 == 0: + if (value + 1) % 500 == 0: print("thread {} at {}".format(number, value)) expected = "value{value}".format(value=value) line = Line('series' + str(value), diff --git a/pyinflux/client/__init__.py b/pyinflux/client/__init__.py index b485861..feb5381 100644 --- a/pyinflux/client/__init__.py +++ b/pyinflux/client/__init__.py @@ -32,13 +32,16 @@ class Line(object): @staticmethod def escape_value(obj): + DBLQ='"' if (isinstance(obj, float) or isinstance(obj, int) or isinstance(obj, bool)): return str(obj) else: obj = str(obj) - return "\"" + obj.replace("\\", "\\\\").replace("\"", "\\\"") + "\"" + obj = obj.replace('\\', '\\\\') + obj = obj.replace(DBLQ, '\\"') + return DBLQ + obj + DBLQ @staticmethod def escape_fields(kvlist): @@ -50,18 +53,10 @@ class Line(object): kvlist)) def __repr__(self): - """ - >>> print(repr(Line('test', [('a','b')], [('value','asd\\\\')]))) - <Line key=test tags=[('a', 'b')] fields=[('value', 'asd\\\\')] timestamp=None> - """ return "<{} key={} tags={} fields={} timestamp={}>".format( self.__class__.__name__, self.key, self.tags, self.fields, self.timestamp) def __str__(self): - """ - >>> print(Line('test', [('a','b')], [('value','asd\\\\')])) - test,a=b value="asd\\\\" - """ result = self.escape_identifier(self.key) if self.tags: @@ -138,6 +133,9 @@ class Influx: class InfluxDB(Influx): + """ + like Influx but with a predefined database + """ def __init__(self, db: str, host: str, port: int = 8086, username: str = None, password: str = None): super().__init__(host, port, username, password) self._db = db diff --git a/pyinflux/parser/__init__.py b/pyinflux/parser/__init__.py index 2de6c0e..4b91e99 100644 --- a/pyinflux/parser/__init__.py +++ b/pyinflux/parser/__init__.py @@ -11,27 +11,15 @@ except ImportError as e: from pyinflux import client -def parse_lines(lines): +def parse_lines(lines: str): """ Parse multiple Write objects separeted by new-line character. - - >>> print(LineParser.parse("foo b=1")) - foo b=1 - - >>> lines = [] - >>> lines += ['cpu field=123'] - >>> lines += ['cpu,host=serverA,region=us-west field1=1,field2=2'] - >>> lines += ['cpu,host=serverA,region=us-west field1=1,field2=2 1234'] - >>> print("\\n".join(map(str, parse_lines("\\n".join(lines))))) - cpu field=123 - cpu,host=serverA,region=us-west field1=1,field2=2 - cpu,host=serverA,region=us-west field1=1,field2=2 1234 """ writes = map(LineParser.parse, lines.split("\n")) return list(writes) -class LineParser(object): +class LineTokenizer: specs = [ ('Comma', (r',',)), ('Space', (r' ',)), @@ -44,111 +32,50 @@ class LineParser(object): ] @classmethod - def tokenize(klass, line : str): + def tokenize(klass, line: str): tokenizer = make_tokenizer(klass.specs) return list(tokenizer(line)) - @staticmethod - def parse(line): - """ - Parse a line from the POST request into a Write object. - - >>> line='cpu a=1'; LineParser.parse(line); print(LineParser.parse(line)) - <Line key=cpu tags=[] fields=[('a', 1)] timestamp=None> - cpu a=1 - - >>> print(LineParser.parse('yahoo.CHFGBP\\=X.ask,tag=foobar value=10.2')) - yahoo.CHFGBP\=X.ask,tag=foobar value=10.2 - - >>> LineParser.parse('cpu,host=serverA,region=us-west foo="bar"') - <Line key=cpu tags=[('host', 'serverA'), ('region', 'us-west')] fields=[('foo', 'bar')] timestamp=None> - - >>> print(LineParser.parse('cpu host="serverA",region="us-west"')) - cpu host="serverA",region="us-west" - - >>> line='cpu\\,01 host="serverA",region="us-west"'; \\ - ... LineParser.parse(line); print(LineParser.parse(line)) - <Line key=cpu,01 tags=[] fields=[('host', 'serverA'), ('region', 'us-west')] timestamp=None> - cpu\,01 host="serverA",region="us-west" - - >>> LineParser.parse('cpu host="server A",region="us west"') - <Line key=cpu tags=[] fields=[('host', 'server A'), ('region', 'us west')] timestamp=None> - - >>> line='cpu ho\\=st="server A",region="us west"'; \\ - ... LineParser.parse(line); print(LineParser.parse(line)) - <Line key=cpu tags=[] fields=[('ho=st', 'server A'), ('region', 'us west')] timestamp=None> - cpu ho\=st="server A",region="us west" - - >>> print(LineParser.parse('cpu,ho\=st=server\ A field=123')) - cpu,ho\=st=server\ A field=123 - - # error: double name is accepted - >>> print(LineParser.parse('cpu,foo=bar,foo=bar field=123,field=123')) - cpu,foo=bar,foo=bar field=123,field=123 - - >>> print(LineParser.parse('cpu field12=12')) - cpu field12=12 - - >>> print(LineParser.parse('cpu field12=12 123123123')) - cpu field12=12 123123123 - - >>> try: print(LineParser.parse('cpu field12=12 1231abcdef123')) - ... except NoParseError: pass - - >>> print(LineParser.parse('cpu,x=3,y=4,z=6 field\ name="HH \\\\\\"World",x="asdf foo"')) - cpu,x=3,y=4,z=6 field\\ name="HH \\"World",x="asdf foo" - - >>> print(LineParser.parse("cpu,x=3 field\ name=\\"HH \\\\\\"World\\",x=\\"asdf foo\\"")) - cpu,x=3 field\\ name="HH \\"World",x="asdf foo" - - >>> print(LineParser.parse("cpu foo=\\"bar\\" 12345")) - cpu foo="bar" 12345 - - >>> line='"measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\\\="string field value, only \\\\" need be quoted"'; \\ - ... LineParser.parse(line); print(LineParser.parse(line)) - <Line key="measurement with quotes" tags=[('tag key with spaces', 'tag,value,with"commas"')] fields=[('field_key\\\\', 'string field value, only " need be quoted')] timestamp=None> - "measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\="string field value, only \\\" need be quoted" - - >>> LineParser.parse('disk_free value=442221834240,working\ directories="C:\My Documents\Stuff for examples,C:\My Documents"') - <Line key=disk_free tags=[] fields=[('value', 442221834240), ('working directories', 'C:\\\\My Documents\\\\Stuff for examples,C:\\\\My Documents')] timestamp=None> - - >>> LineParser.parse('disk_free value=442221834240,working\ directories="C:\My Documents\Stuff for examples,C:\My Documents" 123') - <Line key=disk_free tags=[] fields=[('value', 442221834240), ('working directories', 'C:\\\\My Documents\\\\Stuff for examples,C:\\\\My Documents')] timestamp=123> - - >>> print(LineParser.parse('foo,foo=2 "field key with space"="string field"')) - foo,foo=2 field\ key\ with\ space="string field" - - >>> print(LineParser.parse('foo,foo=2 field_key\\\\\\="string field"')) - foo,foo=2 field_key\\\\="string field" - >>> print(LineParser.parse('foo,foo=2 field_key="string\\\\" field"')) - foo,foo=2 field_key="string\\" field" +class LineParser: + @staticmethod + def parse_identifier(line: str): + """Parses just the identifer (first element) of the write""" + tokval = lambda t: t.value + joinval = "".join + someToken = lambda type: some(lambda t: t.type == type) - >>> line='foo field0="tag",field1=t,field2=true,field3=True,field4=TRUE'; \\ - ... LineParser.parse(line); print(LineParser.parse(line)) - <Line key=foo tags=[] fields=[('field0', 'tag'), ('field1', True), ('field2', True), ('field3', True), ('field4', True)] timestamp=None> - foo field0="tag",field1=True,field2=True,field3=True,field4=True + char = someToken('Char') >> tokval + space = someToken('Space') >> tokval + comma = someToken('Comma') >> tokval + quote = someToken('Quote') >> tokval + escape = someToken('Escape') >> tokval + equal = someToken('Equal') >> tokval - >>> line='foo field1=f,field2=false,field3=False,field4=FALSE,field5="fag"'; \\ - ... LineParser.parse(line); print(LineParser.parse(line)) - <Line key=foo tags=[] fields=[('field1', False), ('field2', False), ('field3', False), ('field4', False), ('field5', 'fag')] timestamp=None> - foo field1=False,field2=False,field3=False,field4=False,field5="fag" + escape_space = skip(escape) + space >> joinval + escape_comma = skip(escape) + comma >> joinval + escape_equal = skip(escape) + equal >> joinval + escape_escape = skip(escape) + escape >> joinval - >>> line='"measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\\\="string field value, only \\\\" need be quoted"'; \\ - ... LineParser.parse(line); print(LineParser.parse(line)) - <Line key="measurement with quotes" tags=[('tag key with spaces', 'tag,value,with"commas"')] fields=[('field_key\\\\', 'string field value, only " need be quoted')] timestamp=None> - "measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\\="string field value, only \\" need be quoted" + plain_int_text = someToken('Int') >> tokval + plain_float_text = someToken('Float') >> tokval - >>> LineParser.parse('"measurement\ with\ quotes" foo=1') - <Line key="measurement with quotes" tags=[] fields=[('foo', 1)] timestamp=None> + identifier = many(char | plain_float_text | plain_int_text | + escape_space | escape_comma | escape_equal | + escape_escape | plain_int_text | quote) >> joinval - >>> print(LineParser.parse('K.5S,Ccpvo="eSLyE" value="7F\\\\\\\\\\\\""')) - K.5S,Ccpvo="eSLyE" value="7F\\\\\\\\\\\"" + toplevel = identifier >> (lambda x: x) + parsed = toplevel.parse(LineTokenizer.tokenize(line)) + if len(parsed) == 0: + raise NoParseError('parsed nothing') + else: + return parsed - >>> print(LineParser.parse('K.5S,Ccpvo=a\\ b value=1')) - K.5S,Ccpvo=a\\ b value=1 + @staticmethod + def parse(line: str): + """ + Parse a line from the POST request into a Write object. """ - tokval = lambda t: t.value joinval = "".join someToken = lambda type: some(lambda t: t.type == type) @@ -183,8 +110,8 @@ class LineParser(object): identifier = many(char | plain_float_text | plain_int_text | escape_space | escape_comma | escape_equal | escape_escape | plain_int_text | quote) >> joinval - quoted_text_ = many(escape_quote | space | plain_int_text | - plain_float_text | char | comma | + quoted_text_ = many(escape_escape | escape_quote | space | + plain_int_text | plain_float_text | char | comma | escape) >> joinval quoted_text = skip(quote) + quoted_text_ + skip(quote) unquoted_text = many(escape_space | escape_comma | @@ -199,10 +126,10 @@ class LineParser(object): tag = identifier + skip(equal) + identifier >> (lambda x: (x[0], x[1])) - def setter(obj, propert): + def setter(obj, property): def r(val): - setattr(obj, propert, val) - return (propert, val) + setattr(obj, property, val) + return (property, val) return r @@ -217,6 +144,6 @@ class LineParser(object): maybe(skip(space) + plain_int >> setter(write, "timestamp")) + \ skip(finished) >> (lambda x: x) - result = toplevel.parse(LineParser.tokenize(line)) + result = toplevel.parse(LineTokenizer.tokenize(line)) # pprint(result) return write diff --git a/pyinflux/test/__init__.py b/pyinflux/test/__init__.py new file mode 100644 index 0000000..b259470 --- /dev/null +++ b/pyinflux/test/__init__.py @@ -0,0 +1,2 @@ +from .test_parser import * +from .test_client import *
\ No newline at end of file diff --git a/pyinflux/test/test_client.py b/pyinflux/test/test_client.py new file mode 100644 index 0000000..2c9108f --- /dev/null +++ b/pyinflux/test/test_client.py @@ -0,0 +1,36 @@ +import json +import codecs +from unittest import TestCase +from pyinflux.client import Line, QueryResultOption +from io import BytesIO + + +class TestLine(TestCase): + def test_line(self): + self.assertEqual(str(Line('test', [('a', 'b')], [('value', 'asd\\\\')])), + r'test,a=b value="asd\\\\"') + + self.assertEqual(repr(Line('test', [('a', 'b')], [('value', 'asd\\\\')])), + r"<Line key=test tags=[('a', 'b')] fields=[('value', 'asd\\\\')] timestamp=None>") + + +class TestQueryResultOption(TestCase): + def test_json(self): + testobject = {'123': 456, '789': '456'} + buf = BytesIO() + json.dump(testobject, codecs.getwriter('utf-8')(buf)) + + buf.seek(0) + qro = QueryResultOption(lambda: buf) + self.assertEqual(testobject, qro.as_json()) + self.assertEqual(testobject, qro.as_json()) + + def test_text(self): + testobject = {'123': 456, '789': '456'} + buf = BytesIO() + json.dump(testobject, codecs.getwriter('utf-8')(buf)) + + buf.seek(0) + qro = QueryResultOption(lambda: buf) + self.assertEqual(json.dumps(testobject), qro.as_text()) + self.assertEqual(json.dumps(testobject), qro.as_text()) diff --git a/pyinflux/test/test_parser.py b/pyinflux/test/test_parser.py new file mode 100644 index 0000000..e4db23b --- /dev/null +++ b/pyinflux/test/test_parser.py @@ -0,0 +1,169 @@ +from unittest import TestCase +from pyinflux.parser import LineTokenizer, LineParser, parse_lines +from pyinflux.client import Line +from funcparserlib.lexer import Token +from funcparserlib.parser import NoParseError + + +class TestTokenize(TestCase): + def test_tokenize(self): + self.assertEqual(LineTokenizer.tokenize("cpu,host=serverA,region=us-west field1=1,field2=2"), + [Token('Char', 'c'), Token('Char', 'p'), Token('Char', 'u'), Token('Comma', ','), + Token('Char', 'h'), Token('Char', 'o'), Token('Char', 's'), Token('Char', 't'), + Token('Equal', '='), Token('Char', 's'), Token('Char', 'e'), Token('Char', 'r'), + Token('Char', 'v'), Token('Char', 'e'), Token('Char', 'r'), Token('Char', 'A'), + Token('Comma', ','), Token('Char', 'r'), Token('Char', 'e'), Token('Char', 'g'), + Token('Char', 'i'), Token('Char', 'o'), Token('Char', 'n'), Token('Equal', '='), + Token('Char', 'u'), Token('Char', 's'), Token('Char', '-'), Token('Char', 'w'), + Token('Char', 'e'), Token('Char', 's'), Token('Char', 't'), Token('Space', ' '), + Token('Char', 'f'), Token('Char', 'i'), Token('Char', 'e'), Token('Char', 'l'), + Token('Char', 'd'), Token('Int', '1'), Token('Equal', '='), Token('Int', '1'), + Token('Comma', ','), Token('Char', 'f'), Token('Char', 'i'), Token('Char', 'e'), + Token('Char', 'l'), Token('Char', 'd'), Token('Int', '2'), Token('Equal', '='), + Token('Int', '2')]) + + +class TestParseIdentifier(TestCase): + def test_identifier(self): + self.assertEqual(LineParser.parse_identifier('cpu a=1'), "cpu") + self.assertEqual(LineParser.parse_identifier('yahoo.CHFGBP\\=X.ask,tag=foobar value=10.2'), + "yahoo.CHFGBP=X.ask") + self.assertEqual(LineParser.parse_identifier('cpu,host=serverA,region=us-west foo="bar"'), "cpu") + self.assertEqual(LineParser.parse_identifier( + r'"measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\\="string field value, only \\" need be quoted"'), + "\"measurement with quotes\"") + + try: + LineParser.parse_identifier('') + self.fail() + except NoParseError: + pass + + try: + print(LineParser.parse_identifier(',')) + self.fail() + except NoParseError: + pass + + +class TestParseLine(TestCase): + def do_test(self, string: str, verify_line: Line): + line = LineParser.parse(string) + self.assertEqual(line.key, verify_line.key) + self.assertEqualLine(line, verify_line) + self.assertEqual(str(line), string) + + def assertEqualLine(self, line1, line2): + self.assertEqual(dict(line1.tags), dict(line2.tags)) + self.assertEqual(dict(line1.fields), dict(line2.fields)) + self.assertEqual(line1.timestamp, line2.timestamp) + + def test_parse_lines(self): + self.assertEqual("".join(map(str, parse_lines("foo b=1"))), 'foo b=1') + + text = """\ +cpu field=123 +cpu,host=serverA,region=us-west field1=1,field2=2 +cpu,host=serverA,region=us-west field1=1,field2=2 1234""" + writes = parse_lines(text) + self.assertEqual("\n".join(map(str, writes)), text) + + def test_parse(self): + self.do_test("cpu a=1", Line("cpu", {}, {'a': 1}, None)) + self.do_test('yahoo.CHFGBP\\=X.ask,tag=foobar value=10.2', + Line('yahoo.CHFGBP=X.ask', {'tag': 'foobar'}, {'value': 10.2})) + + self.assertEqual(repr(LineParser.parse('cpu,host=serverA,region=us-west foo="bar"')), + '''<Line key=cpu tags=[('host', 'serverA'), ('region', 'us-west')] fields=[('foo', 'bar')] timestamp=None>''') + + self.assertEqual(str(LineParser.parse('cpu host="serverA",region="us-west"')), + 'cpu host="serverA",region="us-west"') + + self.do_test('cpu\\,01 host="serverA",region="us-west"', + Line('cpu,01', {}, {'host': 'serverA', 'region': 'us-west'}, None)) + + self.do_test('cpu host="server A",region="us west"', + Line('cpu', {}, dict([('host', 'server A'), ('region', 'us west')]), None)) + + self.do_test('cpu ho\\=st="server A",region="us west"', + Line('cpu', {}, dict([('ho=st', 'server A'), ('region', 'us west')]), None)) + + self.assertEqual(str(LineParser.parse('cpu,ho\=st=server\ A field=123')), + 'cpu,ho\=st=server\ A field=123') + + # error: double name is accepted + self.assertEqual(str(LineParser.parse('cpu,foo=bar,foo=bar field=123,field=123')), + 'cpu,foo=bar,foo=bar field=123,field=123') + + self.assertEqual(str(LineParser.parse('cpu field12=12')), 'cpu field12=12') + self.assertEqual(str(LineParser.parse('cpu field12=12 123123123')), 'cpu field12=12 123123123') + + try: + LineParser.parse('cpu field12=12 1231abcdef123') + self.fail() + except NoParseError: + pass + + self.assertEqual(str(LineParser.parse('cpu,x=3,y=4,z=6 field\\ name="HH \\\"World",x="asdf foo"')), + 'cpu,x=3,y=4,z=6 field\\ name="HH \\"World",x="asdf foo"') + + self.assertEqual(str(LineParser.parse('cpu,x=3 field\\ name="HH \\"World",x="asdf foo"')), + 'cpu,x=3 field\\ name="HH \\"World",x="asdf foo"') + + self.do_test('cpu foo="bar" 12345', Line('cpu', {}, {'foo': 'bar'}, 12345)) + + self.do_test(r'"measurement\ with\ quotes" foo=1', + Line('"measurement with quotes"', {}, {'foo': 1}, None)) + + self.do_test(r'a$b,cp="asdf" value="fo \\ o\""', + Line("a$b", {'cp': '"asdf"'}, {'value': r'fo \ o"'})) + self.assertEqualLine(LineParser.parse(r'a$b,cp="asdf" value="fo \ o\""'), + Line("a$b", {'cp': '"asdf"'}, {'value': r'fo \ o"'})) + self.assertEqual(str(Line("a$b", {'cp': '"asdf"'}, {'value': r'fo \ o"'})), + r'a$b,cp="asdf" value="fo \\ o\""') + + self.assertEqualLine(LineParser.parse(r'test value="7\\\""'), + Line('test', {}, {'value': r'7\"'}, None)) + + self.do_test('K.5S,Ccpvo=a\\ b value=1', Line('K.5S', {'Ccpvo': 'a b'}, {'value': 1})) + + self.assertEqualLine(LineParser.parse('foo field1=f,field2=false,field3=False,field4=FALSE,field5="fag"'), + Line('foo', {}, + {'field4': False, 'field3': False, 'field2': False, 'field1': False, 'field5': 'fag'}, + None)) + + self.assertEqualLine(LineParser.parse('foo field0="tag",field1=t,field2=true,field3=True,field4=TRUE'), + Line('foo', {}, + {'field4': True, 'field0': 'tag', 'field3': True, 'field2': True, 'field1': True}, + None)) + self.assertEqual(str(LineParser.parse('foo field0="tag",field1=t,field2=true,field3=True,field4=TRUE')), + 'foo field0="tag",field1=True,field2=True,field3=True,field4=True') + + self.assertEqual(str(LineParser.parse('foo,foo=2 field_key="string\\" field"')), + 'foo,foo=2 field_key="string\\" field"') + + self.assertEqual(str(LineParser.parse('foo,foo=2 field_key\\\\="string field"')), + 'foo,foo=2 field_key\\\\="string field"') + + self.assertEqual(str(LineParser.parse('foo,foo=2 "field key with space"="string field"')), + 'foo,foo=2 field\ key\ with\ space="string field"') + + self.assertEqualLine(LineParser.parse( + r'disk_free value=442221834240,working\ directories="C:\My Documents\Stuff for examples,C:\My Documents" 123'), + Line('disk_free', {}, {'value': 442221834240, + 'working directories': r'C:\My Documents\Stuff for examples,C:\My Documents'}, 123)) + + self.assertEqualLine(LineParser.parse( + r'disk_free value=442221834240,working\ directories="C:\My Documents\Stuff for examples,C:\My Documents"'), + Line('disk_free', {}, {'value': 442221834240, + 'working directories': r'C:\My Documents\Stuff for examples,C:\My Documents'}, + None)) + + self.assertEqualLine(LineParser.parse( + r'"measurement\ with\ quotes",tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\="string field value, only \" need be quoted"'), + Line("measurement with quotes", + {'tag key with spaces': 'tag,value,with"commas"'}, + {'field_key\\': 'string field value, only " need be quoted'}, None)) + self.assertEqual(str(Line("measurement with quotes", {'tag key with spaces': 'tag,value,with"commas"'}, + {'field_key\\': 'string field value, only " need be quoted'}, None)), + r'measurement\ with\ quotes,tag\ key\ with\ spaces=tag\,value\,with"commas" field_key\\="string field value, only \" need be quoted"') @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -from distutils.core import setup +from setuptools import setup version = '0.1' @@ -12,6 +12,7 @@ setup(name='pyinflux', packages=['pyinflux.client', 'pyinflux.parser'], url='https://github.com/yvesf/pyinflux', install_requires=[], + tests_require=['funcparserlib==0.3.6'], extras_require={'parser': ['funcparserlib==0.3.6']}, classifiers=[ "Programming Language :: Python", @@ -1,7 +1,6 @@ #!/usr/bin/env python3 -from doctest import testmod -from pyinflux import client, parser +import unittest +from pyinflux import client, parser, test if __name__ == '__main__': - testmod(m=client) - testmod(m=parser) + unittest.TextTestRunner().run(unittest.findTestCases(test))
\ No newline at end of file |