From 08b882f4cf284bed3f8f80f6e47d37037753a22b Mon Sep 17 00:00:00 2001 From: Yves Fischer Date: Mon, 25 Jan 2016 22:14:26 +0100 Subject: commit --- .../yvesf/classifieds/ClassifiedParserMain.groovy | 87 ++++++++++++++++++ .../org/xapek/yvesf/classifieds/Dumper.groovy | 25 +++++ .../org/xapek/yvesf/classifieds/Model.groovy | 102 +++++++++++++++++++++ .../org/xapek/yvesf/classifieds/util/Fail.groovy | 40 ++++++++ .../xapek/yvesf/classifieds/util/Failable.groovy | 14 +++ .../xapek/yvesf/classifieds/util/Success.groovy | 36 ++++++++ 6 files changed, 304 insertions(+) create mode 100644 src/main/groovy/org/xapek/yvesf/classifieds/ClassifiedParserMain.groovy create mode 100644 src/main/groovy/org/xapek/yvesf/classifieds/Dumper.groovy create mode 100644 src/main/groovy/org/xapek/yvesf/classifieds/Model.groovy create mode 100644 src/main/groovy/org/xapek/yvesf/classifieds/util/Fail.groovy create mode 100644 src/main/groovy/org/xapek/yvesf/classifieds/util/Failable.groovy create mode 100644 src/main/groovy/org/xapek/yvesf/classifieds/util/Success.groovy (limited to 'src') diff --git a/src/main/groovy/org/xapek/yvesf/classifieds/ClassifiedParserMain.groovy b/src/main/groovy/org/xapek/yvesf/classifieds/ClassifiedParserMain.groovy new file mode 100644 index 0000000..499a37a --- /dev/null +++ b/src/main/groovy/org/xapek/yvesf/classifieds/ClassifiedParserMain.groovy @@ -0,0 +1,87 @@ +package org.xapek.yvesf.classifieds + +import groovy.json.JsonSlurper +import groovyx.net.http.HTTPBuilder +import org.apache.http.HttpResponse +import org.xapek.yvesf.classifieds.util.Fail +import org.xapek.yvesf.classifieds.util.Failable +import org.xapek.yvesf.classifieds.util.Success + +class ClassifiedParserMain { + static final url = 'http://www.glocals.com/classifieds/housing-and-real-estate/&get_classified_flats' + + static void main(String[] args) { + handleData(args.length == 1 ? readJsonFile(args.first()) : readNetwork()).with { + if (it instanceof Fail) { + println "Failed: ${it.error}" + System.exit(1) + } else { + Dumper.dump(it.value, new PrintWriter(System.out)) + System.exit(0) + } + } + } + + static Object readJsonFile(String f) { + final jsonParser = new JsonSlurper() + return jsonParser.parse(new File(f)) + } + + static Object readNetwork() { + final http = new HTTPBuilder(url) + final jsonParser = new JsonSlurper() + http.post( + body: ['start': '20', 'limit': '20', 'form[bl_city_network]': '-1'], + headers: ['User-Agent': 'Mozilla/5.0 Firefox/3.0.4', + 'Accept' : 'application/json']) { HttpResponse resp -> + assert resp.statusLine.statusCode < 300 && resp.statusLine.statusCode >= 200: + "HTTP Request failed with status code ${resp.statusLine.statusCode}" + return jsonParser.parse(resp.entity.content) + } as Object + } + + private static Failable handleData(Object p) { + expectMap(p) << { Map m -> + getOrFail(m, 'totalCount', Number) << { Number totalCount -> + getOrFail(m, 'classifieds', List) << { List rawClassifieds -> + all(rawClassifieds.collect { handleClassified(it) }) << { List classifieds -> + final Model.ClassifiedList classifiedList = new Model.ClassifiedList(totalCount: totalCount) + classifiedList.addAll(classifieds) + return new Success(classifiedList) + } + } + } + } + } + + private static Failable handleClassified(Object p) { + expectMap(p) << { Map map -> + return new Success(new Model.Classified(map)) // here I cheat a bit + } + } + + static Failable> all(Collection> values) { + if (values.every { it.success }) { + return new Success>(values.collect { it.value }) + } else { + return new Fail("Not all values are successful, first error: ${values.find { !it.success }.error}") + } + } + + static Failable expectMap(Object data) { + if (data instanceof Map) return new Success(data) + else return new Fail('object is not instanceof Map') + } + + static Failable getOrFail(Map map, String key, Class clazz = Object) { + if (!map.containsKey(key)) return new Fail('key not in map') + else { + final x = map.get(key) + if (clazz.isAssignableFrom(x.getClass())) { + return new Success(x as T) + } else { + return new Fail("Key ${key} found but not of type ${clazz}") + } + } + } +} diff --git a/src/main/groovy/org/xapek/yvesf/classifieds/Dumper.groovy b/src/main/groovy/org/xapek/yvesf/classifieds/Dumper.groovy new file mode 100644 index 0000000..884166d --- /dev/null +++ b/src/main/groovy/org/xapek/yvesf/classifieds/Dumper.groovy @@ -0,0 +1,25 @@ +package org.xapek.yvesf.classifieds + +import groovy.xml.MarkupBuilder +import org.xapek.yvesf.classifieds.Model.ClassifiedList + + +class Dumper { + static dump(ClassifiedList classifiedList, PrintWriter writer) { + new MarkupBuilder(writer).root { + rss(version: '2.0') { + channel { + title('Glocals') + } + classifiedList.each { Model.Classified classified -> + item { + title("${classified.type} - ${classified.title} - ${classified.composedLocation}") + guid(classified.id) + author(classified.memberName) + description(classified.description) + } + } + } + } + } +} diff --git a/src/main/groovy/org/xapek/yvesf/classifieds/Model.groovy b/src/main/groovy/org/xapek/yvesf/classifieds/Model.groovy new file mode 100644 index 0000000..ee0b11d --- /dev/null +++ b/src/main/groovy/org/xapek/yvesf/classifieds/Model.groovy @@ -0,0 +1,102 @@ +package org.xapek.yvesf.classifieds + +class Model { + static class Classified { + //0 = available" -> "Flexible" + //1 = city" -> "France" + //2 = column_headings" -> "0" + //3 = contact" -> "0795354458" + //4 = currency" -> "CHF" + //5 = date" -> "Jan 25, 16" + //6 = description" -> "Fully furnished house of 270 sqm in total.
\r\n
\r\nVery nice and big house of 270sqm (included 90 sqm2 of basement) with 5 bedrooms, 2 showerooms,1 bathroom and a guest toilet. 70 sqm of closed garage. Wine cellar. Lots of storage.
\r\n
\r\nBig living/dining room with fireplace. Big kitchen all equipped.
\r\n
\r\nLaundry area and storage room.
\r\n
\r\n1000 sqm of land.
\r\n
\r\nLocated in Valleiry 20 minutes from Geneva.
\r\n
\r\n3'000 €
\r\n
\r\nCan be rent unfurnished TBD
\r\n
\r\nNo agencies, thanks!" + //7 = expired" -> "0" + //8 = id" -> "67199" + //9 = location" -> "Valleiry" + //10 ="mark" -> "Mark / Save Ad" + //11 ="mem_first_name" -> "petitemanga" + //12 ="mem_id" -> "16088" + //13 ="mem_link" -> "javascript:open_login_popup();" + //14 ="mem_name" -> "petitemanga" + //15 ="mem_name_js" -> "petitemanga" + //16 ="mem_photo" -> "http://cdn.glocals.com/sites/glocals/_static_media/public/members/empty54.gif" + //17 ="network" -> "Geneva" + //18 ="photo1" -> "/_media/board_flat1/67/67199_bl_photo_eedba.jpg" + //19 ="photo2" -> "/_media/board_flat1/67/67199_bl_photo2_58cc1.jpg" + //20 ="photo3" -> "/_media/board_flat1/67/67199_bl_photo3_a5502.jpg" + //21 ="photo4" -> "/_media/board_flat1/67/67199_bl_photo4_63653.jpg" + //22 ="photos" -> "1" + //23 ="price" -> "3000" + //24 ="rooms" -> "7" + //25 ="status" -> "null" + //26 ="title" -> "Fully furnished house of 270 sqm and huge terrace of 90 sqm" + //27 ="title_js" -> "Fully furnished house of 270 sqm and huge terrace of 90 sqm" + //28 ="type" -> "Apts / Housing for rent" + //29 ="views" -> "8" + private Map map + + Classified(Map m) { + this.@map = m + } + + Map getMap() { + return map + } + + String getComposedLocation() { + return "${map.get('location')}, ${map.get('city')}" + } + + String getType() { + return map.get('type') as String + } + + String getAvailable() { + return map.get('available') as String + } + + String getCity() { + return map.get('city') + } + + String getPrice() { + return map.get('price') as String + } + + String getCurrency() { + return map.get('currency') as String + } + + String getViews() { + return map.get('views') as String + } + + String getTitle() { + return map.get('title') as String + } + + String getId() { + return map.get('id') as String + } + + String getMemberName() { + return map.get('mem_name') as String + } + + String getDescription() { + return map.get('description') as String + } + } + + static class ClassifiedList extends ArrayList { + private long totalCount + + void setTotalCount(long totalCount) { + this.@totalCount = totalCount + } + + @Override + String toString() { + return "Classified List: totalCount=${totalCount} actualCount=${size()}" + } + } +} diff --git a/src/main/groovy/org/xapek/yvesf/classifieds/util/Fail.groovy b/src/main/groovy/org/xapek/yvesf/classifieds/util/Fail.groovy new file mode 100644 index 0000000..c58d6ef --- /dev/null +++ b/src/main/groovy/org/xapek/yvesf/classifieds/util/Fail.groovy @@ -0,0 +1,40 @@ +package org.xapek.yvesf.classifieds.util + +import groovy.transform.CompileStatic +import groovy.transform.ToString + + +@CompileStatic +@Newify +@ToString(includeFields = true, includeNames = true) +class Fail implements Failable { + private String error + + Fail() { + this.@error = null + } + + Fail(String error) { + this.@error = error; + } + + @Override + boolean isSuccess() { + return false + } + + @Override + Object getValue() { + return null + } + + @Override + String getError() { + return error + } + + @Override + def Failable leftShift(Closure> closure) { + return null + } +} diff --git a/src/main/groovy/org/xapek/yvesf/classifieds/util/Failable.groovy b/src/main/groovy/org/xapek/yvesf/classifieds/util/Failable.groovy new file mode 100644 index 0000000..4ad5e90 --- /dev/null +++ b/src/main/groovy/org/xapek/yvesf/classifieds/util/Failable.groovy @@ -0,0 +1,14 @@ +package org.xapek.yvesf.classifieds.util + +import groovy.transform.CompileStatic + +@CompileStatic +interface Failable { + boolean isSuccess() + + T getValue() + + String getError() + + public Failable leftShift(Closure> closure); +} \ No newline at end of file diff --git a/src/main/groovy/org/xapek/yvesf/classifieds/util/Success.groovy b/src/main/groovy/org/xapek/yvesf/classifieds/util/Success.groovy new file mode 100644 index 0000000..224909d --- /dev/null +++ b/src/main/groovy/org/xapek/yvesf/classifieds/util/Success.groovy @@ -0,0 +1,36 @@ +package org.xapek.yvesf.classifieds.util + +import groovy.transform.CompileStatic +import groovy.transform.ToString + + +@CompileStatic +@Newify +@ToString(includeFields = true, includeNames = true) +class Success implements Failable { + private T value + + public Success(T value) { + this.value = value + } + + @Override + boolean isSuccess() { + return true + } + + @Override + T getValue() { + return value + } + + @Override + String getError() { + return null + } + + @Override + def Failable leftShift(Closure> closure) { + return closure.call(value) + } +} \ No newline at end of file -- cgit v1.2.1