From 63c559ec9027aea94ce66c2a4d0c1d1c4be4dcda Mon Sep 17 00:00:00 2001 From: Rik Veenboer Date: Sun, 30 Nov 2014 12:23:51 +0000 Subject: [PATCH] Initial commit --- .gitignore | 7 + build.gradle | 31 ++ evalsha.txt | 10 + src/main/java/data/Entry.java | 13 + src/main/java/interval/Day.java | 19 + src/main/java/interval/Hour.java | 20 ++ src/main/java/interval/Hour12.java | 7 + src/main/java/interval/Hour2.java | 8 + src/main/java/interval/Hour4.java | 7 + src/main/java/interval/Hour6.java | 7 + src/main/java/interval/Interval.java | 54 +++ src/main/java/interval/Minute.java | 18 + src/main/java/interval/Minute15.java | 7 + src/main/java/interval/Minute30.java | 7 + src/main/java/interval/Month.java | 18 + src/main/java/interval/Week.java | 18 + src/main/java/model/Model.java | 306 ++++++++++++++++ src/main/java/util/DateUtils.java | 44 +++ src/main/java/util/FileUtils.java | 84 +++++ src/main/java/util/LuaUtils.java | 190 ++++++++++ .../java/util/ReverseLineInputStream.java | 68 ++++ src/main/java/util/StringUtils.java | 19 + src/main/java/util/TypeUtils.java | 17 + src/main/lua/anon.lua | 5 + src/main/lua/bit/todo/bit.lua | 260 ++++++++++++++ src/main/lua/bit/todo/hex.lua | 95 +++++ src/main/lua/bit/todo/noki.lua | 332 ++++++++++++++++++ src/main/lua/bit/todo/readme.txt | 143 ++++++++ src/main/lua/bit/todo/utf8.lua | 163 +++++++++ src/main/lua/getall.lua | 6 + src/main/lua/getsome.lua | 2 + src/main/lua/getsomex.lua | 11 + src/main/lua/insert.lua | 18 + src/main/lua/list/slice.lua | 8 + src/main/lua/msgpack.lua | 3 + src/main/lua/returnint.lua | 1 + src/main/lua/returnnil.lua | 1 + src/main/lua/returnset.lua | 1 + src/main/lua/returnstring.lua | 1 + src/main/lua/test1.lua | 2 + src/main/lua/test2.lua | 2 + src/main/lua/test3.lua | 2 + src/main/lua/util/bla.lua | 3 + src/main/lua/util/getsome.lua | 9 + src/main/resources/flight.txt | 7 + src/main/resources/pvoutput.debug | 10 + src/test/java/test/Flight.java | 99 ++++++ src/test/java/test/Relational.java | 144 ++++++++ src/test/java/test/Test1.java | 36 ++ src/test/java/test/Test2.java | 29 ++ src/test/java/test/Test3.java | 44 +++ src/test/java/test/Test4.java | 29 ++ src/test/java/test/Test5.java | 46 +++ src/test/java/test/Test6.java | 27 ++ src/test/java/test/Test7.java | 17 + .../META-INF/services/interval.Interval | 8 + 56 files changed, 2543 insertions(+) create mode 100644 .gitignore create mode 100644 build.gradle create mode 100644 evalsha.txt create mode 100644 src/main/java/data/Entry.java create mode 100644 src/main/java/interval/Day.java create mode 100644 src/main/java/interval/Hour.java create mode 100644 src/main/java/interval/Hour12.java create mode 100644 src/main/java/interval/Hour2.java create mode 100644 src/main/java/interval/Hour4.java create mode 100644 src/main/java/interval/Hour6.java create mode 100644 src/main/java/interval/Interval.java create mode 100644 src/main/java/interval/Minute.java create mode 100644 src/main/java/interval/Minute15.java create mode 100644 src/main/java/interval/Minute30.java create mode 100644 src/main/java/interval/Month.java create mode 100644 src/main/java/interval/Week.java create mode 100644 src/main/java/model/Model.java create mode 100644 src/main/java/util/DateUtils.java create mode 100644 src/main/java/util/FileUtils.java create mode 100644 src/main/java/util/LuaUtils.java create mode 100644 src/main/java/util/ReverseLineInputStream.java create mode 100644 src/main/java/util/StringUtils.java create mode 100644 src/main/java/util/TypeUtils.java create mode 100644 src/main/lua/anon.lua create mode 100644 src/main/lua/bit/todo/bit.lua create mode 100644 src/main/lua/bit/todo/hex.lua create mode 100644 src/main/lua/bit/todo/noki.lua create mode 100644 src/main/lua/bit/todo/readme.txt create mode 100644 src/main/lua/bit/todo/utf8.lua create mode 100644 src/main/lua/getall.lua create mode 100644 src/main/lua/getsome.lua create mode 100644 src/main/lua/getsomex.lua create mode 100644 src/main/lua/insert.lua create mode 100644 src/main/lua/list/slice.lua create mode 100644 src/main/lua/msgpack.lua create mode 100644 src/main/lua/returnint.lua create mode 100644 src/main/lua/returnnil.lua create mode 100644 src/main/lua/returnset.lua create mode 100644 src/main/lua/returnstring.lua create mode 100644 src/main/lua/test1.lua create mode 100644 src/main/lua/test2.lua create mode 100644 src/main/lua/test3.lua create mode 100644 src/main/lua/util/bla.lua create mode 100644 src/main/lua/util/getsome.lua create mode 100644 src/main/resources/flight.txt create mode 100644 src/main/resources/pvoutput.debug create mode 100644 src/test/java/test/Flight.java create mode 100644 src/test/java/test/Relational.java create mode 100644 src/test/java/test/Test1.java create mode 100644 src/test/java/test/Test2.java create mode 100644 src/test/java/test/Test3.java create mode 100644 src/test/java/test/Test4.java create mode 100644 src/test/java/test/Test5.java create mode 100644 src/test/java/test/Test6.java create mode 100644 src/test/java/test/Test7.java create mode 100644 src/test/resources/META-INF/services/interval.Interval diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..872e67b --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.gradle +.settings +.classpath +.project +bin +build + diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..512da66 --- /dev/null +++ b/build.gradle @@ -0,0 +1,31 @@ +apply plugin: 'java' +apply plugin: 'eclipse' + +repositories { + maven { + url 'https://github.com/Boukefalos/jlibredis/raw/mvn-repo/' + } + mavenCentral() +} + +dependencies { + compile 'redis.clients:jedis:2.6.1' + //compile 'com.github.boukefalos:jlibredis:2.6.2' + + compile 'com.google.guava:guava:18.0' + compile 'com.googlecode.json-simple:json-simple:1.1.1' + compile 'commons-lang:commons-lang:2.6' + compile 'commons-codec:commons-codec:1.9' + compile 'commons-io:commons-io:2.4' + compile 'org.msgpack:msgpack:0.6.11' + + testCompile 'junit:junit:4.+' +} + +sourceSets { + main { + resources { + srcDir 'src/main/lua' + } + } +} \ No newline at end of file diff --git a/evalsha.txt b/evalsha.txt new file mode 100644 index 0000000..30cb4f6 --- /dev/null +++ b/evalsha.txt @@ -0,0 +1,10 @@ +http://evalsha.com/scripts/53cbfe7cfa82734f5a727c5dff280b10c8866817 +http://evalsha.com/scripts/71927ba3cf040801b6a3b3d7081191d403c8d110 +http://evalsha.com/scripts/0c3d9163a97c3cf7605e2526d2ed2c0806f32097 +http://evalsha.com/scripts/140053a7dc99e7d4f095f8723befa934dbc3430a +http://evalsha.com/scripts/00c37234520b7295684ccc128636b2413132684a +http://evalsha.com/scripts/d639c8a2713f88d88d3ad6900e05a64a99aa3ead +http://evalsha.com/scripts/67e165f02e2830b2450acba0fae439a704edb515 +http://evalsha.com/scripts/1c63c9ce6e50bf11c24551bc041b88193b9ad9ec +http://evalsha.com/scripts/8f5250cf6bbdadd57cd092f34426f2bbcea584e5 +http://evalsha.com/scripts/7e8b41429c241e55a96cb1707ef9d8be38e1fba2 \ No newline at end of file diff --git a/src/main/java/data/Entry.java b/src/main/java/data/Entry.java new file mode 100644 index 0000000..bf6e90a --- /dev/null +++ b/src/main/java/data/Entry.java @@ -0,0 +1,13 @@ +package data; + +import java.util.Date; + +public class Entry { + public Date date; + public double value; + + public Entry(Date date, double value) { + this.date = date; + this.value = value; + } +} \ No newline at end of file diff --git a/src/main/java/interval/Day.java b/src/main/java/interval/Day.java new file mode 100644 index 0000000..9d4a8fc --- /dev/null +++ b/src/main/java/interval/Day.java @@ -0,0 +1,19 @@ +package interval; + +import java.util.Calendar; +import java.util.Date; + +public class Day extends Minute { + + public Day() {} + + public Date next() { + calendar.add(Calendar.DAY_OF_MONTH, 1); + return get(); + } + + protected void adjust() { + super.adjust(); + calendar.set(Calendar.HOUR_OF_DAY, 0); + } +} diff --git a/src/main/java/interval/Hour.java b/src/main/java/interval/Hour.java new file mode 100644 index 0000000..ef5939f --- /dev/null +++ b/src/main/java/interval/Hour.java @@ -0,0 +1,20 @@ +package interval; + +import java.util.Calendar; +import java.util.Date; + +public class Hour extends Minute { + public static final int HOURS = 1; + + public Hour() {} + + public Date next() { + calendar.add(Calendar.HOUR, HOURS); + return get(); + } + + protected void adjust() { + super.adjust(); + calendar.set(Calendar.MINUTE, 0); + } +} diff --git a/src/main/java/interval/Hour12.java b/src/main/java/interval/Hour12.java new file mode 100644 index 0000000..64e38a6 --- /dev/null +++ b/src/main/java/interval/Hour12.java @@ -0,0 +1,7 @@ +package interval; + +public class Hour12 extends Hour { + public static final int HOURS = 12; + + public Hour12() {} +} diff --git a/src/main/java/interval/Hour2.java b/src/main/java/interval/Hour2.java new file mode 100644 index 0000000..532a33a --- /dev/null +++ b/src/main/java/interval/Hour2.java @@ -0,0 +1,8 @@ +package interval; + + +public class Hour2 extends Hour { + public static final int HOURS = 2; + + public Hour2() {} +} diff --git a/src/main/java/interval/Hour4.java b/src/main/java/interval/Hour4.java new file mode 100644 index 0000000..4c0cb40 --- /dev/null +++ b/src/main/java/interval/Hour4.java @@ -0,0 +1,7 @@ +package interval; + +public class Hour4 extends Hour { + public static final int HOURS = 4; + + public Hour4() {} +} diff --git a/src/main/java/interval/Hour6.java b/src/main/java/interval/Hour6.java new file mode 100644 index 0000000..ff90496 --- /dev/null +++ b/src/main/java/interval/Hour6.java @@ -0,0 +1,7 @@ +package interval; + +public class Hour6 extends Hour { + public static final int HOURS = 6; + + public Hour6() {} +} diff --git a/src/main/java/interval/Interval.java b/src/main/java/interval/Interval.java new file mode 100644 index 0000000..db4fb38 --- /dev/null +++ b/src/main/java/interval/Interval.java @@ -0,0 +1,54 @@ +package interval; + +import java.util.Calendar; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.Iterator; +import java.util.TimeZone; + +public abstract class Interval implements Iterator { + public static final String TIMEZONE = "Europe/Amsterdam"; + + protected Date endDate; + protected TimeZone timeZone; + protected Calendar calendar; + + public Interval() {} + + public Interval(Date startDate, Date endDate, String timeZoneID) { + setTimeZone(timeZoneID); + setDate(startDate, endDate); + } + + public void setTimeZone(String timeZoneID) { + timeZone = TimeZone.getTimeZone(timeZoneID); + calendar = new GregorianCalendar(TimeZone.getTimeZone(TIMEZONE)); + } + + public Interval(Date startDate, Date endDate) { + this(startDate, endDate, TIMEZONE); + } + + public void setDate(Date startDate, Date endDate) { + if (timeZone == null) { + setTimeZone(TIMEZONE); + } + calendar.setTime(startDate); + adjust(); + this.endDate = endDate; + } + + public void remove() {} + + public Date get() { + return calendar.getTime(); + } + + public abstract Date next(); + + public boolean hasNext() { + return next().compareTo(endDate) < 0; + } + + protected abstract void adjust(); +} diff --git a/src/main/java/interval/Minute.java b/src/main/java/interval/Minute.java new file mode 100644 index 0000000..e935ccd --- /dev/null +++ b/src/main/java/interval/Minute.java @@ -0,0 +1,18 @@ +package interval; + +import java.util.Calendar; +import java.util.Date; + +public class Minute extends Interval { + public static final int MINUTES = 1; + + public Date next() { + calendar.add(Calendar.MINUTE, MINUTES); + return get(); + } + + protected void adjust() { + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MILLISECOND, 0); + } +} diff --git a/src/main/java/interval/Minute15.java b/src/main/java/interval/Minute15.java new file mode 100644 index 0000000..f9b0148 --- /dev/null +++ b/src/main/java/interval/Minute15.java @@ -0,0 +1,7 @@ +package interval; + +public class Minute15 extends Minute { + public static final int MINUTES = 15; + + public Minute15() {} +} diff --git a/src/main/java/interval/Minute30.java b/src/main/java/interval/Minute30.java new file mode 100644 index 0000000..53b075f --- /dev/null +++ b/src/main/java/interval/Minute30.java @@ -0,0 +1,7 @@ +package interval; + +public class Minute30 extends Minute { + public static final int MINUTES = 30; + + public Minute30() {} +} diff --git a/src/main/java/interval/Month.java b/src/main/java/interval/Month.java new file mode 100644 index 0000000..53cf707 --- /dev/null +++ b/src/main/java/interval/Month.java @@ -0,0 +1,18 @@ +package interval; + +import java.util.Date; +import java.util.Calendar; + +public class Month extends Day { + public Month() {} + + public Date next() { + calendar.add(Calendar.MONTH, 1); + return get(); + } + + protected void adjust() { + super.adjust(); + calendar.set(Calendar.DAY_OF_MONTH, 1); + } +} diff --git a/src/main/java/interval/Week.java b/src/main/java/interval/Week.java new file mode 100644 index 0000000..ddc2829 --- /dev/null +++ b/src/main/java/interval/Week.java @@ -0,0 +1,18 @@ +package interval; + +import java.util.Calendar; +import java.util.Date; + +public class Week extends Day { + public Week() {} + + public Date next() { + calendar.add(Calendar.WEEK_OF_MONTH, 1); + return get(); + } + + protected void adjust() { + super.adjust(); + calendar.set(Calendar.DAY_OF_WEEK, 2); + } +} diff --git a/src/main/java/model/Model.java b/src/main/java/model/Model.java new file mode 100644 index 0000000..6eb0c18 --- /dev/null +++ b/src/main/java/model/Model.java @@ -0,0 +1,306 @@ +package model; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.json.simple.JSONArray; + +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.SortingParams; +import redis.clients.jedis.Transaction; +import redis.clients.util.SafeEncoder; +import util.DateUtils; +import util.StringUtils; +import data.Entry; + +public class Model { + public static final int BATCH_SIZE = 10; + + protected Calendar calendar; + + protected JedisPool pool; + protected Jedis jedis; + protected Transaction transaction; + + protected int insertDataBatchCount, insertExtremesBatchStatementCount; + protected PreparedStatement insertDataBatchStatement, insertExtremesBatchStatement; + + //private Pipeline pipeline; + + public Model(Calendar calendar) { + this.calendar = calendar; + pool = new JedisPool("localhost", 16379); + jedis = pool.getResource(); + jedis.select(1); + //clear(); + } + + public void clear() { + Set set = jedis.keys("*"); + if (set.size() > 0) { + jedis.del(set.toArray(new String[0])); + } + } + + public void createDataTable() throws SQLException { + } + + public void createIntervalsTable() throws SQLException { + } + + public void createExtremesTable() throws SQLException { + } + + public Date getFirstEntryDate() throws SQLException { + List list = jedis.sort("data", new SortingParams().limit(1, 1)); + long timestamp = 0; + if (list.size() > 0) { + timestamp = Long.valueOf(list.get(0)); + } + calendar.setTimeInMillis(1000 * timestamp); + return calendar.getTime(); + } + + public Date getLastEntryDate() throws SQLException { + List list = jedis.sort("data", new SortingParams().limit(1, 1).desc()); + long timestamp = 0; + if (list.size() > 0) { + timestamp = Long.valueOf(list.get(0)); + } + calendar.setTimeInMillis(1000 * timestamp); + return calendar.getTime(); + } + + public void insertInterval(String name) throws SQLException { + Long id = jedis.incr("global:next-interval-id"); + String parameter = StringUtils.parameterize(name); + jedis.set(String.format("interval:%d:name", id), name); + jedis.set(String.format("interval:%s:id", parameter), String.valueOf(id)); + } + + public int selectIntervalId(String name) throws Exception { + String parameter = StringUtils.parameterize(name); + return Integer.valueOf(jedis.get(String.format("interval:%s:id", parameter))); + } + + public ArrayList selectDataBetween(Date intervalStartDate, Date intervalEndDate) { +// PreparedStatement selectStatement = connection.prepareStatement("select * from data where date between ? and ?"); +// selectStatement.setLong(1, intervalStartDate.getTime() / 1000); +// selectStatement.setLong(2, intervalEndDate.getTime() / 1000); +// ResultSet resultSet = selectStatement.executeQuery(); + long min = intervalStartDate.getTime() / 1000; + long max = intervalEndDate.getTime() / 1000; + String key = String.format("data:%s:%s", min, max); + + Set dateSet = jedis.zrangeByScore("data", min, max); + + String[] dateArray = dateSet.toArray(new String[0]); + ArrayList entryList = new ArrayList(); + if (dateArray.length == 0) { + return entryList; + } + jedis.sadd(SafeEncoder.encode(key), SafeEncoder.encodeMany(dateArray)); + List valueList = jedis.sort(key, new SortingParams().nosort().get("data:*:value")); + + Iterator iterator = dateSet.iterator(); + int i = 0; + while (iterator.hasNext()) { + calendar.setTimeInMillis(1000 * Long.valueOf(iterator.next())); + entryList.add(new Entry(calendar.getTime(), Double.valueOf(valueList.get(i++)))); + } + return entryList; + } + + public void insertDataBatch(long date, double value) throws SQLException { +// if (pipeline == null) { +// pipeline = jedis.pipelined(); +// pipeline.multi(); +// } +// if (insertDataBatchStatement == null) { +// insertDataBatchStatement = connection.prepareStatement("insert or ignore into data values(?, ?)"); +// insertDataBatchCount = 0; +// } +// setDate(insertDataBatchStatement, 1, date); +// insertDataBatchStatement.setDouble(2, value); +// insertDataBatchStatement.addBatch(); +// +// if(++insertDataBatchCount % BATCH_SIZE == 0) { +// insertDataBatchStatement.executeBatch(); +// System.out.println(insertDataBatchCount); +// } + date /= 1000; + jedis.zadd(String.format("data"), date, String.valueOf(date)); + jedis.set(String.format("data:%s:value", date), String.valueOf(value)); + } + + public void insertDataBatchLast() throws SQLException { + //pipeline.exec(); + //pipeline = null; + } + + public void insertExtremesBatch(int id, Date date, Date firstDate, Date lastDate, int count, Double min, Double max) throws SQLException { +// if (pipeline == null) { +// pipeline = jedis.pipelined(); +// pipeline.multi(); +// } + +// HashMap map = new HashMap(); +// map.put("first", String.valueOf(firstDate.getTime() / 1000)); +// map.put("last", String.valueOf(lastDate.getTime() / 1000)); +// map.put("count", String.valueOf(count)); +// map.put("min", String.valueOf(min)); +// map.put("max", String.valueOf(max)); + + int dateInt = (int) date.getTime() / 1000; + + + String key = String.format("extreme:%d:%d:", id, dateInt); + jedis.set(key + "first", String.valueOf(firstDate.getTime() / 1000)); + jedis.set(key + "last", String.valueOf(lastDate.getTime() / 1000)); + jedis.set(key + "count", String.valueOf(count)); + jedis.set(key + "min", String.valueOf(min)); + jedis.set(key + "max", String.valueOf(max)); + + + jedis.zadd(String.format("interval:%d", id), dateInt, String.valueOf(dateInt)); + + //jedis.hmset(String.format("extreme:%d:%d", id, date.getTime() / 1000), map); + + if(++insertExtremesBatchStatementCount % BATCH_SIZE == 0) { + //pipeline.exec(); + //pipeline.multi(); + System.out.println(insertExtremesBatchStatementCount); + } + } + + public void insertExtremesBatchLast() throws SQLException { + //pipeline.exec(); + //pipeline = null; + } + + protected void listData() throws SQLException { +// ResultSet resultSet = statement.executeQuery("select *,datetime(date, 'unixepoch') as fmt from data"); +// while (resultSet.next()) { +// System.out.println("--------------------------"); +// System.out.println("date = " + resultSet.getString("date")); +// System.out.println("value = " + resultSet.getFloat("value")); +// System.out.println("format = " + resultSet.getString("fmt")); +// } + } + + protected String getField(ResultSet resultSet, String columnLabel) throws SQLException { + return resultSet.getString(columnLabel); + } + + protected void printField(ResultSet resultSet, String columnLabel) throws SQLException { + System.out.printf("%s = %s\n", columnLabel, resultSet.getString(columnLabel)); + } + + protected void setDate(PreparedStatement statement, int parameterIndex, Date date) throws SQLException { + setDate(statement, parameterIndex, date.getTime()); + } + + protected void setDate(PreparedStatement statement, int parameterIndex, long date) throws SQLException { + statement.setLong(parameterIndex, date / 1000); + } + + public ResultSet getExtremes(long startDate, long endDate, String name) throws SQLException { +// PreparedStatement selectStatement = connection.prepareStatement( +// "select extremes.date date, extremes.first date_first, extremes.last date_last, extremes.count count, extremes.min min_value, extremes.max max_value, first.value first_value, last.value last_value " + +// "from extremes " + +// "left join data first on extremes.first = first.date " + +// "left join data last on extremes.last = last.date " + +// "where interval_id = (select id from intervals where name = ?) and extremes.date between ? and ?"); +// selectStatement.setString(1, name); +// setDate(selectStatement, 2, startDate); +// setDate(selectStatement, 3, endDate); +// return selectStatement.executeQuery(); + + return null; + } + + @SuppressWarnings("unchecked") + public String getJSON(long startDate, long endDate, String name) throws SQLException { + ResultSet resultSet = getExtremes( + DateUtils.makeTimestamp(2012, 1, 1, 14, 0, 0, 0), + DateUtils.makeTimestamp(2013, 7, 1, 14, 0, 0, 0), "Day"); + JSONArray jsonList = new JSONArray(); + while (resultSet.next()) { + JSONArray jsonEntry = new JSONArray(); + jsonEntry.add(1000 * resultSet.getLong("date")); + jsonEntry.add(resultSet.getDouble("first_value")); + jsonEntry.add(resultSet.getDouble("min_value")); + jsonEntry.add(resultSet.getDouble("max_value")); + jsonEntry.add(resultSet.getDouble("last_value")); + jsonList.add(jsonEntry); + } + return jsonList.toJSONString(); + } + + + public void test(long startDate, long endDate, String name) throws SQLException { + ResultSet resultSet = getExtremes(startDate, endDate, name); + while (resultSet.next()) { + System.out.println("----------------"); + System.out.printf("# %d\n", resultSet.getRow()); + printField(resultSet, "date"); + printField(resultSet, "date_first"); + printField(resultSet, "count"); + printField(resultSet, "min_value"); + printField(resultSet, "max_value"); + printField(resultSet, "first_value"); + printField(resultSet, "last_value"); + } + } + + public void test() { + /*Transaction transaction = jedis.multi(); + Pipeline pipeline = jedis.pipelined(); + Client client = jedis.getClient();*/ + + // ZRANGEBYSCORE data 1375085460 1375088460 + // SORT data BY NOSORT GET data:*:value + + int min = 1375085460; + int max = 1375088460; + String key = String.format("data:%s:%s", min, max); + + Set set = jedis.zrangeByScore("data", min, max); + jedis.sadd(SafeEncoder.encode(key), SafeEncoder.encodeMany(set.toArray(new String[0]))); + List list = jedis.sort(key, new SortingParams().nosort().get("data:*:value")); + for (String string : list) { + System.out.println(string); + } + + Map a = jedis.hgetAll("extreme:2"); + System.out.println(a.get("date")); + //jedis.sort(date, new SortingParams().nosort().get("extreme:*")); + + + //set = jedis.zrange("interval:1", 0, -1); + + min = 0; + max = Integer.MAX_VALUE; + key = String.format("interval:1:%s:%s", min, max); + + set = jedis.zrangeByScore("interval:1", min, max); + jedis.sadd(SafeEncoder.encode(key), SafeEncoder.encodeMany(set.toArray(new String[0]))); + + list = jedis.sort(key, new SortingParams().nosort().get(new String[] { + "extreme:1:*:count", + "extreme:1:*:first"})); + //Iterator iterator = set.iterator(); + for (String string : list) { + System.out.println(string); + } + } +} diff --git a/src/main/java/util/DateUtils.java b/src/main/java/util/DateUtils.java new file mode 100644 index 0000000..8cba39d --- /dev/null +++ b/src/main/java/util/DateUtils.java @@ -0,0 +1,44 @@ +package util; + +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.TimeZone; + +public class DateUtils { + public static final String TIMEZONE = "Europe/Amsterdam"; + protected static TimeZone timeZone; + protected static Calendar calendar; + + public static void setTimeZone(String timeZone) { + DateUtils.timeZone = TimeZone.getTimeZone(timeZone); + calendar = new GregorianCalendar(DateUtils.timeZone); + } + + public static long makeTimestamp(int year, int month, int day, int hour, int minute, int second, int millisecond) { + checkCalendar(); + calendar.set(Calendar.YEAR, year); + calendar.set(Calendar.MONTH, month - 1); + calendar.set(Calendar.DATE, day); + calendar.set(Calendar.HOUR_OF_DAY, hour); + calendar.set(Calendar.MINUTE, minute); + calendar.set(Calendar.SECOND, second); + calendar.set(Calendar.MILLISECOND, millisecond); + return calendar.getTimeInMillis(); + } + + public static TimeZone getTimeZone() { + checkCalendar(); + return timeZone; + } + + public static Calendar getCalendar() { + checkCalendar(); + return calendar; + } + + protected static void checkCalendar() { + if (calendar == null ) { + setTimeZone(TIMEZONE); + } + } +} diff --git a/src/main/java/util/FileUtils.java b/src/main/java/util/FileUtils.java new file mode 100644 index 0000000..04342f7 --- /dev/null +++ b/src/main/java/util/FileUtils.java @@ -0,0 +1,84 @@ +package util; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.Scanner; + +public class FileUtils { + public static final boolean REVERSE = false; + + public static String resourceToString(String name) throws IOException { + File file = resourceToFile(name); + Scanner scanner = new Scanner(file); + String string = scanner.useDelimiter("\\A").next(); + scanner.close(); + return string; + } + + public static File resourceToFile(String name) throws IOException { + URL url = ClassLoader.getSystemClassLoader().getResource(name); + if (url != null) { + try { + return new File(url.toURI()); + } catch (URISyntaxException e) {} + } + throw new IOException(); + } + + public static BufferedReader resourceToReader(String name) throws IOException { + return resourceToReader(name, REVERSE); + } + + public static BufferedReader resourceToReader(String name, boolean reverse) throws IOException { + File file = resourceToFile(name); + return fileToReader(file, reverse); + } + + public static BufferedReader fileToBufferedReader(File file) throws FileNotFoundException { + return fileToReader(file, REVERSE); + } + + public static BufferedReader fileToReader(File file, boolean reverse) throws FileNotFoundException { + InputStream inputStream = fileToInputStream(file, reverse); + InputStreamReader inputStreamReader = new InputStreamReader(inputStream); + return new BufferedReader(inputStreamReader); + } + + public static InputStream fileToInputStream(File file) throws FileNotFoundException { + return fileToInputStream(file, REVERSE); + } + + public static InputStream fileToInputStream(File file, boolean reverse) throws FileNotFoundException { + InputStream inputStream; + if (reverse) { + inputStream = new ReverseLineInputStream(file); + } else { + inputStream = new FileInputStream(file); + } + return inputStream; + } + + public static InputStream resourceToInputStream(String name) throws IOException { + return resourceToInputStream(name, REVERSE); + } + + public static InputStream resourceToInputStream(String name, boolean reverse) throws IOException { + if (reverse) { + File file = resourceToFile(name); + return fileToInputStream(file, reverse); + } else { + return ClassLoader.getSystemClassLoader().getResourceAsStream(name); + } + } + + public static Scanner resourceToScanner(String string) throws IOException { + return new Scanner(resourceToReader(string)); + } +} diff --git a/src/main/java/util/LuaUtils.java b/src/main/java/util/LuaUtils.java new file mode 100644 index 0000000..a3b9704 --- /dev/null +++ b/src/main/java/util/LuaUtils.java @@ -0,0 +1,190 @@ +package util; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.codec.digest.DigestUtils; + + + + + + + +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.exceptions.JedisDataException; + +public class LuaUtils { + public static final boolean LOAD = true; + public static final List EMPTY = new ArrayList(); + public static final String[] LIBRARY_ARRAY = {"redis", "table", "string", "math", "debug", "cjson", "cmsgpack"}; + public static final List LIBRARY_LIST = Arrays.asList(LIBRARY_ARRAY); + + protected static JedisPool jedisPool; + protected static HashMap hashMap = new HashMap(); + + public static void setPool(JedisPool pool) { + LuaUtils.jedisPool = pool; + } + + public static Object eval(String name, List keys, List args) throws IOException { + Jedis jedis = jedisPool.getResource(); + String hash = getHash(name, jedis); + System.out.println(hash); + Object object = jedis.evalsha(hash, keys, args); + jedisPool.returnResource(jedis); + return object; + } + + public static Object eval(String name, int keys, String... args) throws IOException { + Jedis jedis = getResource(); + String hash = getHash(name, jedis); + try { + return jedis.evalsha(hash, keys, args); + } catch (JedisDataException e) { + if (e.getMessage().startsWith("NOSCRIPT")) { + // Script not loaded correctly + } + System.out.println(e); + throw e; + } finally { + returnResource(jedis); + } + } + + public static Object eval(String name) throws IOException { + return eval(name, 0); + } + + public static Object eval(String string, List params) throws IOException { + return eval(string, EMPTY, params); + } + + protected static String getHash(String name, Jedis jedis) throws IOException { + name = String.format("%s.lua", name); + if (hashMap.containsKey(name)) { + return hashMap.get(name); + } else { + String lua = FileUtils.resourceToString(name); + + Map> libraryMap = new HashMap>(); + + lua = resolveDepencencies(lua, libraryMap); + for (String library : libraryMap.keySet()) { + lua = String.format("local %s = {}\n%s", library, lua); + } + System.out.println("======"); + System.out.println(lua); + System.out.println("======"); + String hash = DigestUtils.sha1Hex(lua); + if (!jedis.scriptLoad(lua).equals(hash)) { + // Hashes don't match + } + hashMap.put(name, hash); + return hash; + } + } + + protected static String resolveDepencencies(String lua, Map> libraryMap) throws IOException { + Pattern pattern = Pattern.compile(String.format("([a-z]+)%s([a-z]+)", Pattern.quote("."))); + Matcher matcher = pattern.matcher(lua); + String depencencyLua = ""; + while (matcher.find()) { + String library = matcher.group(1); + if (!LIBRARY_LIST.contains(library)) { + if (!libraryMap.containsKey(library)) { + libraryMap.put(library, new ArrayList()); + } + List methodList = libraryMap.get(library); + String method = matcher.group(2); + if (!methodList.contains(method)) { + String file = String.format("%s/%s.lua", library, method); + System.out.println(file); + String methodLua = FileUtils.resourceToString(file); + methodList.add(method); + String subDepencencyLua = resolveDepencencies(methodLua, libraryMap); + if (depencencyLua.isEmpty()) { + depencencyLua = subDepencencyLua; + } else if (!subDepencencyLua.isEmpty()) { + depencencyLua = String.format("%s\n%s", depencencyLua, subDepencencyLua); + } + } + } + } + if (depencencyLua.isEmpty()) { + return lua; + } else { + return String.format("%s\n%s", depencencyLua, lua); + } + } + + protected static void returnResource(Jedis jedis) { + jedisPool.returnResource(jedis); + } + + protected static Jedis getResource() { + return jedisPool.getResource(); + } + + public static Object insert(String table, Map hash) throws IOException { + List params = new ArrayList(); + for (final Entry entry : hash.entrySet()) { + params.add(entry.getKey()); + params.add(entry.getValue()); + params.add(StringUtils.parameterize(entry.getValue())); + } + params.add(0, table); + return LuaUtils.eval("insert", params); + } + + @SuppressWarnings("unchecked") + public static List> getAll(String table) throws IOException { + List params = new ArrayList(); + params.add(table); + List> listList = (List>) LuaUtils.eval("getall", params); + List> mapList = new ArrayList>(); + System.out.println(listList.size()); + for (List list : listList) { + mapList.add(TypeUtils.listToMap(list)); + } + return mapList; + } + + @SuppressWarnings("unchecked") + public static List> getSome(String... stringArray) throws IOException { + List> listList = (List>) LuaUtils.eval("getsome", Arrays.asList(stringArray)); + List> mapList = new ArrayList>(); + for (List list : listList) { + Map map = new HashMap(); + for (int i = 1; i < stringArray.length; ++i) { + map.put(stringArray[i], list.get(i - 1)); + } + mapList.add(map); + } + return mapList; + } + + @SuppressWarnings("unchecked") + public static List> getSomeX(String... stringArray) throws IOException { + List list = (List) LuaUtils.eval("getsomex", Arrays.asList(stringArray)); + Iterator iterator = list.iterator(); + List> mapList = new ArrayList>(); + while (iterator.hasNext()) { + Map map = new HashMap(); + for (int i = 1; i < stringArray.length; ++i) { + map.put(stringArray[i], iterator.next()); + } + mapList.add(map); + } + return mapList; + } +} diff --git a/src/main/java/util/ReverseLineInputStream.java b/src/main/java/util/ReverseLineInputStream.java new file mode 100644 index 0000000..330b946 --- /dev/null +++ b/src/main/java/util/ReverseLineInputStream.java @@ -0,0 +1,68 @@ +package util; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; + +public class ReverseLineInputStream extends InputStream { + RandomAccessFile randomAccessFile; + + long currentLineStart = -1; + long currentLineEnd = -1; + long currentPos = -1; + long lastPosInFile = -1; + + public ReverseLineInputStream(File file) throws FileNotFoundException { + randomAccessFile = new RandomAccessFile(file, "r"); + currentLineStart = file.length(); + currentLineEnd = file.length(); + lastPosInFile = file.length() - 1; + currentPos = currentLineEnd; + } + + public void findPrevLine() throws IOException { + currentLineEnd = currentLineStart; + + // No more lines, since at the beginning of the file + if (currentLineEnd == 0) { + currentLineEnd = -1; + currentLineStart = -1; + currentPos = -1; + return; + } + + long filePointer = currentLineStart - 1; + while (true) { + // At start of file so this is the first line in the file. + if (--filePointer < 0) { + break; + } + + randomAccessFile.seek(filePointer); + int readByte = randomAccessFile.readByte(); + + // Ignore last LF in file. search back to find the previous LF. + if (readByte == 0xA && filePointer != lastPosInFile) { + break; + } + } + // Start at pointer +1, after the LF found or at 0 the start of the file. + currentLineStart = filePointer + 1; + currentPos = currentLineStart; + } + + public int read() throws IOException { + if (currentPos < currentLineEnd) { + randomAccessFile.seek(currentPos++); + int readByte = randomAccessFile.readByte(); + return readByte; + } else if (currentPos < 0) { + return -1; + } else { + findPrevLine(); + return read(); + } + } +} \ No newline at end of file diff --git a/src/main/java/util/StringUtils.java b/src/main/java/util/StringUtils.java new file mode 100644 index 0000000..5618bb8 --- /dev/null +++ b/src/main/java/util/StringUtils.java @@ -0,0 +1,19 @@ +package util; + +import com.google.common.base.CharMatcher; +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; + +public class StringUtils { + protected static final char SEPARATOR = '-'; + + public static String parameterize(String input) { + return parameterize(input, SEPARATOR); + } + + public static String parameterize(String input, char separator) { + Splitter splitter = Splitter.on(CharMatcher.JAVA_LETTER_OR_DIGIT.negate()).omitEmptyStrings(); + Joiner joiner = Joiner.on(separator); + return joiner.join(splitter.split(input)).toLowerCase(); + } +} diff --git a/src/main/java/util/TypeUtils.java b/src/main/java/util/TypeUtils.java new file mode 100644 index 0000000..2c8130f --- /dev/null +++ b/src/main/java/util/TypeUtils.java @@ -0,0 +1,17 @@ +package util; + +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +public class TypeUtils { + public static Map listToMap(List list) { + Map map = new HashMap(); + Iterator iterator = list.iterator(); + while (iterator.hasNext()) { + map.put(iterator.next(), iterator.next()); + } + return map; + } +} diff --git a/src/main/lua/anon.lua b/src/main/lua/anon.lua new file mode 100644 index 0000000..dd7760f --- /dev/null +++ b/src/main/lua/anon.lua @@ -0,0 +1,5 @@ +local foo = function(a) + return {a, 2 * a} +end + +return {foo(3), foo(4)} \ No newline at end of file diff --git a/src/main/lua/bit/todo/bit.lua b/src/main/lua/bit/todo/bit.lua new file mode 100644 index 0000000..d9fd4ce --- /dev/null +++ b/src/main/lua/bit/todo/bit.lua @@ -0,0 +1,260 @@ +--[[--------------- +LuaBit v0.4 +------------------- +a bitwise operation lib for lua. + +http://luaforge.net/projects/bit/ + +How to use: +------------------- + bit.bnot(n) -- bitwise not (~n) + bit.band(m, n) -- bitwise and (m & n) + bit.bor(m, n) -- bitwise or (m | n) + bit.bxor(m, n) -- bitwise xor (m ^ n) + bit.brshift(n, bits) -- right shift (n >> bits) + bit.blshift(n, bits) -- left shift (n << bits) + bit.blogic_rshift(n, bits) -- logic right shift(zero fill >>>) + +Please note that bit.brshift and bit.blshift only support number within +32 bits. + +2 utility functions are provided too: + bit.tobits(n) -- convert n into a bit table(which is a 1/0 sequence) + -- high bits first + bit.tonumb(bit_tbl) -- convert a bit table into a number +------------------- + +Under the MIT license. + +copyright(c) 2006~2007 hanzhao (abrash_han@hotmail.com) +--]]--------------- + +do + +------------------------ +-- bit lib implementions + +local function check_int(n) + -- checking not float + if(n - math.floor(n) > 0) then + error("trying to use bitwise operation on non-integer!") + end +end + +local function to_bits(n) + check_int(n) + if(n < 0) then + -- negative + return to_bits(bit.bnot(math.abs(n)) + 1) + end + -- to bits table + local tbl = {} + local cnt = 1 + while (n > 0) do + local last = math.mod(n,2) + if(last == 1) then + tbl[cnt] = 1 + else + tbl[cnt] = 0 + end + n = (n-last)/2 + cnt = cnt + 1 + end + + return tbl +end + +local function tbl_to_number(tbl) + local n = table.getn(tbl) + + local rslt = 0 + local power = 1 + for i = 1, n do + rslt = rslt + tbl[i]*power + power = power*2 + end + + return rslt +end + +local function expand(tbl_m, tbl_n) + local big = {} + local small = {} + if(table.getn(tbl_m) > table.getn(tbl_n)) then + big = tbl_m + small = tbl_n + else + big = tbl_n + small = tbl_m + end + -- expand small + for i = table.getn(small) + 1, table.getn(big) do + small[i] = 0 + end + +end + +local function bit_or(m, n) + local tbl_m = to_bits(m) + local tbl_n = to_bits(n) + expand(tbl_m, tbl_n) + + local tbl = {} + local rslt = math.max(table.getn(tbl_m), table.getn(tbl_n)) + for i = 1, rslt do + if(tbl_m[i]== 0 and tbl_n[i] == 0) then + tbl[i] = 0 + else + tbl[i] = 1 + end + end + + return tbl_to_number(tbl) +end + +local function bit_and(m, n) + local tbl_m = to_bits(m) + local tbl_n = to_bits(n) + expand(tbl_m, tbl_n) + + local tbl = {} + local rslt = math.max(table.getn(tbl_m), table.getn(tbl_n)) + for i = 1, rslt do + if(tbl_m[i]== 0 or tbl_n[i] == 0) then + tbl[i] = 0 + else + tbl[i] = 1 + end + end + + return tbl_to_number(tbl) +end + +local function bit_not(n) + + local tbl = to_bits(n) + local size = math.max(table.getn(tbl), 32) + for i = 1, size do + if(tbl[i] == 1) then + tbl[i] = 0 + else + tbl[i] = 1 + end + end + return tbl_to_number(tbl) +end + +local function bit_xor(m, n) + local tbl_m = to_bits(m) + local tbl_n = to_bits(n) + expand(tbl_m, tbl_n) + + local tbl = {} + local rslt = math.max(table.getn(tbl_m), table.getn(tbl_n)) + for i = 1, rslt do + if(tbl_m[i] ~= tbl_n[i]) then + tbl[i] = 1 + else + tbl[i] = 0 + end + end + + --table.foreach(tbl, print) + + return tbl_to_number(tbl) +end + +local function bit_rshift(n, bits) + check_int(n) + + local high_bit = 0 + if(n < 0) then + -- negative + n = bit_not(math.abs(n)) + 1 + high_bit = 2147483648 -- 0x80000000 + end + + for i=1, bits do + n = n/2 + n = bit_or(math.floor(n), high_bit) + end + return math.floor(n) +end + +-- logic rightshift assures zero filling shift +local function bit_logic_rshift(n, bits) + check_int(n) + if(n < 0) then + -- negative + n = bit_not(math.abs(n)) + 1 + end + for i=1, bits do + n = n/2 + end + return math.floor(n) +end + +local function bit_lshift(n, bits) + check_int(n) + + if(n < 0) then + -- negative + n = bit_not(math.abs(n)) + 1 + end + + for i=1, bits do + n = n*2 + end + return bit_and(n, 4294967295) -- 0xFFFFFFFF +end + +local function bit_xor2(m, n) + local rhs = bit_or(bit_not(m), bit_not(n)) + local lhs = bit_or(m, n) + local rslt = bit_and(lhs, rhs) + return rslt +end + +-------------------- +-- bit lib interface + +bit = { + -- bit operations + bnot = bit_not, + band = bit_and, + bor = bit_or, + bxor = bit_xor, + brshift = bit_rshift, + blshift = bit_lshift, + bxor2 = bit_xor2, + blogic_rshift = bit_logic_rshift, + + -- utility func + tobits = to_bits, + tonumb = tbl_to_number, +} + +end + +--[[ +for i = 1, 100 do + for j = 1, 100 do + if(bit.bxor(i, j) ~= bit.bxor2(i, j)) then + error("bit.xor failed.") + end + end +end +--]] + + + + + + + + + + + + + diff --git a/src/main/lua/bit/todo/hex.lua b/src/main/lua/bit/todo/hex.lua new file mode 100644 index 0000000..8e4bdcb --- /dev/null +++ b/src/main/lua/bit/todo/hex.lua @@ -0,0 +1,95 @@ +--[[--------------- +Hex v0.4 +------------------- +Hex conversion lib for lua. + +How to use: + hex.to_hex(n) -- convert a number to a hex string + hex.to_dec(hex) -- convert a hex string(prefix with '0x' or '0X') to number + +Part of LuaBit(http://luaforge.net/projects/bit/). + +Under the MIT license. + +copyright(c) 2006~2007 hanzhao (abrash_han@hotmail.com) +--]]--------------- + +require 'bit' + +do + +local function to_hex(n) + if(type(n) ~= "number") then + error("non-number type passed in.") + end + + -- checking not float + if(n - math.floor(n) > 0) then + error("trying to apply bitwise operation on non-integer!") + end + + if(n < 0) then + -- negative + n = bit.tobits(bit.bnot(math.abs(n)) + 1) + n = bit.tonumb(n) + end + + hex_tbl = {'A', 'B', 'C', 'D', 'E', 'F'} + hex_str = "" + + while(n ~= 0) do + last = math.mod(n, 16) + if(last < 10) then + hex_str = tostring(last) .. hex_str + else + hex_str = hex_tbl[last-10+1] .. hex_str + end + n = math.floor(n/16) + end + if(hex_str == "") then + hex_str = "0" + end + return "0x" .. hex_str +end + +local function to_dec(hex) + if(type(hex) ~= "string") then + error("non-string type passed in.") + end + + head = string.sub(hex, 1, 2) + + if( head ~= "0x" and head ~= "0X") then + error("wrong hex format, should lead by 0x or 0X.") + end + + v = tonumber(string.sub(hex, 3), 16) + + return v; +end + +-------------------- +-- hex lib interface +hex = { + to_dec = to_dec, + to_hex = to_hex, +} + +end + +--[[ +-- test +d = 4341688 +h = to_hex(d) +print(h) +print(to_dec(h)) + + +for i = 1, 100000 do + h = hex.to_hex(i) + d = hex.to_dec(h) + if(d ~= i) then + error("failed " .. i .. ", " .. h) + end +end +--]] \ No newline at end of file diff --git a/src/main/lua/bit/todo/noki.lua b/src/main/lua/bit/todo/noki.lua new file mode 100644 index 0000000..e844421 --- /dev/null +++ b/src/main/lua/bit/todo/noki.lua @@ -0,0 +1,332 @@ +--[[--------------- +Noki v0.4 +------------------- +Noki is a toolkit to convert Nokia PC Suite backuped SMS to a +unicode .txt file, which is more accessible than the original +.nfb or .nfc. + +It works well for Nokia PC Suite 6.5.12 and my mobile phone is +Nokia 7360. There might be some compatibility problem if you +use earlier version of the PC Suite. + +How to use: + noki.save_sms('nokia.nfb', 'sms.txt') + +Under the MIT license. + +Noki is a part of LuaBit(http://luaforge.net/projects/bit/). + +copyright(c) 2006~2007 hanzhao (abrash_han@hotmail.com) +--]]--------------- + +require 'hex' +require 'bit' + +do +-- globals +local RETURN = '\13\0\10\0' +local SMS_FILE = '\\MPAPI\\MESSAGES' +local SMS_INBOX = 'PIT_MESSAGE_INBOX' +local SMS_OUTBOX = 'PIT_MESSAGE_OUTBOX' +local SMS_ARCHIVEBOX = 'PIT_MESSAGE_ARCHIVE' + +-- output decorator +local SMS_INBOX_DEC = '[INBOX] ' +local SMS_OUTBOX_DEC = '[OUTBOX] ' +local SMS_ARCHIVE_DEC = '[ARCHIVE] ' + +-- box type +local BoxType = { + NON = 0, + IN = 1, + OUT = 2, + ARCHIVE = 3, +} + +-- feed each char with an extra \0 +local function asci_to_uni(asci) + --print("-------") + local uni = "" + for i = 1, string.len(asci) do + local str = string.format('%c\0', string.byte(asci, i)) + --print(string.len(str)) + uni = uni .. str + + end + return uni +end + +local function asci_padding(asci, pad) + local uni = "" + for i = 1, string.len(asci) do + local str = string.format('%c', string.byte(asci, i)) + --print(string.len(str)) + uni = uni .. str .. pad + + end + return uni +end + +-- shrink the \0 in uni code string +local function uni_to_asci(uni) + local asci = '' + --print('uni len ' .. string.len(uni)) + for i = 1, string.len(uni), 2 do + asci = asci .. string.sub(uni, i, i) + end + return asci +end + +local function reader(str) + local index = 1 + return function (n) + --print('reader>>> idx ' .. index .. " n " .. n) + local sub = string.sub(str, index, index + n - 1) + --[[print(hex.to_hex(string.byte(sub, 1))) + print(hex.to_hex(string.byte(sub, 2))) + print(hex.to_hex(string.byte(sub, 3))) + print(hex.to_hex(string.byte(sub, 4))) + --]] + index = index + n + return sub + end +end + +local function read_number(read, n) + local str = read(n) + local rslt = 0 + for i = 1, n do + local v = string.byte(str, i) + rslt = bit.bor(rslt, bit.blshift(v, (i-1)*8)) + end + return rslt +end + +local function read_int(read) + return read_number(read, 4) +end + + +local function read_short(read) + return read_number(read, 2) +end + +local function read_nfb_string(read) + local len = read_int(read) + local unistr = read(len*2) + return unistr +end + +local function read_nfb_header(read) + local nfb_header = { + ver = read_int(read), + firmware = read_nfb_string(read), + phone = read_nfb_string(read), + entries = read_int(read), + } + return nfb_header +end + +local function read_nfb_file(read) + local nfbf = {} + nfbf.path = read_nfb_string(read) + + nfbf.nbytes = read_int(read) + + nfbf.bytes = read(nfbf.nbytes) + local stamp = read_int(read) + + return nfbf +end + +local function read_nfb_dir(read) + local nfbd = { + path = read_nfb_string(read) + } + return nfbd +end + +local function save_entry(fp, tbl) + for k, v in pairs(tbl) do + fp:write(v) + fp:write(RETURN) + end +end + +-- save sms entries +local function save_sms(fp, ctnt) + -- print("save sms ----") + local in_box = asci_padding(SMS_INBOX, "%z") + local out_box = asci_padding(SMS_OUTBOX, "%z") + local archive_box = asci_padding(SMS_ARCHIVEBOX, "%z") + local line_s = asci_padding("1020", "%z") + local head = asci_padding("1033", "%z") + local tail = asci_padding("1040", "%z") + local service_center_tail = asci_padding("1080", "%z") + local phone_nb = "%+%z%d%z%d%z[%d%z]+" -- default is type 145 with '+' + local phone_nb_129 = string.rep("%d%z", 11) -- phone number type 129 without '+' + local time = "[%d%z]+%-%z%d%z%d%z%-%z%d%z%d%zT%z%d%z%d%z:%z%d%z%d%z" + + local pattern = "([^\10]+)\13%z\10%z" + local line_end = "\13%z\10%z" + local lineb, linee = string.find(ctnt, line_end) + local start = 1 + local line_number = 1 + while(lineb and linee) do + local line = string.sub(ctnt, start, lineb - 1) + --line = string.sub(ctnt, gb, ge) + local type = BoxType.NON + --print('capture ' .. string.len(line)) + --print(uni_to_asci(box)) + if(string.find(line, in_box)) then + fp:write(asci_to_uni(SMS_INBOX_DEC)) + type = BoxType.IN + elseif(string.find(line, out_box)) then + fp:write(asci_to_uni(SMS_OUTBOX_DEC)) + type = BoxType.OUT + elseif(string.find(line, archive_box)) then + fp:write(asci_to_uni(SMS_ARCHIVE_DEC)) + type = BoxType.ARCHIVE + else + --print(uni_to_asci(line)) + io.close(fp) + --error('unknown sms type') + + return + end + + hb, he = string.find(line, head) + tb, te = string.find(line, tail) + + local first_number = "" + -- service center address + sb, se = string.find(line, phone_nb, tb) + --print("" .. sb .. ", " .. se) + if(sb and se) then + --print(uni_to_asci(string.sub(line, sb, se))) + -- keep the find number, if the second find for sender address fails + -- then this number is the sender address + first_number = string.sub(line, sb, se) + else + sb, se = string.find(line, phone_nb_129, tb) + if(not (sb and se)) then + --io.close(fp) + --error("error service center address") + --return + first_number = "empty number" + -- nokia's pc suite may leave the serivce center address empty + end + end + + -- sender address + se_old = se + sb, se = string.find(line, phone_nb, se) + --print("" .. sb .. ", " .. se) + local sender_address = "" + if(sb and se) then + --print(uni_to_asci(string.sub(line, sb, se))) + sender_address = string.sub(line, sb, se) + else + sb, se = string.find(line, phone_nb_129, se_old) + if(not (sb and se)) then + --[[ + print(line_number) + io.close(fp) + error("error sender address") + --]] + sender_address = first_number + end + end + -- write sender + fp:write(sender_address) + fp:write(" \0") + + -- date time + -- out box have no date time slot + if(type ~= BoxType.OUT and first_number ~= "empty number") then + tmb, tme = string.find(line, time, se) + --print('' .. tmb .. ", " .. tme) + if(tmb and tme) then + --print(uni_to_asci(string.sub(line, tmb+1, tme))) + else + io.close(fp) + error("error reading date time") + return + end + fp:write(string.sub(line, tmb+1, tme)) + end + fp:write(RETURN) + + fp:write(string.sub(line, he+3, tb-3)) + + fp:write(RETURN) + fp:write(RETURN) + --end + start = linee + 1 + lineb, linee = string.find(ctnt, line_end, linee) + line_number = line_number + 1 + end +end + +-- save sms from a .nfc or .nfb file to a unicode .txt file +local function save_nfx_to(from, too) + local fp = io.open(too, 'wb') + if(not fp) then + error("error opening file " .. too .. " to write") + return + end + v = string.format('%c%c', 255, 254) + -- unicode .txt 'FF FE' + fp:write(v) + + -- read the .nfc file + local nokia = io.open(from, 'rb') + if(not nokia) then + error("error open file " .. from .. " to read") + end + local ctnt = nokia:read("*a") + io.close(nokia) + + local read = reader(ctnt) + + local header = read_nfb_header(read) + --print(header.ver) + --print(header.entries) + + for i=1, header.entries do + --print(i) + local type = read_int(read) + if(type == 1) then + -- file entry + --print('file') + local fe = read_nfb_file(read) + --save_entry(fp, fe) + if(uni_to_asci(fe.path) == SMS_FILE) then + local smsctnt = fe.bytes + --print('sms len ' .. fe.nbytes) + save_sms(fp, smsctnt) + return + end + + elseif(type == 2) then + -- dir entry + --print('dir') + local fd = read_nfb_dir(read) + --save_entry(fp, fd) + else + io.close(fp) + error('unknown entry type : ' .. hex.to_hex(type)) + end + end + io.close(fp) +end + +-- noki interface -- +noki = { + save_sms = save_nfx_to +} + +end -- end block + +-- sample +-- noki.save_sms('nokia2.nfb', 'sms2.txt') \ No newline at end of file diff --git a/src/main/lua/bit/todo/readme.txt b/src/main/lua/bit/todo/readme.txt new file mode 100644 index 0000000..7a0a24e --- /dev/null +++ b/src/main/lua/bit/todo/readme.txt @@ -0,0 +1,143 @@ +LuaBit +------ +LuaBit is a bitwise operation lib completely written in Lua. It's +written in the belief that Lua is self-contained. + +The supported operations are: not, and, or, xor, right shift, logic +right shift and left shift. + +Several utilities are designed to leverage the power of bit operation: + 1. hex: a dec <-> hex number converter + 2. utf8: convert utf8 string to ucs2 + 3. noki: convert nokia pc suite backuped SMS file to .txt + +Under the MIT license. + +Visit http://luaforge.net/projects/bit/ to get latest version. + +Status +------ +Now LuaBit is in v0.4. +Release date: Mar 18, 2007 + +Content +------- +3 files are there for LuaBit: + 1) bit.lua + is the bitwise operation lib, all operations are implemented here. + + 2) hex.lua + is a helper lib for ease of using hex numbers with bitwise + operation. + + 3) noki.lua + a utility(based on bit and hex) to convert Nokia PC Suite backuped + SMS to a unicode .txt file, which is more accessible than the + original .nfb or .nfc file. + + 4) utf8.lua + convert utf8 string to ucs2 string + +How to use +---------- +Bit +--- +Just require 'bit' in your project and the bit lib will be +available: + bit.bnot(n) -- bitwise not (~n) + bit.band(m, n) -- bitwise and (m & n) + bit.bor(m, n) -- bitwise or (m | n) + bit.bxor(m, n) -- bitwise xor (m ^ n) + bit.brshift(n, bits) -- right shift (n >> bits) + bit.blshift(n, bits) -- left shift (n << bits) + bit.blogic_rshift(n, bits) -- logic right shift(zero fill >>>) + +Please note that bit.brshift and bit.blshift only support number within +32 bits. + +2 utility functions are provided too: + bit.tobits(n) -- convert n into a bit table(which is a 1/0 sequence) + -- high bits first + bit.tonumb(bit_tbl) -- convert a bit table into a number + +Hex +--- +For ease of using hex numbers, a utility hex lib is also included in +LuaBit. You can require 'hex' to use them: + hex.to_hex(n) -- convert a number to a hex string + hex.to_dec(hex) -- convert a hex string(prefix with '0x' or '0X') to number + +With hex, you can write code like: + bit.band(258, hex.to_dec('0xFF')) +to get the lower 8 bits of 258, that's 2. + +Noki +---- +require 'noki', to save your sms to .txt file: + noki.save_sms('nokia.nfb', 'sms.txt') +and you can view the output sms.txt in notepad or other editor which +support unicode. + +Utf8 +---- +require 'utf8', to convert a utf8 string: + ucs2_string = utf8.utf_to_uni(utf8_string) + +History +------- +v0.4 +* utf8 to ucs2 converter(utf8.lua). +* clean up for compatible with Lua5.1 and 5.0. +* add 'How to use' section for bit.lua and hex.lua. + +v0.3 +* noki added as an application of bit. +* typo correction. + +v0.2 +* add logic right shift(>>>) support: bit.blogic_rshift. +* add 2 utility functions: bit.tobits and bit.tonumb. +* update hex.to_hex(in hex.lua) to support negative number. + +v0.1 +LuaBit is written when I do my own game project(Fio at http://fio.edithis.info). +When loading resources, I have to do some bit operation. And I do not +like the embedded way of bit operation. So I decide to implement those +ops in lua. And that's LuaBit. It's not as fast as the embedded one, but +it works. And Lua is self-contained :-) + +To-Do List +--------- +v0.1 +It'll be useful if LuaBit support those bitwise op like: + bit.band(258, '0xFF') +ease to type and use. This will be supported in next release. + +v0.2 +I decide to delay this feature to later version for it'll mess up the +interface of LuaBit. + +v0.3 +May more utility functions add to Noki - phonebook might be a nice candidate. + +v0.4 +There's no UCS2 -> UTF8 convertion now, this feature may add in next release +or when the project need. + +Noki'll be be exluded from LuaBit in next release; I decide to let Noki grow +into a powerful tool to support more Nokia PC Suite backup format(.nfb, +.nfc and .nbu). + +Trial Noki demo at http://nokisms.googlepages.com/(in Chinese) + +Known issues +------------ +LuaBit doesn't play very well with negative number. The return value of the +bitwise operations might change to positive when applied on negative numbers +though the bit sequence is correct. So if you want do some arithmetic with +the result of bit operation, be careful. + +Feedback +-------- +Please send your comments, bugs, patches or change request to +hanzhao(abrash_han@hotmail.com). diff --git a/src/main/lua/bit/todo/utf8.lua b/src/main/lua/bit/todo/utf8.lua new file mode 100644 index 0000000..84ee8d4 --- /dev/null +++ b/src/main/lua/bit/todo/utf8.lua @@ -0,0 +1,163 @@ +--[[--------------- +Utf8 v0.4 +------------------- +utf8 -> unicode ucs2 converter + +How to use: +to convert: + ucs2_string = utf8.utf_to_uni(utf8_string) + +to view a string in hex: + utf8.print_hex(str) + +Under the MIT license. + +Utf8 is a part of LuaBit Project(http://luaforge.net/projects/bit/). + +copyright(c) 2007 hanzhao (abrash_han@hotmail.com) +--]]--------------- + +require 'hex' +require 'bit' + +do + local BYTE_1_HEAD = hex.to_dec('0x00') -- 0### #### + local BYTE_2_HEAD = hex.to_dec('0xC0') -- 110# #### + local BYTE_3_HEAD = hex.to_dec('0xE0') -- 1110 #### + + -- mask to get the head + local BYTE_1_MASK = hex.to_dec('0x80') -- 1### #### + local BYTE_2_MASK = hex.to_dec('0xE0') -- 111# #### + local BYTE_3_MASK = hex.to_dec('0xF0') -- 1111 #### + + -- tail byte mask + local TAIL_MASK = hex.to_dec('0x3F') -- 10## #### + + local mask_tbl = { + BYTE_3_MASK, + BYTE_2_MASK, + BYTE_1_MASK, + } + local head_tbl = { + BYTE_3_HEAD, + BYTE_2_HEAD, + BYTE_1_HEAD, + } + + local len_tbl = { + [BYTE_1_HEAD] = 1, + [BYTE_2_HEAD] = 2, + [BYTE_3_HEAD] = 3, + } + + local function utf_read_char(utf, start) + local head_byte = string.byte(utf, start) + --print('head byte ' .. hex.to_hex(head_byte)) + for m = 1, table.getn(mask_tbl) do + local mask = mask_tbl[m] + -- head match + local head = bit.band(head_byte, mask) + --print('head ' .. hex.to_hex(head) .. ' ' .. hex.to_hex(mask)) + if(head == head_tbl[m]) then + local len = len_tbl[head_tbl[m]] + --print('len ' .. len) + local tail_idx = start + len - 1 + local char = 0 + -- tail + for i = tail_idx, start + 1, -1 do + local tail_byte = string.byte(utf, i) + local byte = bit.band(tail_byte, TAIL_MASK) + --print('byte ' .. hex.to_hex(byte).. ' = ' .. hex.to_hex(tail_byte) .. '&'..hex.to_hex(TAIL_MASK)) + if(tail_idx - i > 0) then + local sft = bit.blshift(byte, (tail_idx - i) * 6) + --print('shift ' .. hex.to_hex(sft) .. ' ' .. hex.to_hex(byte) .. ' ' .. ((tail_idx - i) * 6)) + char = bit.bor(char, sft) + --print('char ' .. hex.to_hex(char)) + else + char = byte + end + end -- tails + + -- add head + local head_val = bit.band(head_byte, bit.bnot(mask)) + --print('head val ' .. hex.to_hex(head_val)) + head_val = bit.blshift(head_val, (len-1) * 6) + --print('head val ' .. hex.to_hex(head_val)) + char = bit.bor(head_val, char) + --print('char ' .. hex.to_hex(char)) + + return char, len + end -- if head match + end -- for mask + error('not find proper head mask') + end + + local function print_hex(str) + local cat = '' + for i=1, string.len(str) do + cat = cat .. ' ' .. hex.to_hex(string.byte(str, i)) + end + print(cat) + end + + local HI_MASK = hex.to_dec('0xF0') + local LO_MASK = hex.to_dec('0xFF') + + local function char_to_str(char) + local hi, lo = bit.brshift(char, 8), bit.band(char, LO_MASK) + -- print(hex.to_hex(char)..' '..hex.to_hex(hi)..' ' .. hex.to_hex(lo)) + if(hi == 0) then + return string.format('%c\0', lo) + elseif(lo == 0) then + return string.format('\0%c', hi) + else + return string.format('%c%c', lo, hi) + end + end + + local function utf_to_uni(utf) + local n = string.len(utf) + local i = 1 + local uni = '' + while(i <= n) do + --print('---') + char, len = utf_read_char(utf, i) + i = i + len + --print(string.len(char_to_str(char))) + + uni = uni..char_to_str(char) + end + --print_hex(uni) + return uni + end + + -- interface + utf8 = { + utf_to_uni = utf_to_uni, + print_hex = print_hex, + } + +end + +--[[ +-- test +byte_3 = string.format('%c%c%c', hex.to_dec('0xE7'), hex.to_dec('0x83'), hex.to_dec('0xad')) +print(string.len(byte_3)) +utf8.utf_to_uni(byte_3) +--]] +--[[ +byte_2 = string.format('%c%c', hex.to_dec('0xC2'), hex.to_dec('0x9D')) +utf8.utf_to_uni(byte_2) + +byte_1 = string.format('%c', hex.to_dec('0xB')) +utf8.utf_to_uni(byte_1) +--]] +--[[ +test_mul = string.format( +'%c%c%c%c%c%c%c%c%c', +hex.to_dec('0xE8'),hex.to_dec('0xAF'), hex.to_dec('0xBA'), +hex.to_dec('0xE5'),hex.to_dec('0x9F'), hex.to_dec('0xBA'), +hex.to_dec('0xE4'),hex.to_dec('0xBA'), hex.to_dec('0x9A')) + +utf8.print_hex(utf8.utf_to_uni(test_mul)) +--]] \ No newline at end of file diff --git a/src/main/lua/getall.lua b/src/main/lua/getall.lua new file mode 100644 index 0000000..b00067f --- /dev/null +++ b/src/main/lua/getall.lua @@ -0,0 +1,6 @@ +local result = {} +local ids = redis.call('SMEMBERS', ARGV[1] .. ':_id') +for _, id in pairs(ids) do + result[#result + 1] = redis.call('HGETALL', ARGV[1] .. ':' .. id) +end +return result \ No newline at end of file diff --git a/src/main/lua/getsome.lua b/src/main/lua/getsome.lua new file mode 100644 index 0000000..248ab7f --- /dev/null +++ b/src/main/lua/getsome.lua @@ -0,0 +1,2 @@ +local fields = list.slice(ARGV, 2) +return util.getsome(ARGV[1], fields) \ No newline at end of file diff --git a/src/main/lua/getsomex.lua b/src/main/lua/getsomex.lua new file mode 100644 index 0000000..b896e37 --- /dev/null +++ b/src/main/lua/getsomex.lua @@ -0,0 +1,11 @@ +local result = {} +local ids = redis.call('SMEMBERS', ARGV[1] .. ':_id') +for _, id in pairs(ids) do + result[#result + 1] = redis.call('HGETALL', ARGV[1] .. ':' .. id) +end +local fields = {} +for i = 2, #ARGV do + fields[#fields + 1] = 'GET' + fields[#fields + 1] = ARGV[1] .. ':*:' .. ARGV[i] +end +return redis.call('SORT', ARGV[1] .. ':_id', 'BY', 'NOSORT', unpack(fields)) \ No newline at end of file diff --git a/src/main/lua/insert.lua b/src/main/lua/insert.lua new file mode 100644 index 0000000..d250e2d --- /dev/null +++ b/src/main/lua/insert.lua @@ -0,0 +1,18 @@ +local id = redis.call('INCR', '_id:' .. ARGV[1]) +redis.call('SADD', ARGV[1] .. ':_id', id) -- table:_id + +local hash = {} +for i = 2, #ARGV, 3 do + hash[#hash + 1] = ARGV[i] + hash[#hash + 1] = ARGV[i + 1] + redis.call('SET', ARGV[1] .. ':' .. id .. ':' .. ARGV[i], ARGV[i + 1]) -- table:id:field + redis.call('SADD', ARGV[1] .. ':' .. ARGV[i] .. ':' .. ARGV[i + 2], id) -- table:field:value + -- redis.call('SADD', ARGV[1] .. ':' .. ARGV[i], ARGV[i + 2]) -- table:field +end +--[[ +for i = 1, 10, 1 do + redis.call('KEYS', '*') +end +--]] +redis.call('HMSET', ARGV[1] .. ':' .. id, unpack(hash)) -- table:id +return id \ No newline at end of file diff --git a/src/main/lua/list/slice.lua b/src/main/lua/list/slice.lua new file mode 100644 index 0000000..d02b129 --- /dev/null +++ b/src/main/lua/list/slice.lua @@ -0,0 +1,8 @@ +function list.slice(inx, offset, length) + length = length or #inx + local out = {} + for i = offset, length do + out[#out + 1] = inx[i] + end + return out +end \ No newline at end of file diff --git a/src/main/lua/msgpack.lua b/src/main/lua/msgpack.lua new file mode 100644 index 0000000..a3ffabc --- /dev/null +++ b/src/main/lua/msgpack.lua @@ -0,0 +1,3 @@ +local a = cmsgpack.pack("abcdefg") +local b = cmsgpack.unpack(a) +return cjson.decode(ARGV[1]) \ No newline at end of file diff --git a/src/main/lua/returnint.lua b/src/main/lua/returnint.lua new file mode 100644 index 0000000..731a89e --- /dev/null +++ b/src/main/lua/returnint.lua @@ -0,0 +1 @@ +return 123 \ No newline at end of file diff --git a/src/main/lua/returnnil.lua b/src/main/lua/returnnil.lua new file mode 100644 index 0000000..bd6def0 --- /dev/null +++ b/src/main/lua/returnnil.lua @@ -0,0 +1 @@ +return nil \ No newline at end of file diff --git a/src/main/lua/returnset.lua b/src/main/lua/returnset.lua new file mode 100644 index 0000000..7021204 --- /dev/null +++ b/src/main/lua/returnset.lua @@ -0,0 +1 @@ +return {'a', 'b', 3} \ No newline at end of file diff --git a/src/main/lua/returnstring.lua b/src/main/lua/returnstring.lua new file mode 100644 index 0000000..6ac070c --- /dev/null +++ b/src/main/lua/returnstring.lua @@ -0,0 +1 @@ +return 'abc' \ No newline at end of file diff --git a/src/main/lua/test1.lua b/src/main/lua/test1.lua new file mode 100644 index 0000000..cf554cd --- /dev/null +++ b/src/main/lua/test1.lua @@ -0,0 +1,2 @@ +local msg = 'Hello, world!' +return msg \ No newline at end of file diff --git a/src/main/lua/test2.lua b/src/main/lua/test2.lua new file mode 100644 index 0000000..22f865d --- /dev/null +++ b/src/main/lua/test2.lua @@ -0,0 +1,2 @@ +local msg = string.format('Hello from %s',_VERSION) +return msg \ No newline at end of file diff --git a/src/main/lua/test3.lua b/src/main/lua/test3.lua new file mode 100644 index 0000000..22f865d --- /dev/null +++ b/src/main/lua/test3.lua @@ -0,0 +1,2 @@ +local msg = string.format('Hello from %s',_VERSION) +return msg \ No newline at end of file diff --git a/src/main/lua/util/bla.lua b/src/main/lua/util/bla.lua new file mode 100644 index 0000000..ba0f37f --- /dev/null +++ b/src/main/lua/util/bla.lua @@ -0,0 +1,3 @@ +function util.bla() + +end \ No newline at end of file diff --git a/src/main/lua/util/getsome.lua b/src/main/lua/util/getsome.lua new file mode 100644 index 0000000..3323698 --- /dev/null +++ b/src/main/lua/util/getsome.lua @@ -0,0 +1,9 @@ +function util.getsome(table, fields) + local result = {} + local ids = redis.call('SMEMBERS', table .. ':_id') + for _, id in pairs(ids) do + result[#result + 1] = redis.call('HMGET', table .. ':' .. id, unpack(fields)) + end + util.bla() + return result +end \ No newline at end of file diff --git a/src/main/resources/flight.txt b/src/main/resources/flight.txt new file mode 100644 index 0000000..e43c1b6 --- /dev/null +++ b/src/main/resources/flight.txt @@ -0,0 +1,7 @@ +Id Departure Time Airline Departure City Arrival City +1 03/23/2013 1:30 pm Virgin America Los Angeles New York +2 03/23/2013 1:45 pm Virgin America San Francisco New York +3 03/23/2013 1:45 pm American Airlines San Francisco New York +4 03/23/2013 1:50 pm American Airlines Los Angeles Boston +5 03/23/2013 2:45 pm Southwest San Francisco New York +6 03/23/2013 3:30 pm Southwest San Francisco New York \ No newline at end of file diff --git a/src/main/resources/pvoutput.debug b/src/main/resources/pvoutput.debug new file mode 100644 index 0000000..13926b3 --- /dev/null +++ b/src/main/resources/pvoutput.debug @@ -0,0 +1,10 @@ +[[".\/pvoutput.php","130","1256","233.8","1204DQ0116"],837969,["336",232.76916666667,1354450617,3.4],{"d":"20121202","t":"13:16","v1":130,"v2":1256,"v6":233.8,"v5":3.4}] +[[".\/pvoutput.php","230","1247","234.8","1204DQ0116"],1218479,["336",338.46638888889,1354450918,3.4],{"d":"20121202","t":"13:21","v1":230,"v2":1247,"v6":234.8,"v5":3.4}] +[[".\/pvoutput.php","340","1209","234.8","1204DQ0116"],1592908,["336",442.47444444444,1354451223,3.4],{"d":"20121202","t":"13:27","v1":340,"v2":1209,"v6":234.8,"v5":3.4}] +[[".\/pvoutput.php","440","1187","233.8","1204DQ0116"],1951498,["336",542.08277777778,1354451523,3.4],{"d":"20121202","t":"13:32","v1":440,"v2":1187,"v6":233.8,"v5":3.4}] +[[".\/pvoutput.php","540","1166","233.8","1204DQ0116"],2310029,["336",641.67472222222,1354451827,3.4],{"d":"20121202","t":"13:37","v1":540,"v2":1166,"v6":233.8,"v5":3.4}] +[[".\/pvoutput.php","630","1141","235.8","1204DQ0116"],2655467,["336",737.62972222222,1354452127,3.4],{"d":"20121202","t":"13:42","v1":630,"v2":1141,"v6":235.8,"v5":3.4}] +[[".\/pvoutput.php","730","1119","232.8","1204DQ0116"],2994637,["336",831.84361111111,1354452427,3.5],{"d":"20121202","t":"13:47","v1":730,"v2":1119,"v6":232.8,"v5":3.5}] +[[".\/pvoutput.php","820","1113","231.8","1204DQ0116"],3327468,["336",924.29666666667,1354452727,3.6],{"d":"20121202","t":"13:52","v1":820,"v2":1113,"v6":231.8,"v5":3.6}] +[[".\/pvoutput.php","910","387","231.8","1204DQ0116"],3656613,["336",1015.7258333333,1354453027,3.6],{"d":"20121202","t":"13:57","v1":910,"v2":387,"v6":231.8,"v5":3.6}] +[[".\/pvoutput.php","940","283","228.8","1204DQ0116"],3753684,["336",1042.69,1354453328,3.7],{"d":"20121202","t":"14:02","v1":940,"v2":283,"v6":228.8,"v5":3.7}] \ No newline at end of file diff --git a/src/test/java/test/Flight.java b/src/test/java/test/Flight.java new file mode 100644 index 0000000..c556724 --- /dev/null +++ b/src/test/java/test/Flight.java @@ -0,0 +1,99 @@ +package test; + +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Scanner; +import java.util.Set; + +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import util.FileUtils; +import util.LuaUtils; + +public class Flight { + private SimpleDateFormat inputSDF; + private JedisPool pool; + + public Flight() { + inputSDF = new SimpleDateFormat("MM/dd/yyyy hh:mm a"); + pool = new JedisPool("localhost", 16379); + LuaUtils.setPool(pool); + } + + public static void main(String[] args) throws Exception { + new Flight().start(); + } + + private void start() throws IOException, ParseException { + clear(); + insert(); + list(); + } + + private void clear() { + Jedis jedis = pool.getResource(); + Set set = jedis.keys("*"); + if (set.size() > 0) { + jedis.del(set.toArray(new String[0])); + } + pool.returnResource(jedis); + } + + private void insert() throws IOException { + // http://robots.thoughtbot.com/post/46335890055/redis-set-intersection-using-sets-to-filter-data + Scanner lineScanner = FileUtils.resourceToScanner("flight.txt"); + lineScanner.nextLine(); + while (lineScanner.hasNextLine()) { + Scanner scanner = new Scanner(lineScanner.nextLine()); + HashMap map = new HashMap(); + scanner.useDelimiter("\t"); + scanner.nextInt(); // id + String date = scanner.next(); + long time; + try { + time = inputSDF.parse(date).getTime() / 1000; + map.put("departure_time", String.valueOf(time)); + map.put("airline", scanner.next()); + map.put("departure", scanner.next()); + map.put("arrival", scanner.next()); + long id = (Long) LuaUtils.insert("departures", map); + System.out.println(id); + + Jedis jedis = pool.getResource(); + jedis.zadd("departures:departure_time", time, String.valueOf(id)); + pool.returnResource(jedis); + } catch (ParseException e) { + e.printStackTrace(); + } + scanner.close(); + } + lineScanner.close(); + } + + private void list() throws IOException { + Jedis jedis = pool.getResource(); + System.out.println(jedis.hgetAll("departures:1")); + pool.returnResource(jedis); + + List> listMap; + + listMap = LuaUtils.getSome("departures", "airline", "arrival"); + for (Map map : listMap) { + System.out.println(map); + } + + listMap = LuaUtils.getSomeX("departures", "airline", "arrival"); + for (Map map : listMap) { + System.out.println(map); + } + + listMap = LuaUtils.getAll("departures"); + for (Map map : listMap) { + System.out.println(map); + } + } +} diff --git a/src/test/java/test/Relational.java b/src/test/java/test/Relational.java new file mode 100644 index 0000000..53e835c --- /dev/null +++ b/src/test/java/test/Relational.java @@ -0,0 +1,144 @@ +package test; + +import interval.Interval; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.sql.SQLException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.ServiceLoader; + +import model.Model; + +import org.json.simple.JSONArray; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import data.Entry; +import util.DateUtils; +import util.FileUtils; + +public class Relational { + protected Model model; + protected SimpleDateFormat inputSDF; + protected SimpleDateFormat outputSDF; + + public Relational() throws SQLException { + model = new Model(DateUtils.getCalendar()); + inputSDF = new SimpleDateFormat("yyyyMMdd HH:mm"); + inputSDF.setTimeZone(DateUtils.getTimeZone()); + outputSDF = new SimpleDateFormat("dd/MM/yyyy HH:mm:ss"); + inputSDF.setTimeZone(DateUtils.getTimeZone()); + } + + public static void main(String[] args) throws FileNotFoundException { + try { + new Relational().start(); + } catch (SQLException e) { + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void start() throws Exception { + insertFromFile(); + generateIntervals(); +// model.test( +// DateUtils.makeTimestamp(2012, 1, 1, 14, 0, 0, 0), +// DateUtils.makeTimestamp(2013, 7, 1, 14, 0, 0, 0), "Day"); + model.test(); + } + + public void insertFromFile() throws SQLException, IOException { + Date endDate; + boolean checkDate = ((endDate = model.getLastEntryDate()) != null); + System.out.println(endDate); + JSONParser parser = new JSONParser(); + BufferedReader bufferedReader = FileUtils.resourceToReader("pvoutput.debug", true); + String line; + int i = 0; + while ((line = bufferedReader.readLine()) != null) { + Object object = null; + try { + JSONArray json = (JSONArray) parser.parse(line); + object = parser.parse(json.get(3).toString()); + } catch (ParseException e) { + continue; + } + @SuppressWarnings("unchecked") + HashMap map = (HashMap) object; + String value = String.valueOf(map.get("v5")); + if (!value.equals("null")) { + try { + Date date = inputSDF.parse(String.format("%s %s", map.get("d"), map.get("t"))); + if (checkDate && date.compareTo(endDate) < 0) { + break; + } + if (++i > 100) break; + model.insertDataBatch(date.getTime(), Double.parseDouble(value)); + } catch (java.text.ParseException e) { + continue; + } + } + } + bufferedReader.close(); + model.insertDataBatchLast(); + } + + public void generateIntervals() throws Exception { + Date startDate = model.getFirstEntryDate(); + Date endDate = model.getLastEntryDate(); + + for (Object object : ServiceLoader.load(interval.Interval.class)) { + Interval interval = (Interval) object; + interval.setDate(startDate, endDate); + processInterval(interval); + } + } + + protected void processInterval(Interval interval) throws Exception { + String name = interval.getClass().getSimpleName(); + model.insertInterval(name); + int id = model.selectIntervalId(name); + + System.out.printf("[%d] %s\n", id, name); + System.out.println("========================================"); + + do { + Date intervalStartDate = interval.get(); + Date intervalEndDate = interval.next(); + + ArrayList entryList = model.selectDataBetween(intervalStartDate, intervalEndDate); + + int count = entryList.size(); + if (count > 0) { + System.out.printf("%s - %s (%d)\n", formatDate(intervalStartDate), formatDate(intervalEndDate), count); + ArrayList valueList = new ArrayList(); + for (Entry entry : entryList) { + System.out.printf("%s\t%f\n", formatDate(entry.date), entry.value); + valueList.add(entry.value); + } + model.insertExtremesBatch( + id, + intervalStartDate, + entryList.get(0).date, + entryList.get(count - 1).date, + count, + Collections.min(valueList), + Collections.max(valueList)); + } + } + while (interval.hasNext()); + model.insertExtremesBatchLast(); + } + + protected String formatDate(Date date) { + return outputSDF.format(date.getTime()); + } +} diff --git a/src/test/java/test/Test1.java b/src/test/java/test/Test1.java new file mode 100644 index 0000000..3443ef9 --- /dev/null +++ b/src/test/java/test/Test1.java @@ -0,0 +1,36 @@ +package test; + +import java.util.Set; + +import redis.clients.jedis.Jedis; + +public class Test1 { + + public static void main(String[] args) { + Jedis jedis = new Jedis("localhost", 16379); + //jedis.select(0); + + jedis.set("foo", "bar"); + + String value = jedis.get("foo"); + System.out.println(value); + + jedis.del("item"); + for (int i = 0; i < 10; ++i ) { + jedis.zadd(String.format("item"), 2*i, String.valueOf(Math.random())); + //jedis.sadd(String.format("items"), String.valueOf(i)); + } + + Set set = jedis.zrange("item", 0, -1); + for (String item : set) { + System.out.println(item); + } + System.out.println("-----------"); + set = jedis.zrangeByScore("item", 4, 7); + for (String item : set) { + System.out.println(item); + } + + //jedis.close(); + } +} diff --git a/src/test/java/test/Test2.java b/src/test/java/test/Test2.java new file mode 100644 index 0000000..4632be0 --- /dev/null +++ b/src/test/java/test/Test2.java @@ -0,0 +1,29 @@ +package test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.json.simple.JSONValue; + +import redis.clients.jedis.JedisPool; +import util.LuaUtils; + +public class Test2 { + public static void main(String[] args) throws IOException { + JedisPool pool = new JedisPool("localhost", 16379); + LuaUtils.setPool(pool); + + Object o1 = LuaUtils.getSome("departures", "airline", "arrival"); + System.out.println(o1); + + List src = new ArrayList(); + src.add("msgpack"); + src.add("kumofs"); + src.add("viver"); + + String arg = JSONValue.toJSONString(src); + Object object = LuaUtils.eval("msgpack", 0, arg); + System.out.println(object); + } +} \ No newline at end of file diff --git a/src/test/java/test/Test3.java b/src/test/java/test/Test3.java new file mode 100644 index 0000000..3b81864 --- /dev/null +++ b/src/test/java/test/Test3.java @@ -0,0 +1,44 @@ +package test; + +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import redis.clients.jedis.Pipeline; +import util.StringUtils; + +public class Test3 { + + public static void main(String[] args) { + System.out.println(StringUtils.parameterize("aDa halod823d!!@")); + + JedisPool pool = new JedisPool("localhost", 16379); + Jedis jedis1 = pool.getResource(); + jedis1.select(2); + + jedis1.del("bla"); + jedis1.del("da"); + + Jedis jedis2 = pool.getResource(); + + Pipeline pipeline = jedis2.pipelined(); + pipeline.select(2); + for (int i = 0; i < 10000; ++i) { + pipeline.sadd("bla", String.valueOf(i)); + if (i % 10 == 0) { + jedis1.sadd("da", String.valueOf(i)); + } + if (i % 100 == 0) { + System.out.println(jedis1.scard("bla")); + System.out.println(jedis1.scard("da")); + } + } + + + pipeline.sync(); + + System.out.println(jedis1.scard("bla")); + System.out.println(jedis1.scard("da")); + + pool.returnResource(jedis1); + pool.returnResource(jedis2); + } +} diff --git a/src/test/java/test/Test4.java b/src/test/java/test/Test4.java new file mode 100644 index 0000000..9d8f5fb --- /dev/null +++ b/src/test/java/test/Test4.java @@ -0,0 +1,29 @@ +package test; + +import java.util.ArrayList; +import java.util.List; + +import redis.clients.jedis.JedisShardInfo; +import redis.clients.jedis.ShardedJedis; + +public class Test4 { + + public static void main(String[] args) { + List shards = new ArrayList(); + JedisShardInfo si = new JedisShardInfo("localhost", 16379); + //si.setPassword("foobared"); + shards.add(si); + ShardedJedis jedis = new ShardedJedis(shards); + + jedis.set("fooder", "bar"); + for (int i = 0; i < 100; ++i ) { + jedis.zadd(String.format("item"), i, String.valueOf(i)); + } + + //jedis.zrange(key, start, end) + String value = jedis.get("fooder"); + System.out.println(value); + + //jedis.close(); + } +} diff --git a/src/test/java/test/Test5.java b/src/test/java/test/Test5.java new file mode 100644 index 0000000..6dbed1f --- /dev/null +++ b/src/test/java/test/Test5.java @@ -0,0 +1,46 @@ +package test; + +import java.io.IOException; +import java.security.NoSuchAlgorithmException; +import java.util.Date; + +import model.Model; + +import org.apache.commons.codec.digest.DigestUtils; + +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; +import util.DateUtils; +import util.FileUtils; +import util.LuaUtils; + +public class Test5 { + + public static void main(String[] args) throws NoSuchAlgorithmException, IOException { + JedisPool pool = new JedisPool("localhost", 16379); + + Jedis jedis = new Jedis("localhost", 16379); + jedis.select(0); + + String lua1 = FileUtils.resourceToString("test1.lua"); + String lua2 = FileUtils.resourceToString("test2.lua"); + + String lua1hash = DigestUtils.sha1Hex(lua1); + String lua2hash = DigestUtils.sha1Hex(lua2); + + System.out.println(jedis.eval(lua1, 0)); + System.out.println(jedis.scriptLoad(lua1)); + System.out.println(jedis.scriptLoad(lua2)); + System.out.println(jedis.evalsha(lua1hash, 0)); + System.out.println(jedis.evalsha(lua2hash, 0)); + + jedis.select(1); + Model model = new Model(DateUtils.getCalendar()); + System.out.println(model.selectDataBetween(new Date(1372629840000L), new Date(1375076100000L))); + + LuaUtils.setPool(pool); + System.out.println(LuaUtils.eval("test3")); + + //jedis.close(); + } +} diff --git a/src/test/java/test/Test6.java b/src/test/java/test/Test6.java new file mode 100644 index 0000000..9c3931a --- /dev/null +++ b/src/test/java/test/Test6.java @@ -0,0 +1,27 @@ +package test; + +import java.io.IOException; + +import redis.clients.jedis.JedisPool; +import util.LuaUtils; + +public class Test6 { + public static void main(String[] args) throws IOException { + JedisPool pool = new JedisPool("localhost", 16379); + LuaUtils.setPool(pool); + + Object o1 = LuaUtils.eval("returnnil"); + System.out.println("1 returnnil: " + o1); + + Object o2 = LuaUtils.eval("returnint"); + System.out.println("2 returnint: " + o2); + + Object o3 = LuaUtils.eval("returnstring"); + System.out.println("3 returnstring: " + o3); + + Object o4 = LuaUtils.eval("returnset"); + System.out.println("4 returnset: " + o4); + + // solves: https://github.com/xetorthio/jedis/issues/383 + } +} diff --git a/src/test/java/test/Test7.java b/src/test/java/test/Test7.java new file mode 100644 index 0000000..e5eee9e --- /dev/null +++ b/src/test/java/test/Test7.java @@ -0,0 +1,17 @@ +package test; + +import java.io.IOException; + +import redis.clients.jedis.JedisPool; +import util.LuaUtils; + +public class Test7 { + public static void main(String[] args) throws IOException { + JedisPool pool = new JedisPool("localhost", 16379); + LuaUtils.setPool(pool); + + // http://blog.jupo.org/2013/06/12/bitwise-lua-operations-in-redis/ + Object o1 = LuaUtils.eval("anon"); + System.out.println(o1); + } +} \ No newline at end of file diff --git a/src/test/resources/META-INF/services/interval.Interval b/src/test/resources/META-INF/services/interval.Interval new file mode 100644 index 0000000..bbe3b62 --- /dev/null +++ b/src/test/resources/META-INF/services/interval.Interval @@ -0,0 +1,8 @@ +interval.Hour +interval.Hour2 +interval.Hour4 +interval.Hour6 +interval.Hour12 +interval.Day +interval.Week +interval.Month \ No newline at end of file