From ad10f91b607f71b9a2bd53566ad3f25cbeb0cc9b Mon Sep 17 00:00:00 2001 From: Jungtaek Lim Date: Fri, 12 Sep 2014 17:50:43 +0900 Subject: [PATCH 1/2] Use lookup table when calculating CRC16 XMODEM * I borrowed it from https://github.com/mp911de/lettuce/commit/b9219314802fb0236b0bb39e8e38cc0e5b3077ec --- .../redis/clients/util/JedisClusterCRC16.java | 59 ++++++++++++++----- .../jedis/tests/benchmark/CRC16Benchmark.java | 27 +++++++++ .../tests/utils/JedisClusterCRC16Test.java | 43 ++++++++++++++ 3 files changed, 114 insertions(+), 15 deletions(-) create mode 100644 src/test/java/redis/clients/jedis/tests/benchmark/CRC16Benchmark.java create mode 100644 src/test/java/redis/clients/jedis/tests/utils/JedisClusterCRC16Test.java diff --git a/src/main/java/redis/clients/util/JedisClusterCRC16.java b/src/main/java/redis/clients/util/JedisClusterCRC16.java index 270dbf0..552dc17 100644 --- a/src/main/java/redis/clients/util/JedisClusterCRC16.java +++ b/src/main/java/redis/clients/util/JedisClusterCRC16.java @@ -1,8 +1,32 @@ package redis.clients.util; +/** + * CRC16 Implementation according to CCITT standard + * Polynomial : 1021 (x^16 + x^12 + x^5 + 1) + * + * @see: http://redis.io/topics/cluster-spec + * Appendix A. CRC16 reference implementation in ANSI C + */ public class JedisClusterCRC16 { - public final static int polynomial = 0x1021; // Represents x^16+x^12+x^5+1 - + public static final int LOOKUP_TABLE[] = { 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50A5, 0x60C6, 0x70E7, 0x8108, 0x9129, + 0xA14A, 0xB16B, 0xC18C, 0xD1AD, 0xE1CE, 0xF1EF, 0x1231, 0x0210, 0x3273, 0x2252, 0x52B5, 0x4294, 0x72F7, 0x62D6, + 0x9339, 0x8318, 0xB37B, 0xA35A, 0xD3BD, 0xC39C, 0xF3FF, 0xE3DE, 0x2462, 0x3443, 0x0420, 0x1401, 0x64E6, 0x74C7, + 0x44A4, 0x5485, 0xA56A, 0xB54B, 0x8528, 0x9509, 0xE5EE, 0xF5CF, 0xC5AC, 0xD58D, 0x3653, 0x2672, 0x1611, 0x0630, + 0x76D7, 0x66F6, 0x5695, 0x46B4, 0xB75B, 0xA77A, 0x9719, 0x8738, 0xF7DF, 0xE7FE, 0xD79D, 0xC7BC, 0x48C4, 0x58E5, + 0x6886, 0x78A7, 0x0840, 0x1861, 0x2802, 0x3823, 0xC9CC, 0xD9ED, 0xE98E, 0xF9AF, 0x8948, 0x9969, 0xA90A, 0xB92B, + 0x5AF5, 0x4AD4, 0x7AB7, 0x6A96, 0x1A71, 0x0A50, 0x3A33, 0x2A12, 0xDBFD, 0xCBDC, 0xFBBF, 0xEB9E, 0x9B79, 0x8B58, + 0xBB3B, 0xAB1A, 0x6CA6, 0x7C87, 0x4CE4, 0x5CC5, 0x2C22, 0x3C03, 0x0C60, 0x1C41, 0xEDAE, 0xFD8F, 0xCDEC, 0xDDCD, + 0xAD2A, 0xBD0B, 0x8D68, 0x9D49, 0x7E97, 0x6EB6, 0x5ED5, 0x4EF4, 0x3E13, 0x2E32, 0x1E51, 0x0E70, 0xFF9F, 0xEFBE, + 0xDFDD, 0xCFFC, 0xBF1B, 0xAF3A, 0x9F59, 0x8F78, 0x9188, 0x81A9, 0xB1CA, 0xA1EB, 0xD10C, 0xC12D, 0xF14E, 0xE16F, + 0x1080, 0x00A1, 0x30C2, 0x20E3, 0x5004, 0x4025, 0x7046, 0x6067, 0x83B9, 0x9398, 0xA3FB, 0xB3DA, 0xC33D, 0xD31C, + 0xE37F, 0xF35E, 0x02B1, 0x1290, 0x22F3, 0x32D2, 0x4235, 0x5214, 0x6277, 0x7256, 0xB5EA, 0xA5CB, 0x95A8, 0x8589, + 0xF56E, 0xE54F, 0xD52C, 0xC50D, 0x34E2, 0x24C3, 0x14A0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 0xA7DB, 0xB7FA, + 0x8799, 0x97B8, 0xE75F, 0xF77E, 0xC71D, 0xD73C, 0x26D3, 0x36F2, 0x0691, 0x16B0, 0x6657, 0x7676, 0x4615, 0x5634, + 0xD94C, 0xC96D, 0xF90E, 0xE92F, 0x99C8, 0x89E9, 0xB98A, 0xA9AB, 0x5844, 0x4865, 0x7806, 0x6827, 0x18C0, 0x08E1, + 0x3882, 0x28A3, 0xCB7D, 0xDB5C, 0xEB3F, 0xFB1E, 0x8BF9, 0x9BD8, 0xABBB, 0xBB9A, 0x4A75, 0x5A54, 0x6A37, 0x7A16, + 0x0AF1, 0x1AD0, 0x2AB3, 0x3A92, 0xFD2E, 0xED0F, 0xDD6C, 0xCD4D, 0xBDAA, 0xAD8B, 0x9DE8, 0x8DC9, 0x7C26, 0x6C07, + 0x5C64, 0x4C45, 0x3CA2, 0x2C83, 0x1CE0, 0x0CC1, 0xEF1F, 0xFF3E, 0xCF5D, 0xDF7C, 0xAF9B, 0xBFBA, 0x8FD9, 0x9FF8, + 0x6E17, 0x7E36, 0x4E55, 0x5E74, 0x2E93, 0x3EB2, 0x0ED1, 0x1EF0, }; public static int getSlot(String key) { int s = key.indexOf("{"); @@ -15,20 +39,25 @@ public class JedisClusterCRC16 { return getCRC16(key) % 16384; } - private static int getCRC16(String key) { + /** + * Create a CRC16 checksum from the bytes. + * implementation is from mp911de/lettuce + * + * @author Mark Paluch + * @param bytes + * @return CRC16 as integer value + */ + public static int getCRC16(byte[] bytes) { int crc = 0x0000; - for (byte b : key.getBytes()) { - for (int i = 0; i < 8; i++) { - boolean bit = ((b >> (7 - i) & 1) == 1); - boolean c15 = ((crc >> 15 & 1) == 1); - crc <<= 1; - // If coefficient of bit and remainder polynomial = 1 xor crc - // with polynomial - if (c15 ^ bit) - crc ^= polynomial; - } - } - return crc &= 0xffff ; + for (byte b : bytes) { + crc = ((crc << 8) ^ LOOKUP_TABLE[((crc >> 8) ^ (b & 0xFF)) & 0xFF]) & 0xFFFF; + } + return crc & 0xFFFF; } + + public static int getCRC16(String key) { + return getCRC16(key.getBytes()); + } + } \ No newline at end of file diff --git a/src/test/java/redis/clients/jedis/tests/benchmark/CRC16Benchmark.java b/src/test/java/redis/clients/jedis/tests/benchmark/CRC16Benchmark.java new file mode 100644 index 0000000..2b78ae8 --- /dev/null +++ b/src/test/java/redis/clients/jedis/tests/benchmark/CRC16Benchmark.java @@ -0,0 +1,27 @@ +package redis.clients.jedis.tests.benchmark; + +import java.util.Calendar; + +import redis.clients.util.JedisClusterCRC16; + +public class CRC16Benchmark { + private static final int TOTAL_OPERATIONS = 10000000; + + private static String[] TEST_SET = { + "", "123456789", "sfger132515", "hae9Napahngaikeethievubaibogiech", + "AAAAAAAAAAAAAAAAAAAAAA", "Hello, World!" + }; + + public static void main(String[] args) { + long begin = Calendar.getInstance().getTimeInMillis(); + + for (int n = 0; n <= TOTAL_OPERATIONS; n++) { + JedisClusterCRC16.getSlot(TEST_SET[n % TEST_SET.length]); + } + + long elapsed = Calendar.getInstance().getTimeInMillis() - begin; + + System.out.println(((1000 * TOTAL_OPERATIONS) / elapsed) + " ops"); + } + +} \ No newline at end of file diff --git a/src/test/java/redis/clients/jedis/tests/utils/JedisClusterCRC16Test.java b/src/test/java/redis/clients/jedis/tests/utils/JedisClusterCRC16Test.java new file mode 100644 index 0000000..7876d7e --- /dev/null +++ b/src/test/java/redis/clients/jedis/tests/utils/JedisClusterCRC16Test.java @@ -0,0 +1,43 @@ +package redis.clients.jedis.tests.utils; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import org.junit.Test; + +import redis.clients.util.JedisClusterCRC16; + +public class JedisClusterCRC16Test { + + @Test + public void testGetCRC16() throws Exception { + Map solutions = prepareSolutionSet(); + + for (Entry entry : solutions.entrySet()) { + // string version + assertEquals(entry.getValue().intValue(), JedisClusterCRC16.getCRC16(entry.getKey())); + + // byte array version + assertEquals(entry.getValue().intValue(), JedisClusterCRC16.getCRC16(entry.getKey().getBytes())); + } + } + + @Test + public void testGetSlot() { + assertEquals(7186, JedisClusterCRC16.getSlot("51")); + } + + private Map prepareSolutionSet() { + Map solutionMap = new HashMap(); + solutionMap.put("", 0x0); + solutionMap.put("123456789", 0x31C3); + solutionMap.put("sfger132515", 0xA45C); + solutionMap.put("hae9Napahngaikeethievubaibogiech", 0x58CE); + solutionMap.put("AAAAAAAAAAAAAAAAAAAAAA", 0x92cd); + solutionMap.put("Hello, World!", 0x4FD6); + return solutionMap; + } +} From 10307ecd2e2da6a2639855a955ae8a2bdfb26f14 Mon Sep 17 00:00:00 2001 From: Jungtaek Lim Date: Fri, 12 Sep 2014 18:00:35 +0900 Subject: [PATCH 2/2] Hide lookup table to prevent broken --- src/main/java/redis/clients/util/JedisClusterCRC16.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/redis/clients/util/JedisClusterCRC16.java b/src/main/java/redis/clients/util/JedisClusterCRC16.java index 552dc17..45d831e 100644 --- a/src/main/java/redis/clients/util/JedisClusterCRC16.java +++ b/src/main/java/redis/clients/util/JedisClusterCRC16.java @@ -8,7 +8,7 @@ package redis.clients.util; * Appendix A. CRC16 reference implementation in ANSI C */ public class JedisClusterCRC16 { - public static final int LOOKUP_TABLE[] = { 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50A5, 0x60C6, 0x70E7, 0x8108, 0x9129, + private static final int LOOKUP_TABLE[] = { 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50A5, 0x60C6, 0x70E7, 0x8108, 0x9129, 0xA14A, 0xB16B, 0xC18C, 0xD1AD, 0xE1CE, 0xF1EF, 0x1231, 0x0210, 0x3273, 0x2252, 0x52B5, 0x4294, 0x72F7, 0x62D6, 0x9339, 0x8318, 0xB37B, 0xA35A, 0xD3BD, 0xC39C, 0xF3FF, 0xE3DE, 0x2462, 0x3443, 0x0420, 0x1401, 0x64E6, 0x74C7, 0x44A4, 0x5485, 0xA56A, 0xB54B, 0x8528, 0x9509, 0xE5EE, 0xF5CF, 0xC5AC, 0xD58D, 0x3653, 0x2672, 0x1611, 0x0630,