From e2d814880236ef7dad06152033140d607d5b6aac Mon Sep 17 00:00:00 2001 From: Yaourt Date: Thu, 4 Nov 2010 15:59:50 +0100 Subject: [PATCH] Binary key & values seems to be implemented now --- .../clients/jedis/BinaryShardedJedis.java | 380 ++++++++++++++++++ .../redis/clients/jedis/ShardedJedis.java | 26 +- .../clients/jedis/ShardedJedisPipeline.java | 6 +- src/main/java/redis/clients/util/Hashing.java | 11 +- .../java/redis/clients/util/MurmurHash.java | 8 +- src/main/java/redis/clients/util/Sharded.java | 17 +- .../redis/clients/jedis/tests/JedisTest.java | 2 +- .../clients/jedis/tests/ProtocolTest.java | 2 +- 8 files changed, 417 insertions(+), 35 deletions(-) create mode 100644 src/main/java/redis/clients/jedis/BinaryShardedJedis.java diff --git a/src/main/java/redis/clients/jedis/BinaryShardedJedis.java b/src/main/java/redis/clients/jedis/BinaryShardedJedis.java new file mode 100644 index 0000000..4f6821d --- /dev/null +++ b/src/main/java/redis/clients/jedis/BinaryShardedJedis.java @@ -0,0 +1,380 @@ +package redis.clients.jedis; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import redis.clients.jedis.BinaryClient.LIST_POSITION; +import redis.clients.util.Hashing; +import redis.clients.util.Sharded; + + +public class BinaryShardedJedis extends Sharded implements BinaryJedisCommands { + public BinaryShardedJedis(List shards) { + super(shards); + } + + public BinaryShardedJedis(List shards, Hashing algo) { + super(shards, algo); + } + + public BinaryShardedJedis(List shards, Pattern keyTagPattern) { + super(shards, keyTagPattern); + } + + public BinaryShardedJedis(List shards, Hashing algo, + Pattern keyTagPattern) { + super(shards, algo, keyTagPattern); + } + + public void disconnect() throws IOException { + for (JedisShardInfo jedis : getAllShards()) { + jedis.getResource().disconnect(); + } + } + + protected Jedis create(JedisShardInfo shard) { + return new Jedis(shard); + } + + public String set(byte[] key, byte[] value) { + Jedis j = getShard(key); + return j.set(key, value); + } + + public byte[] get(byte[] key) { + Jedis j = getShard(key); + return j.get(key); + } + + public Integer exists(byte[] key) { + Jedis j = getShard(key); + return j.exists(key); + } + + public String type(byte[] key) { + Jedis j = getShard(key); + return j.type(key); + } + + public Integer expire(byte[] key, int seconds) { + Jedis j = getShard(key); + return j.expire(key, seconds); + } + + public Integer expireAt(byte[] key, long unixTime) { + Jedis j = getShard(key); + return j.expireAt(key, unixTime); + } + + public Integer ttl(byte[] key) { + Jedis j = getShard(key); + return j.ttl(key); + } + + public byte[] getSet(byte[] key, byte[] value) { + Jedis j = getShard(key); + return j.getSet(key, value); + } + + public Integer setnx(byte[] key, byte[] value) { + Jedis j = getShard(key); + return j.setnx(key, value); + } + + public String setex(byte[] key, int seconds, byte[] value) { + Jedis j = getShard(key); + return j.setex(key, seconds, value); + } + + public Integer decrBy(byte[] key, int integer) { + Jedis j = getShard(key); + return j.decrBy(key, integer); + } + + public Integer decr(byte[] key) { + Jedis j = getShard(key); + return j.decr(key); + } + + public Integer incrBy(byte[] key, int integer) { + Jedis j = getShard(key); + return j.incrBy(key, integer); + } + + public Integer incr(byte[] key) { + Jedis j = getShard(key); + return j.incr(key); + } + + public Integer append(byte[] key, byte[] value) { + Jedis j = getShard(key); + return j.append(key, value); + } + + public byte[] substr(byte[] key, int start, int end) { + Jedis j = getShard(key); + return j.substr(key, start, end); + } + + public Integer hset(byte[] key, byte[] field, byte[] value) { + Jedis j = getShard(key); + return j.hset(key, field, value); + } + + public byte[] hget(byte[] key, byte[] field) { + Jedis j = getShard(key); + return j.hget(key, field); + } + + public Integer hsetnx(byte[] key, byte[] field, byte[] value) { + Jedis j = getShard(key); + return j.hsetnx(key, field, value); + } + + public String hmset(byte[] key, Map hash) { + Jedis j = getShard(key); + return j.hmset(key, hash); + } + + public List hmget(byte[] key, byte[]... fields) { + Jedis j = getShard(key); + return j.hmget(key, fields); + } + + public Integer hincrBy(byte[] key, byte[] field, int value) { + Jedis j = getShard(key); + return j.hincrBy(key, field, value); + } + + public Integer hexists(byte[] key, byte[] field) { + Jedis j = getShard(key); + return j.hexists(key, field); + } + + public Integer hdel(byte[] key, byte[] field) { + Jedis j = getShard(key); + return j.hdel(key, field); + } + + public Integer hlen(byte[] key) { + Jedis j = getShard(key); + return j.hlen(key); + } + + public List hkeys(byte[] key) { + Jedis j = getShard(key); + return j.hkeys(key); + } + + public List hvals(byte[] key) { + Jedis j = getShard(key); + return j.hvals(key); + } + + public Map hgetAll(byte[] key) { + Jedis j = getShard(key); + return j.hgetAll(key); + } + + public Integer rpush(byte[] key, byte[] string) { + Jedis j = getShard(key); + return j.rpush(key, string); + } + + public Integer lpush(byte[] key, byte[] string) { + Jedis j = getShard(key); + return j.lpush(key, string); + } + + public Integer llen(byte[] key) { + Jedis j = getShard(key); + return j.llen(key); + } + + public List lrange(byte[] key, int start, int end) { + Jedis j = getShard(key); + return j.lrange(key, start, end); + } + + public String ltrim(byte[] key, int start, int end) { + Jedis j = getShard(key); + return j.ltrim(key, start, end); + } + + public byte[] lindex(byte[] key, int index) { + Jedis j = getShard(key); + return j.lindex(key, index); + } + + public String lset(byte[] key, int index, byte[] value) { + Jedis j = getShard(key); + return j.lset(key, index, value); + } + + public Integer lrem(byte[] key, int count, byte[] value) { + Jedis j = getShard(key); + return j.lrem(key, count, value); + } + + public byte[] lpop(byte[] key) { + Jedis j = getShard(key); + return j.lpop(key); + } + + public byte[] rpop(byte[] key) { + Jedis j = getShard(key); + return j.rpop(key); + } + + public Integer sadd(byte[] key, byte[] member) { + Jedis j = getShard(key); + return j.sadd(key, member); + } + + public Set smembers(byte[] key) { + Jedis j = getShard(key); + return j.smembers(key); + } + + public Integer srem(byte[] key, byte[] member) { + Jedis j = getShard(key); + return j.srem(key, member); + } + + public byte[] spop(byte[] key) { + Jedis j = getShard(key); + return j.spop(key); + } + + public Integer scard(byte[] key) { + Jedis j = getShard(key); + return j.scard(key); + } + + public Integer sismember(byte[] key, byte[] member) { + Jedis j = getShard(key); + return j.sismember(key, member); + } + + public byte[] srandmember(byte[] key) { + Jedis j = getShard(key); + return j.srandmember(key); + } + + public Integer zadd(byte[] key, double score, byte[] member) { + Jedis j = getShard(key); + return j.zadd(key, score, member); + } + + public Set zrange(byte[] key, int start, int end) { + Jedis j = getShard(key); + return j.zrange(key, start, end); + } + + public Integer zrem(byte[] key, byte[] member) { + Jedis j = getShard(key); + return j.zrem(key, member); + } + + public Double zincrby(byte[] key, double score, byte[] member) { + Jedis j = getShard(key); + return j.zincrby(key, score, member); + } + + public Integer zrank(byte[] key, byte[] member) { + Jedis j = getShard(key); + return j.zrank(key, member); + } + + public Integer zrevrank(byte[] key, byte[] member) { + Jedis j = getShard(key); + return j.zrevrank(key, member); + } + + public Set zrevrange(byte[] key, int start, int end) { + Jedis j = getShard(key); + return j.zrevrange(key, start, end); + } + + public Set zrangeWithScores(byte[] key, int start, int end) { + Jedis j = getShard(key); + return j.zrangeWithScores(key, start, end); + } + + public Set zrevrangeWithScores(byte[] key, int start, int end) { + Jedis j = getShard(key); + return j.zrevrangeWithScores(key, start, end); + } + + public Integer zcard(byte[] key) { + Jedis j = getShard(key); + return j.zcard(key); + } + + public Double zscore(byte[] key, byte[] member) { + Jedis j = getShard(key); + return j.zscore(key, member); + } + + public List sort(byte[] key) { + Jedis j = getShard(key); + return j.sort(key); + } + + public List sort(byte[] key, SortingParams sortingParameters) { + Jedis j = getShard(key); + return j.sort(key, sortingParameters); + } + + public Integer zcount(byte[] key, double min, double max) { + Jedis j = getShard(key); + return j.zcount(key, min, max); + } + + public Set zrangeByScore(byte[] key, double min, double max) { + Jedis j = getShard(key); + return j.zrangeByScore(key, min, max); + } + + public Set zrangeByScore(byte[] key, double min, double max, + int offset, int count) { + Jedis j = getShard(key); + return j.zrangeByScore(key, min, max, offset, count); + } + + public Set zrangeByScoreWithScores(byte[] key, double min, double max) { + Jedis j = getShard(key); + return j.zrangeByScoreWithScores(key, min, max); + } + + public Set zrangeByScoreWithScores(byte[] key, double min, + double max, int offset, int count) { + Jedis j = getShard(key); + return j.zrangeByScoreWithScores(key, min, max, offset, count); + } + + public Integer zremrangeByRank(byte[] key, int start, int end) { + Jedis j = getShard(key); + return j.zremrangeByRank(key, start, end); + } + + public Integer zremrangeByScore(byte[] key, double start, double end) { + Jedis j = getShard(key); + return j.zremrangeByScore(key, start, end); + } + + public Integer linsert(byte[] key, LIST_POSITION where, byte[] pivot, + byte[] value) { + Jedis j = getShard(key); + return j.linsert(key, where, pivot, value); + } + + public List pipelined(ShardedJedisPipeline shardedJedisPipeline) { + shardedJedisPipeline.setShardedJedis(this); + shardedJedisPipeline.execute(); + return shardedJedisPipeline.getResults(); + } +} \ No newline at end of file diff --git a/src/main/java/redis/clients/jedis/ShardedJedis.java b/src/main/java/redis/clients/jedis/ShardedJedis.java index a56f7dc..1fda008 100644 --- a/src/main/java/redis/clients/jedis/ShardedJedis.java +++ b/src/main/java/redis/clients/jedis/ShardedJedis.java @@ -1,17 +1,14 @@ package redis.clients.jedis; -import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; -import redis.clients.jedis.Client.LIST_POSITION; +import redis.clients.jedis.BinaryClient.LIST_POSITION; import redis.clients.util.Hashing; -import redis.clients.util.Sharded; -public class ShardedJedis extends Sharded implements - JedisCommands { +public class ShardedJedis extends BinaryShardedJedis implements JedisCommands { public ShardedJedis(List shards) { super(shards); } @@ -24,21 +21,10 @@ public class ShardedJedis extends Sharded implements super(shards, keyTagPattern); } - public ShardedJedis(List shards, Hashing algo, - Pattern keyTagPattern) { + public ShardedJedis(List shards, Hashing algo, Pattern keyTagPattern) { super(shards, algo, keyTagPattern); } - public void disconnect() throws IOException { - for (JedisShardInfo jedis : getAllShards()) { - jedis.getResource().disconnect(); - } - } - - protected Jedis create(JedisShardInfo shard) { - return new Jedis(shard); - } - public String set(String key, String value) { Jedis j = getShard(key); return j.set(key, value); @@ -371,10 +357,4 @@ public class ShardedJedis extends Sharded implements Jedis j = getShard(key); return j.linsert(key, where, pivot, value); } - - public List pipelined(ShardedJedisPipeline shardedJedisPipeline) { - shardedJedisPipeline.setShardedJedis(this); - shardedJedisPipeline.execute(); - return shardedJedisPipeline.getResults(); - } } \ No newline at end of file diff --git a/src/main/java/redis/clients/jedis/ShardedJedisPipeline.java b/src/main/java/redis/clients/jedis/ShardedJedisPipeline.java index c4e4881..ad57c0f 100644 --- a/src/main/java/redis/clients/jedis/ShardedJedisPipeline.java +++ b/src/main/java/redis/clients/jedis/ShardedJedisPipeline.java @@ -4,10 +4,10 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import redis.clients.jedis.Client.LIST_POSITION; +import redis.clients.jedis.BinaryClient.LIST_POSITION; public abstract class ShardedJedisPipeline { - private ShardedJedis jedis; + private BinaryShardedJedis jedis; private List results = new ArrayList(); private class FutureResult { @@ -22,7 +22,7 @@ public abstract class ShardedJedisPipeline { } } - public void setShardedJedis(ShardedJedis jedis) { + public void setShardedJedis(BinaryShardedJedis jedis) { this.jedis = jedis; } diff --git a/src/main/java/redis/clients/util/Hashing.java b/src/main/java/redis/clients/util/Hashing.java index 291416f..5eaa023 100644 --- a/src/main/java/redis/clients/util/Hashing.java +++ b/src/main/java/redis/clients/util/Hashing.java @@ -3,13 +3,19 @@ package redis.clients.util; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import redis.clients.jedis.Protocol; + public interface Hashing { public static final Hashing MURMUR_HASH = new MurmurHash(); public static final Hashing MD5 = new Hashing() { private MessageDigest md5 = null; // avoid recurring construction - + public long hash(String key) { + return hash(key.getBytes(Protocol.UTF8)); + } + + public long hash(byte[] key) { if (md5 == null) { try { md5 = MessageDigest.getInstance("MD5"); @@ -20,7 +26,7 @@ public interface Hashing { } md5.reset(); - md5.update(key.getBytes()); + md5.update(key); byte[] bKey = md5.digest(); long res = ((long) (bKey[3] & 0xFF) << 24) | ((long) (bKey[2] & 0xFF) << 16) @@ -30,4 +36,5 @@ public interface Hashing { }; public long hash(String key); + public long hash(byte[] key); } \ No newline at end of file diff --git a/src/main/java/redis/clients/util/MurmurHash.java b/src/main/java/redis/clients/util/MurmurHash.java index 0294a25..934f7cd 100644 --- a/src/main/java/redis/clients/util/MurmurHash.java +++ b/src/main/java/redis/clients/util/MurmurHash.java @@ -20,6 +20,8 @@ package redis.clients.util; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import redis.clients.jedis.Protocol; + /** * This is a very fast, non-cryptographic hash suitable for general hash-based * lookup. See http://murmurhash.googlepages.com/ for more details. @@ -156,7 +158,11 @@ public class MurmurHash implements Hashing { return h; } + public long hash(byte[] key) { + return hash64A(key, 0x1234ABCD); + } + public long hash(String key) { - return hash64A(key.getBytes(), 0x1234ABCD); + return hash(key.getBytes(Protocol.UTF8)); } } \ No newline at end of file diff --git a/src/main/java/redis/clients/util/Sharded.java b/src/main/java/redis/clients/util/Sharded.java index 9a4d7b7..d0f4f75 100644 --- a/src/main/java/redis/clients/util/Sharded.java +++ b/src/main/java/redis/clients/util/Sharded.java @@ -69,10 +69,19 @@ public class Sharded> { } } - public R getShard(String key) { - return nodes.floorEntry(algo.hash(getKeyTag(key))).getValue() - .getResource(); - } + public R getShard(byte[] key) { + return nodes + .floorEntry(algo.hash(key)) + .getValue() + .getResource(); + } + + public R getShard(String key) { + return nodes + .floorEntry(algo.hash(getKeyTag(key))) + .getValue() + .getResource(); + } public S getShardInfo(String key) { return nodes.floorEntry(algo.hash(getKeyTag(key))).getValue(); diff --git a/src/test/java/redis/clients/jedis/tests/JedisTest.java b/src/test/java/redis/clients/jedis/tests/JedisTest.java index 851c796..d1ebf0a 100644 --- a/src/test/java/redis/clients/jedis/tests/JedisTest.java +++ b/src/test/java/redis/clients/jedis/tests/JedisTest.java @@ -26,7 +26,7 @@ public class JedisTest extends JedisCommandTestBase { bigdata[b] = (byte) ((byte) b % 255); } Map hash = new HashMap(); - hash.put("data", new String(bigdata, RedisOutputStream.CHARSET)); + hash.put("data", new String(bigdata, Protocol.UTF8)); String status = jedis.hmset("foo", hash); assertEquals("OK", status); diff --git a/src/test/java/redis/clients/jedis/tests/ProtocolTest.java b/src/test/java/redis/clients/jedis/tests/ProtocolTest.java index f8f4723..0ba6f54 100644 --- a/src/test/java/redis/clients/jedis/tests/ProtocolTest.java +++ b/src/test/java/redis/clients/jedis/tests/ProtocolTest.java @@ -25,7 +25,7 @@ public class ProtocolTest extends Assert { PipedOutputStream pos = new PipedOutputStream(pis); Protocol protocol = new Protocol(); - protocol.sendCommand(new RedisOutputStream(pos), "GET", "SOMEKEY"); + protocol.sendCommand(new RedisOutputStream(pos), Protocol.Command.GET, "SOMEKEY".getBytes(Protocol.UTF8)); pos.close(); String expectedCommand = "*2\r\n$3\r\nGET\r\n$7\r\nSOMEKEY\r\n";