Added sharding benchmark and removed some imports

This commit is contained in:
Jonathan Leibiusky
2010-09-30 21:04:21 -03:00
parent 39618506e4
commit 6a1e141064
10 changed files with 2997 additions and 2945 deletions

View File

@@ -10,10 +10,7 @@ import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import redis.clients.jedis.Client;
import redis.clients.jedis.Client.LIST_POSITION; import redis.clients.jedis.Client.LIST_POSITION;
import redis.clients.util.RedisInputStream;
import redis.clients.util.ShardInfo;
public class Jedis { public class Jedis {
private Client client = null; private Client client = null;
@@ -992,10 +989,7 @@ public class Jedis {
* <p> * <p>
* Hint: the obvious use of LTRIM is together with LPUSH/RPUSH. For example: * Hint: the obvious use of LTRIM is together with LPUSH/RPUSH. For example:
* <p> * <p>
* {@code * {@code lpush("mylist", "someelement"); ltrim("mylist", 0, 99); * }
* lpush("mylist", "someelement");
* ltrim("mylist", 0, 99);
* }
* <p> * <p>
* The above two commands will push elements in the list taking care that * The above two commands will push elements in the list taking care that
* the list will not grow without limits. This is very useful when using * the list will not grow without limits. This is very useful when using
@@ -2298,7 +2292,9 @@ public class Jedis {
Set<Tuple> set = new LinkedHashSet<Tuple>(); Set<Tuple> set = new LinkedHashSet<Tuple>();
Iterator<String> iterator = membersWithScores.iterator(); Iterator<String> iterator = membersWithScores.iterator();
while (iterator.hasNext()) { while (iterator.hasNext()) {
set.add(new Tuple(iterator.next(), Double.valueOf(iterator.next()))); set
.add(new Tuple(iterator.next(), Double.valueOf(iterator
.next())));
} }
return set; return set;
} }

View File

@@ -1,7 +1,6 @@
package redis.clients.jedis; package redis.clients.jedis;
import redis.clients.util.FixedResourcePool; import redis.clients.util.FixedResourcePool;
import redis.clients.util.ShardInfo;
public class JedisPool extends FixedResourcePool<Jedis> { public class JedisPool extends FixedResourcePool<Jedis> {
private String host; private String host;

View File

@@ -8,7 +8,6 @@ import java.util.regex.Pattern;
import redis.clients.jedis.Client.LIST_POSITION; import redis.clients.jedis.Client.LIST_POSITION;
import redis.clients.util.Hashing; import redis.clients.util.Hashing;
import redis.clients.util.ShardInfo;
import redis.clients.util.Sharded; import redis.clients.util.Sharded;
public class ShardedJedis extends Sharded<Jedis, JedisShardInfo> { public class ShardedJedis extends Sharded<Jedis, JedisShardInfo> {
@@ -24,7 +23,8 @@ public class ShardedJedis extends Sharded<Jedis,JedisShardInfo> {
super(shards, keyTagPattern); super(shards, keyTagPattern);
} }
public ShardedJedis(List<JedisShardInfo> shards, Hashing algo, Pattern keyTagPattern) { public ShardedJedis(List<JedisShardInfo> shards, Hashing algo,
Pattern keyTagPattern) {
super(shards, algo, keyTagPattern); super(shards, algo, keyTagPattern);
} }

View File

@@ -14,16 +14,19 @@ public class Sharded<R, S extends ShardInfo<R>> {
private final Hashing algo; private final Hashing algo;
/** /**
* The default pattern used for extracting a key tag. * The default pattern used for extracting a key tag. The pattern must have
* The pattern must have a group (between parenthesis), which delimits the tag to be hashed. * a group (between parenthesis), which delimits the tag to be hashed. A
* A null pattern avoids applying the regular expression for each lookup, improving performance a little bit * null pattern avoids applying the regular expression for each lookup,
* is key tags aren't being used. * improving performance a little bit is key tags aren't being used.
*/ */
private Pattern tagPattern = null; private Pattern tagPattern = null;
public static final Pattern DEFAULT_KEY_TAG_PATTERN = Pattern.compile("\\{(.+?)\\}"); // the tag is anything between {} // the tag is anything between {}
public static final Pattern DEFAULT_KEY_TAG_PATTERN = Pattern
.compile("\\{(.+?)\\}");
public Sharded(List<S> shards) { public Sharded(List<S> shards) {
this(shards, Hashing.MURMUR_HASH); // MD5 is really not good as we works with 64-bits not 128 this(shards, Hashing.MURMUR_HASH); // MD5 is really not good as we works
// with 64-bits not 128
} }
public Sharded(List<S> shards, Hashing algo) { public Sharded(List<S> shards, Hashing algo) {
@@ -32,7 +35,9 @@ public class Sharded<R, S extends ShardInfo<R>> {
} }
public Sharded(List<S> shards, Pattern tagPattern) { public Sharded(List<S> shards, Pattern tagPattern) {
this(shards, Hashing.MURMUR_HASH, tagPattern); // MD5 is really not good as we works with 64-bits not 128 this(shards, Hashing.MURMUR_HASH, tagPattern); // MD5 is really not good
// as we works with
// 64-bits not 128
} }
public Sharded(List<S> shards, Hashing algo, Pattern tagPattern) { public Sharded(List<S> shards, Hashing algo, Pattern tagPattern) {
@@ -46,23 +51,27 @@ public class Sharded<R, S extends ShardInfo<R>> {
int totalWeight = 0; int totalWeight = 0;
for (ShardInfo shard : shards) { for (ShardInfo<?> shard : shards) {
totalWeight += shard.getWeight(); totalWeight += shard.getWeight();
} }
long oneForthOfStep = (1L << 62) / totalWeight; // 62 vs 64 to normalize math in Long long oneForthOfStep = (1L << 62) / totalWeight; // 62 vs 64 to normalize
// math in Long
long floor = Long.MIN_VALUE; long floor = Long.MIN_VALUE;
for (int i = 0; i != shards.size(); ++i) { for (int i = 0; i != shards.size(); ++i) {
final S shardInfo = shards.get(i); final S shardInfo = shards.get(i);
shardInfo.initResource(); shardInfo.initResource();
nodes.put(floor, shardInfo); nodes.put(floor, shardInfo);
floor += 4 * oneForthOfStep * shardInfo.getWeight(); // *4 to compensate 62 vs 64 floor += 4 * oneForthOfStep * shardInfo.getWeight(); // *4 to
// compensate
// 62 vs 64
} }
} }
public R getShard(String key) { public R getShard(String key) {
return nodes.floorEntry(algo.hash(getKeyTag(key))).getValue().getResource(); return nodes.floorEntry(algo.hash(getKeyTag(key))).getValue()
.getResource();
} }
public S getShardInfo(String key) { public S getShardInfo(String key) {
@@ -70,10 +79,11 @@ public class Sharded<R, S extends ShardInfo<R>> {
} }
/** /**
* A key tag is a special pattern inside a key that, if preset, is the only part of the key hashed * A key tag is a special pattern inside a key that, if preset, is the only
* in order to select the server for this key. * part of the key hashed in order to select the server for this key.
* *
* @see http://code.google.com/p/redis/wiki/FAQ#I'm_using_some_form_of_key_hashing_for_partitioning,_but_wh * @see http://code.google.com/p/redis/wiki/FAQ#I
* 'm_using_some_form_of_key_hashing_for_partitioning,_but_wh
* @param key * @param key
* @return The tag if it exists, or the original key * @return The tag if it exists, or the original key
*/ */

View File

@@ -11,7 +11,6 @@ import java.util.HashSet;
import java.util.Set; import java.util.Set;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test; import org.junit.Test;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;

View File

@@ -10,7 +10,6 @@ import redis.clients.jedis.JedisShardInfo;
import redis.clients.jedis.Protocol; import redis.clients.jedis.Protocol;
import redis.clients.jedis.tests.commands.JedisCommandTestBase; import redis.clients.jedis.tests.commands.JedisCommandTestBase;
import redis.clients.util.RedisOutputStream; import redis.clients.util.RedisOutputStream;
import redis.clients.util.ShardInfo;
public class JedisTest extends JedisCommandTestBase { public class JedisTest extends JedisCommandTestBase {
@Test @Test
@@ -36,7 +35,8 @@ public class JedisTest extends JedisCommandTestBase {
@Test @Test
public void connectWithShardInfo() { public void connectWithShardInfo() {
JedisShardInfo shardInfo = new JedisShardInfo("localhost", Protocol.DEFAULT_PORT); JedisShardInfo shardInfo = new JedisShardInfo("localhost",
Protocol.DEFAULT_PORT);
shardInfo.setPassword("foobared"); shardInfo.setPassword("foobared");
Jedis jedis = new Jedis(shardInfo); Jedis jedis = new Jedis(shardInfo);
jedis.get("foo"); jedis.get("foo");

View File

@@ -11,7 +11,6 @@ import org.junit.Test;
import redis.clients.jedis.Jedis; import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPipeline; import redis.clients.jedis.JedisPipeline;
import redis.clients.jedis.Protocol;
import redis.clients.jedis.tests.HostAndPortUtil.HostAndPort; import redis.clients.jedis.tests.HostAndPortUtil.HostAndPort;
public class PipeliningTest extends Assert { public class PipeliningTest extends Assert {

View File

@@ -2,7 +2,6 @@ package redis.clients.jedis.tests;
import java.io.BufferedInputStream; import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.io.PipedInputStream; import java.io.PipedInputStream;
@@ -50,13 +49,13 @@ public class ProtocolTest extends Assert {
@Test @Test
public void fragmentedBulkReply() { public void fragmentedBulkReply() {
FragmentedByteArrayInputStream fis = new FragmentedByteArrayInputStream("$30\r\n012345678901234567890123456789\r\n".getBytes()); FragmentedByteArrayInputStream fis = new FragmentedByteArrayInputStream(
"$30\r\n012345678901234567890123456789\r\n".getBytes());
Protocol protocol = new Protocol(); Protocol protocol = new Protocol();
String response = (String) protocol.read(new RedisInputStream(fis)); String response = (String) protocol.read(new RedisInputStream(fis));
assertEquals("012345678901234567890123456789", response); assertEquals("012345678901234567890123456789", response);
} }
@Test @Test
public void nullBulkReply() { public void nullBulkReply() {
InputStream is = new ByteArrayInputStream("$-1\r\n".getBytes()); InputStream is = new ByteArrayInputStream("$-1\r\n".getBytes());

View File

@@ -12,12 +12,12 @@ import redis.clients.jedis.JedisShardInfo;
import redis.clients.jedis.ShardedJedis; import redis.clients.jedis.ShardedJedis;
import redis.clients.jedis.tests.HostAndPortUtil.HostAndPort; import redis.clients.jedis.tests.HostAndPortUtil.HostAndPort;
import redis.clients.util.Hashing; import redis.clients.util.Hashing;
import redis.clients.util.ShardInfo;
import redis.clients.util.Sharded;
public class ShardedJedisTest extends Assert { public class ShardedJedisTest extends Assert {
private static HostAndPort redis1 = HostAndPortUtil.getRedisServers().get(0); private static HostAndPort redis1 = HostAndPortUtil.getRedisServers()
private static HostAndPort redis2 = HostAndPortUtil.getRedisServers().get(1); .get(0);
private static HostAndPort redis2 = HostAndPortUtil.getRedisServers()
.get(1);
@Test @Test
public void checkSharding() throws IOException { public void checkSharding() throws IOException {
@@ -25,8 +25,8 @@ public class ShardedJedisTest extends Assert {
shards.add(new JedisShardInfo(redis1.host, redis1.port)); shards.add(new JedisShardInfo(redis1.host, redis1.port));
shards.add(new JedisShardInfo(redis2.host, redis2.port)); shards.add(new JedisShardInfo(redis2.host, redis2.port));
ShardedJedis jedis = new ShardedJedis(shards); ShardedJedis jedis = new ShardedJedis(shards);
ShardInfo s1 = jedis.getShardInfo("a"); JedisShardInfo s1 = jedis.getShardInfo("a");
ShardInfo s2 = jedis.getShardInfo("b"); JedisShardInfo s2 = jedis.getShardInfo("b");
assertNotSame(s1, s2); assertNotSame(s1, s2);
} }
@@ -84,27 +84,29 @@ public class ShardedJedisTest extends Assert {
j.disconnect(); j.disconnect();
} }
/** @author muriloq@gmail.com */
@Test @Test
public void checkKeyTags() { public void checkKeyTags() {
List<JedisShardInfo> shards = new ArrayList<JedisShardInfo>(); List<JedisShardInfo> shards = new ArrayList<JedisShardInfo>();
shards.add(new JedisShardInfo(redis1.host, redis1.port)); shards.add(new JedisShardInfo(redis1.host, redis1.port));
shards.add(new JedisShardInfo(redis2.host, redis2.port)); shards.add(new JedisShardInfo(redis2.host, redis2.port));
ShardedJedis jedis = new ShardedJedis(shards, ShardedJedis.DEFAULT_KEY_TAG_PATTERN); ShardedJedis jedis = new ShardedJedis(shards,
ShardedJedis.DEFAULT_KEY_TAG_PATTERN);
assertEquals(jedis.getKeyTag("foo"), "foo"); assertEquals(jedis.getKeyTag("foo"), "foo");
assertEquals(jedis.getKeyTag("foo{bar}"), "bar"); assertEquals(jedis.getKeyTag("foo{bar}"), "bar");
assertEquals(jedis.getKeyTag("foo{bar}}"),"bar"); // default pattern is non greedy assertEquals(jedis.getKeyTag("foo{bar}}"), "bar"); // default pattern is
assertEquals(jedis.getKeyTag("{bar}foo"),"bar"); // Key tag may appear anywhere // non greedy
assertEquals(jedis.getKeyTag("f{bar}oo"),"bar"); // Key tag may appear anywhere assertEquals(jedis.getKeyTag("{bar}foo"), "bar"); // Key tag may appear
// anywhere
assertEquals(jedis.getKeyTag("f{bar}oo"), "bar"); // Key tag may appear
// anywhere
ShardInfo s1 = jedis.getShardInfo("abc{bar}"); JedisShardInfo s1 = jedis.getShardInfo("abc{bar}");
ShardInfo s2 = jedis.getShardInfo("foo{bar}"); JedisShardInfo s2 = jedis.getShardInfo("foo{bar}");
assertSame(s1, s2); assertSame(s1, s2);
ShardInfo s3 = jedis.getShardInfo("a"); JedisShardInfo s3 = jedis.getShardInfo("a");
ShardInfo s4 = jedis.getShardInfo("b"); JedisShardInfo s4 = jedis.getShardInfo("b");
assertNotSame(s3, s4); assertNotSame(s3, s4);
ShardedJedis jedis2 = new ShardedJedis(shards); ShardedJedis jedis2 = new ShardedJedis(shards);
@@ -112,9 +114,8 @@ public class ShardedJedisTest extends Assert {
assertEquals(jedis2.getKeyTag("foo"), "foo"); assertEquals(jedis2.getKeyTag("foo"), "foo");
assertNotSame(jedis2.getKeyTag("foo{bar}"), "bar"); assertNotSame(jedis2.getKeyTag("foo{bar}"), "bar");
ShardInfo s5 = jedis2.getShardInfo("foo{bar}"); JedisShardInfo s5 = jedis2.getShardInfo("foo{bar}");
ShardInfo s6 = jedis2.getShardInfo("abc{bar}"); JedisShardInfo s6 = jedis2.getShardInfo("abc{bar}");
assertNotSame(s5, s6); assertNotSame(s5, s6);
} }
} }

View File

@@ -0,0 +1,49 @@
package redis.clients.jedis.tests.benchmark;
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.List;
import redis.clients.jedis.JedisShardInfo;
import redis.clients.jedis.ShardedJedis;
import redis.clients.jedis.tests.HostAndPortUtil;
import redis.clients.jedis.tests.HostAndPortUtil.HostAndPort;
public class ShardedBenchmark {
private static HostAndPort hnp1 = HostAndPortUtil.getRedisServers().get(0);
private static HostAndPort hnp2 = HostAndPortUtil.getRedisServers().get(1);
private static final int TOTAL_OPERATIONS = 100000;
public static void main(String[] args) throws UnknownHostException,
IOException {
List<JedisShardInfo> shards = new ArrayList<JedisShardInfo>();
JedisShardInfo shard = new JedisShardInfo(hnp1.host, hnp1.port);
shard.setPassword("foobared");
shards.add(shard);
shard = new JedisShardInfo(hnp2.host, hnp2.port);
shard.setPassword("foobared");
shards.add(shard);
ShardedJedis jedis = new ShardedJedis(shards);
Collection<JedisShardInfo> allShards = jedis.getAllShards();
for (JedisShardInfo jedisShardInfo : allShards) {
jedisShardInfo.getResource().flushAll();
}
long begin = Calendar.getInstance().getTimeInMillis();
for (int n = 0; n <= TOTAL_OPERATIONS; n++) {
String key = "foo" + n;
jedis.set(key, "bar" + n);
jedis.get(key);
}
long elapsed = Calendar.getInstance().getTimeInMillis() - begin;
jedis.disconnect();
System.out.println(((1000 * 2 * TOTAL_OPERATIONS) / elapsed) + " ops");
}
}