rik/jlibredis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added sharding benchmark and removed some imports

Browse Source
This commit is contained in:
Jonathan Leibiusky
2010-09-30 21:04:21 -03:00
parent 39618506e4
commit 6a1e141064
10 changed files with 2997 additions and 2945 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5662
src/main/java/redis/clients/jedis/Jedis.java
View File

File diff suppressed because it is too large Load Diff

1
src/main/java/redis/clients/jedis/JedisPool.java
View File

@@ -1,7 +1,6 @@
package redis.clients.jedis;
import redis.clients.util.FixedResourcePool;
import redis.clients.util.ShardInfo;
public class JedisPool extends FixedResourcePool<Jedis> {
private String host;

12
src/main/java/redis/clients/jedis/ShardedJedis.java
View File

@@ -8,10 +8,9 @@ import java.util.regex.Pattern;
import redis.clients.jedis.Client.LIST_POSITION;
import redis.clients.util.Hashing;
import redis.clients.util.ShardInfo;
import redis.clients.util.Sharded;
public class ShardedJedis extends Sharded<Jedis,JedisShardInfo> {
public class ShardedJedis extends Sharded<Jedis, JedisShardInfo> {
public ShardedJedis(List<JedisShardInfo> shards) {
super(shards);
}
@@ -21,13 +20,14 @@ public class ShardedJedis extends Sharded<Jedis,JedisShardInfo> {
}
public ShardedJedis(List<JedisShardInfo> shards, Pattern keyTagPattern) {
super(shards, keyTagPattern);
super(shards, keyTagPattern);
}
public ShardedJedis(List<JedisShardInfo> shards, Hashing algo, Pattern keyTagPattern) {
super(shards, algo, keyTagPattern);
public ShardedJedis(List<JedisShardInfo> shards, Hashing algo,
Pattern keyTagPattern) {
super(shards, algo, keyTagPattern);
}
public String set(String key, String value) {
Jedis j = getShard(key);
return j.set(key, value);

100
src/main/java/redis/clients/util/Sharded.java
View File

@@ -8,85 +8,95 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Sharded<R, S extends ShardInfo<R>> {
public static final int DEFAULT_WEIGHT = 1;
public static final int DEFAULT_WEIGHT = 1;
private TreeMap<Long, S> nodes;
private final Hashing algo;
/**
* The default pattern used for extracting a key tag.
* The pattern must have a group (between parenthesis), which delimits the tag to be hashed.
* A null pattern avoids applying the regular expression for each lookup, improving performance a little bit
* is key tags aren't being used.
/**
* The default pattern used for extracting a key tag. The pattern must have
* a group (between parenthesis), which delimits the tag to be hashed. A
* null pattern avoids applying the regular expression for each lookup,
* improving performance a little bit is key tags aren't being used.
*/
private Pattern tagPattern = null;
public static final Pattern DEFAULT_KEY_TAG_PATTERN = Pattern.compile("\\{(.+?)\\}"); // the tag is anything between {}
private Pattern tagPattern = null;
// the tag is anything between {}
public static final Pattern DEFAULT_KEY_TAG_PATTERN = Pattern
.compile("\\{(.+?)\\}");
public Sharded(List<S> shards) {
this(shards, Hashing.MURMUR_HASH); // MD5 is really not good as we works with 64-bits not 128
this(shards, Hashing.MURMUR_HASH); // MD5 is really not good as we works
// with 64-bits not 128
}
public Sharded(List<S> shards, Hashing algo) {
this.algo = algo;
initialize(shards);
this.algo = algo;
initialize(shards);
}
public Sharded(List<S> shards, Pattern tagPattern) {
this(shards, Hashing.MURMUR_HASH, tagPattern); // MD5 is really not good as we works with 64-bits not 128
this(shards, Hashing.MURMUR_HASH, tagPattern); // MD5 is really not good
// as we works with
// 64-bits not 128
}
public Sharded(List<S> shards, Hashing algo, Pattern tagPattern) {
this.algo = algo;
this.tagPattern = tagPattern;
initialize(shards);
this.algo = algo;
this.tagPattern = tagPattern;
initialize(shards);
}
private void initialize(List<S> shards) {
nodes = new TreeMap<Long, S>();
nodes = new TreeMap<Long, S>();
int totalWeight = 0;
int totalWeight = 0;
for (ShardInfo shard : shards) {
totalWeight += shard.getWeight();
}
for (ShardInfo<?> shard : shards) {
totalWeight += shard.getWeight();
}
long oneForthOfStep = (1L << 62) / totalWeight; // 62 vs 64 to normalize math in Long
long oneForthOfStep = (1L << 62) / totalWeight; // 62 vs 64 to normalize
// math in Long
long floor = Long.MIN_VALUE;
for (int i = 0; i != shards.size(); ++i) {
final S shardInfo = shards.get(i);
shardInfo.initResource();
nodes.put(floor, shardInfo);
floor += 4 * oneForthOfStep * shardInfo.getWeight(); // *4 to compensate 62 vs 64
}
long floor = Long.MIN_VALUE;
for (int i = 0; i != shards.size(); ++i) {
final S shardInfo = shards.get(i);
shardInfo.initResource();
nodes.put(floor, shardInfo);
floor += 4 * oneForthOfStep * shardInfo.getWeight(); // *4 to
// compensate
// 62 vs 64
}
}
public R getShard(String key) {
return nodes.floorEntry(algo.hash(getKeyTag(key))).getValue().getResource();
return nodes.floorEntry(algo.hash(getKeyTag(key))).getValue()
.getResource();
}
public S getShardInfo(String key) {
return nodes.floorEntry(algo.hash(getKeyTag(key))).getValue();
return nodes.floorEntry(algo.hash(getKeyTag(key))).getValue();
}
/**
* A key tag is a special pattern inside a key that, if preset, is the only part of the key hashed
* in order to select the server for this key.
* A key tag is a special pattern inside a key that, if preset, is the only
* part of the key hashed in order to select the server for this key.
*
* @see http://code.google.com/p/redis/wiki/FAQ#I'm_using_some_form_of_key_hashing_for_partitioning,_but_wh
* @see http://code.google.com/p/redis/wiki/FAQ#I
* 'm_using_some_form_of_key_hashing_for_partitioning,_but_wh
* @param key
* @return The tag if it exists, or the original key
* @return The tag if it exists, or the original key
*/
public String getKeyTag(String key){
if (tagPattern != null){
Matcher m = tagPattern.matcher(key);
if (m.find())
return m.group(1);
}
return key;
public String getKeyTag(String key) {
if (tagPattern != null) {
Matcher m = tagPattern.matcher(key);
if (m.find())
return m.group(1);
}
return key;
}
public Collection<S> getAllShards() {
return Collections.unmodifiableCollection(nodes.values());
return Collections.unmodifiableCollection(nodes.values());
}
}

1
src/test/java/redis/clients/jedis/tests/JedisNewCommandsCheckTest.java
View File

@@ -11,7 +11,6 @@ import java.util.HashSet;
import java.util.Set;
import org.junit.Assert;
import org.junit.Ignore;
import org.junit.Test;
import redis.clients.jedis.Jedis;

4
src/test/java/redis/clients/jedis/tests/JedisTest.java
View File

@@ -10,7 +10,6 @@ import redis.clients.jedis.JedisShardInfo;
import redis.clients.jedis.Protocol;
import redis.clients.jedis.tests.commands.JedisCommandTestBase;
import redis.clients.util.RedisOutputStream;
import redis.clients.util.ShardInfo;
public class JedisTest extends JedisCommandTestBase {
@Test
@@ -36,7 +35,8 @@ public class JedisTest extends JedisCommandTestBase {
@Test
public void connectWithShardInfo() {
JedisShardInfo shardInfo = new JedisShardInfo("localhost", Protocol.DEFAULT_PORT);
JedisShardInfo shardInfo = new JedisShardInfo("localhost",
Protocol.DEFAULT_PORT);
shardInfo.setPassword("foobared");
Jedis jedis = new Jedis(shardInfo);
jedis.get("foo");

21
src/test/java/redis/clients/jedis/tests/PipeliningTest.java
View File

@@ -11,21 +11,20 @@ import org.junit.Test;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPipeline;
import redis.clients.jedis.Protocol;
import redis.clients.jedis.tests.HostAndPortUtil.HostAndPort;
public class PipeliningTest extends Assert {
private static HostAndPort hnp = HostAndPortUtil.getRedisServers().get(0);
private Jedis jedis;
private static HostAndPort hnp = HostAndPortUtil.getRedisServers().get(0);
@Before
public void setUp() throws Exception {
jedis = new Jedis(hnp.host, hnp.port, 500);
jedis.connect();
jedis.auth("foobared");
jedis.flushAll();
}
private Jedis jedis;
@Before
public void setUp() throws Exception {
jedis = new Jedis(hnp.host, hnp.port, 500);
jedis.connect();
jedis.auth("foobared");
jedis.flushAll();
}
@Test
public void pipeline() throws UnknownHostException, IOException {

11
src/test/java/redis/clients/jedis/tests/ProtocolTest.java
View File

@@ -2,7 +2,6 @@ package redis.clients.jedis.tests;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.PipedInputStream;
@@ -50,13 +49,13 @@ public class ProtocolTest extends Assert {
@Test
public void fragmentedBulkReply() {
FragmentedByteArrayInputStream fis = new FragmentedByteArrayInputStream("$30\r\n012345678901234567890123456789\r\n".getBytes());
Protocol protocol = new Protocol();
String response = (String) protocol.read(new RedisInputStream(fis));
assertEquals("012345678901234567890123456789", response);
FragmentedByteArrayInputStream fis = new FragmentedByteArrayInputStream(
"$30\r\n012345678901234567890123456789\r\n".getBytes());
Protocol protocol = new Protocol();
String response = (String) protocol.read(new RedisInputStream(fis));
assertEquals("012345678901234567890123456789", response);
}
@Test
public void nullBulkReply() {
InputStream is = new ByteArrayInputStream("$-1\r\n".getBytes());

81
src/test/java/redis/clients/jedis/tests/ShardedJedisTest.java
View File

@@ -12,21 +12,21 @@ import redis.clients.jedis.JedisShardInfo;
import redis.clients.jedis.ShardedJedis;
import redis.clients.jedis.tests.HostAndPortUtil.HostAndPort;
import redis.clients.util.Hashing;
import redis.clients.util.ShardInfo;
import redis.clients.util.Sharded;
public class ShardedJedisTest extends Assert {
private static HostAndPort redis1 = HostAndPortUtil.getRedisServers().get(0);
private static HostAndPort redis2 = HostAndPortUtil.getRedisServers().get(1);
private static HostAndPort redis1 = HostAndPortUtil.getRedisServers()
.get(0);
private static HostAndPort redis2 = HostAndPortUtil.getRedisServers()
.get(1);
@Test
public void checkSharding() throws IOException {
List<JedisShardInfo> shards = new ArrayList<JedisShardInfo>();
shards.add(new JedisShardInfo(redis1.host, redis1.port));
shards.add(new JedisShardInfo(redis2.host, redis2.port));
ShardedJedis jedis = new ShardedJedis(shards);
ShardInfo s1 = jedis.getShardInfo("a");
ShardInfo s2 = jedis.getShardInfo("b");
JedisShardInfo s1 = jedis.getShardInfo("a");
JedisShardInfo s2 = jedis.getShardInfo("b");
assertNotSame(s1, s2);
}
@@ -83,38 +83,39 @@ public class ShardedJedisTest extends Assert {
assertEquals("bar1", j.get("b"));
j.disconnect();
}
/** @author muriloq@gmail.com */
@Test
public void checkKeyTags(){
List<JedisShardInfo> shards = new ArrayList<JedisShardInfo>();
shards.add(new JedisShardInfo(redis1.host, redis1.port));
shards.add(new JedisShardInfo(redis2.host, redis2.port));
ShardedJedis jedis = new ShardedJedis(shards, ShardedJedis.DEFAULT_KEY_TAG_PATTERN);
assertEquals(jedis.getKeyTag("foo"),"foo");
assertEquals(jedis.getKeyTag("foo{bar}"),"bar");
assertEquals(jedis.getKeyTag("foo{bar}}"),"bar"); // default pattern is non greedy
assertEquals(jedis.getKeyTag("{bar}foo"),"bar"); // Key tag may appear anywhere
assertEquals(jedis.getKeyTag("f{bar}oo"),"bar"); // Key tag may appear anywhere
ShardInfo s1 = jedis.getShardInfo("abc{bar}");
ShardInfo s2 = jedis.getShardInfo("foo{bar}");
assertSame(s1, s2);
ShardInfo s3 = jedis.getShardInfo("a");
ShardInfo s4 = jedis.getShardInfo("b");
assertNotSame(s3, s4);
ShardedJedis jedis2 = new ShardedJedis(shards);
assertEquals(jedis2.getKeyTag("foo"),"foo");
assertNotSame(jedis2.getKeyTag("foo{bar}"),"bar");
ShardInfo s5 = jedis2.getShardInfo("foo{bar}");
ShardInfo s6 = jedis2.getShardInfo("abc{bar}");
assertNotSame(s5, s6);
}
@Test
public void checkKeyTags() {
List<JedisShardInfo> shards = new ArrayList<JedisShardInfo>();
shards.add(new JedisShardInfo(redis1.host, redis1.port));
shards.add(new JedisShardInfo(redis2.host, redis2.port));
ShardedJedis jedis = new ShardedJedis(shards,
ShardedJedis.DEFAULT_KEY_TAG_PATTERN);
assertEquals(jedis.getKeyTag("foo"), "foo");
assertEquals(jedis.getKeyTag("foo{bar}"), "bar");
assertEquals(jedis.getKeyTag("foo{bar}}"), "bar"); // default pattern is
// non greedy
assertEquals(jedis.getKeyTag("{bar}foo"), "bar"); // Key tag may appear
// anywhere
assertEquals(jedis.getKeyTag("f{bar}oo"), "bar"); // Key tag may appear
// anywhere
JedisShardInfo s1 = jedis.getShardInfo("abc{bar}");
JedisShardInfo s2 = jedis.getShardInfo("foo{bar}");
assertSame(s1, s2);
JedisShardInfo s3 = jedis.getShardInfo("a");
JedisShardInfo s4 = jedis.getShardInfo("b");
assertNotSame(s3, s4);
ShardedJedis jedis2 = new ShardedJedis(shards);
assertEquals(jedis2.getKeyTag("foo"), "foo");
assertNotSame(jedis2.getKeyTag("foo{bar}"), "bar");
JedisShardInfo s5 = jedis2.getShardInfo("foo{bar}");
JedisShardInfo s6 = jedis2.getShardInfo("abc{bar}");
assertNotSame(s5, s6);
}
}

49
src/test/java/redis/clients/jedis/tests/benchmark/ShardedBenchmark.java Normal file
View File

@@ -0,0 +1,49 @@
package redis.clients.jedis.tests.benchmark;
import java.io.IOException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.List;
import redis.clients.jedis.JedisShardInfo;
import redis.clients.jedis.ShardedJedis;
import redis.clients.jedis.tests.HostAndPortUtil;
import redis.clients.jedis.tests.HostAndPortUtil.HostAndPort;
public class ShardedBenchmark {
private static HostAndPort hnp1 = HostAndPortUtil.getRedisServers().get(0);
private static HostAndPort hnp2 = HostAndPortUtil.getRedisServers().get(1);
private static final int TOTAL_OPERATIONS = 100000;
public static void main(String[] args) throws UnknownHostException,
IOException {
List<JedisShardInfo> shards = new ArrayList<JedisShardInfo>();
JedisShardInfo shard = new JedisShardInfo(hnp1.host, hnp1.port);
shard.setPassword("foobared");
shards.add(shard);
shard = new JedisShardInfo(hnp2.host, hnp2.port);
shard.setPassword("foobared");
shards.add(shard);
ShardedJedis jedis = new ShardedJedis(shards);
Collection<JedisShardInfo> allShards = jedis.getAllShards();
for (JedisShardInfo jedisShardInfo : allShards) {
jedisShardInfo.getResource().flushAll();
}
long begin = Calendar.getInstance().getTimeInMillis();
for (int n = 0; n <= TOTAL_OPERATIONS; n++) {
String key = "foo" + n;
jedis.set(key, "bar" + n);
jedis.get(key);
}
long elapsed = Calendar.getInstance().getTimeInMillis() - begin;
jedis.disconnect();
System.out.println(((1000 * 2 * TOTAL_OPERATIONS) / elapsed) + " ops");
}
}