Added sharding using ketama

This commit is contained in:
Jonathan Leibiusky
2010-09-14 01:30:59 -03:00
parent f9e7887e02
commit baab2225f6
5 changed files with 610 additions and 0 deletions

View File

@@ -0,0 +1,98 @@
package redis.clients.util;
import redis.clients.jedis.Protocol;
public class ShardInfo {
@Override
public String toString() {
return "ShardInfo [host=" + host + ", port=" + port + ", weight="
+ weight + "]";
}
private String host;
private int port;
private int timeout;
private int weight;
private String password = null;
public String getHost() {
return host;
}
public int getPort() {
return port;
}
public int getTimeout() {
return timeout;
}
public ShardInfo(String host) {
this(host, Protocol.DEFAULT_PORT);
}
public ShardInfo(String host, int port) {
this(host, port, 2000);
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((host == null) ? 0 : host.hashCode());
result = prime * result + port;
result = prime * result + timeout;
result = prime * result + weight;
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
ShardInfo other = (ShardInfo) obj;
if (host == null) {
if (other.host != null)
return false;
} else if (!host.equals(other.host))
return false;
if (port != other.port)
return false;
if (timeout != other.timeout)
return false;
if (weight != other.weight)
return false;
return true;
}
public ShardInfo(String host, int port, int timeout) {
this(host, port, timeout, Sharded.DEFAULT_WEIGHT);
}
public ShardInfo(String host, int port, int timeout, int weight) {
this.host = host;
this.port = port;
this.timeout = timeout;
this.weight = weight;
}
public String getPassword() {
return password;
}
public void setPassword(String auth) {
this.password = auth;
}
public void setTimeout(int timeout) {
this.timeout = timeout;
}
public int getWeight() {
return this.weight;
}
}

View File

@@ -0,0 +1,102 @@
package redis.clients.util;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
public abstract class Sharded<T> {
public static final int DEFAULT_WEIGHT = 1;
private static MessageDigest md5 = null; // avoid recurring construction
private TreeMap<Long, ShardInfo> nodes;
private int totalWeight;
private Map<ShardInfo, T> resources;
public Sharded(List<ShardInfo> shards) {
initialize(shards);
}
private void initialize(List<ShardInfo> shards) {
nodes = new TreeMap<Long, ShardInfo>();
resources = new HashMap<ShardInfo, T>();
totalWeight = 0;
for (ShardInfo shard : shards) {
totalWeight += shard.getWeight();
}
MessageDigest md5;
try {
md5 = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("++++ no md5 algorythm found");
}
for (ShardInfo shard : shards) {
double factor = Math
.floor(((double) (40 * shards.size() * DEFAULT_WEIGHT))
/ (double) totalWeight);
for (long j = 0; j < factor; j++) {
byte[] d = md5.digest((shard.toString() + "-" + j).getBytes());
for (int h = 0; h < 4; h++) {
Long k = ((long) (d[3 + h * 4] & 0xFF) << 24)
| ((long) (d[2 + h * 4] & 0xFF) << 16)
| ((long) (d[1 + h * 4] & 0xFF) << 8)
| ((long) (d[0 + h * 4] & 0xFF));
nodes.put(k, shard);
}
}
resources.put(shard, create(shard));
}
}
public ShardInfo getShardInfo(String key) {
long hv = calculateHash(key);
return nodes.get(findPointFor(hv));
}
private Long calculateHash(String key) {
if (md5 == null) {
try {
md5 = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("++++ no md5 algorythm found");
}
}
md5.reset();
md5.update(key.getBytes());
byte[] bKey = md5.digest();
long res = ((long) (bKey[3] & 0xFF) << 24)
| ((long) (bKey[2] & 0xFF) << 16)
| ((long) (bKey[1] & 0xFF) << 8) | (long) (bKey[0] & 0xFF);
return res;
}
private Long findPointFor(Long hashK) {
Long k = nodes.ceilingKey(hashK);
if (k == null) {
k = nodes.firstKey();
}
return k;
}
public T getShard(String key) {
ShardInfo shard = getShardInfo(key);
return resources.get(shard);
}
protected abstract T create(ShardInfo shard);
public Collection<T> getAllShards() {
return resources.values();
}
}