/[Apache-SVN]/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/JenkinsHash.java
ViewVC logotype

Contents of /hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/util/JenkinsHash.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 689888 - (show annotations)
Thu Aug 28 16:49:51 2008 UTC (14 months, 4 weeks ago) by stack
File size: 11179 byte(s)
HBASE-849   Speed improvement in JenkinsHash
1 /**
2 * Copyright 2007 The Apache Software Foundation
3 *
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
11 *
12 * http://www.apache.org/licenses/LICENSE-2.0
13 *
14 * Unless required by applicable law or agreed to in writing, software
15 * distributed under the License is distributed on an "AS IS" BASIS,
16 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17 * See the License for the specific language governing permissions and
18 * limitations under the License.
19 */
20
21 package org.apache.hadoop.hbase.util;
22
23 import java.io.FileInputStream;
24 import java.io.IOException;
25
26 /**
27 * Produces 32-bit hash for hash table lookup.
28 *
29 * <pre>lookup3.c, by Bob Jenkins, May 2006, Public Domain.
30 *
31 * You can use this free for any purpose. It's in the public domain.
32 * It has no warranty.
33 * </pre>
34 *
35 * @see <a href="http://burtleburtle.net/bob/c/lookup3.c">lookup3.c</a>
36 * @see <a href="http://www.ddj.com/184410284">Hash Functions (and how this
37 * function compares to others such as CRC, MD?, etc</a>
38 * @see <a href="http://burtleburtle.net/bob/hash/doobs.html">Has update on the
39 * Dr. Dobbs Article</a>
40 */
41 public class JenkinsHash {
42 private static long INT_MASK = 0x00000000ffffffffL;
43 private static long BYTE_MASK = 0x00000000000000ffL;
44
45 private static long rot(long val, int pos) {
46 return ((long)(Integer.rotateLeft(
47 (int)(val & INT_MASK), pos)) & INT_MASK);
48 }
49
50 /**
51 * Alternate form for hashing an entire byte array
52 *
53 * @param bytes
54 * @return hash value
55 */
56 public static int hash(byte[] bytes) {
57 return hash(bytes, bytes.length, -1);
58 }
59
60 /**
61 * Alternate form for hashing an entire byte array
62 *
63 * @param bytes
64 * @param initval
65 * @return hash value
66 */
67 public static int hash(byte[] bytes, int initval) {
68 return hash(bytes, bytes.length, initval);
69 }
70
71 /**
72 * taken from hashlittle() -- hash a variable-length key into a 32-bit value
73 *
74 * @param key the key (the unaligned variable-length array of bytes)
75 * @param nbytes number of bytes to include in hash
76 * @param initval can be any integer value
77 * @return a 32-bit value. Every bit of the key affects every bit of the
78 * return value. Two keys differing by one or two bits will have totally
79 * different hash values.
80 *
81 * <p>The best hash table sizes are powers of 2. There is no need to do mod
82 * a prime (mod is sooo slow!). If you need less than 32 bits, use a bitmask.
83 * For example, if you need only 10 bits, do
84 * <code>h = (h & hashmask(10));</code>
85 * In which case, the hash table should have hashsize(10) elements.
86 *
87 * <p>If you are hashing n strings byte[][] k, do it like this:
88 * for (int i = 0, h = 0; i < n; ++i) h = hash( k[i], h);
89 *
90 * <p>By Bob Jenkins, 2006. bob_jenkins@burtleburtle.net. You may use this
91 * code any way you wish, private, educational, or commercial. It's free.
92 *
93 * <p>Use for hash table lookup, or anything where one collision in 2^^32 is
94 * acceptable. Do NOT use for cryptographic purposes.
95 */
96 public static int hash(byte[] key, int nbytes, int initval) {
97 int length = nbytes;
98 long a, b, c; // We use longs because we don't have unsigned ints
99 a = b = c = (0x00000000deadbeefL + length + initval) & INT_MASK;
100 int offset = 0;
101 for (; length > 12; offset += 12, length -= 12) {
102 a = (a + (key[offset + 0] & BYTE_MASK)) & INT_MASK;
103 a = (a + (((key[offset + 1] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
104 a = (a + (((key[offset + 2] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
105 a = (a + (((key[offset + 3] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
106 b = (b + (key[offset + 4] & BYTE_MASK)) & INT_MASK;
107 b = (b + (((key[offset + 5] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
108 b = (b + (((key[offset + 6] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
109 b = (b + (((key[offset + 7] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
110 c = (c + (key[offset + 8] & BYTE_MASK)) & INT_MASK;
111 c = (c + (((key[offset + 9] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
112 c = (c + (((key[offset + 10] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
113 c = (c + (((key[offset + 11] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
114
115 /*
116 * mix -- mix 3 32-bit values reversibly.
117 * This is reversible, so any information in (a,b,c) before mix() is
118 * still in (a,b,c) after mix().
119 *
120 * If four pairs of (a,b,c) inputs are run through mix(), or through
121 * mix() in reverse, there are at least 32 bits of the output that
122 * are sometimes the same for one pair and different for another pair.
123 *
124 * This was tested for:
125 * - pairs that differed by one bit, by two bits, in any combination
126 * of top bits of (a,b,c), or in any combination of bottom bits of
127 * (a,b,c).
128 * - "differ" is defined as +, -, ^, or ~^. For + and -, I transformed
129 * the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
130 * is commonly produced by subtraction) look like a single 1-bit
131 * difference.
132 * - the base values were pseudorandom, all zero but one bit set, or
133 * all zero plus a counter that starts at zero.
134 *
135 * Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that
136 * satisfy this are
137 * 4 6 8 16 19 4
138 * 9 15 3 18 27 15
139 * 14 9 3 7 17 3
140 * Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing for
141 * "differ" defined as + with a one-bit base and a two-bit delta. I
142 * used http://burtleburtle.net/bob/hash/avalanche.html to choose
143 * the operations, constants, and arrangements of the variables.
144 *
145 * This does not achieve avalanche. There are input bits of (a,b,c)
146 * that fail to affect some output bits of (a,b,c), especially of a.
147 * The most thoroughly mixed value is c, but it doesn't really even
148 * achieve avalanche in c.
149 *
150 * This allows some parallelism. Read-after-writes are good at doubling
151 * the number of bits affected, so the goal of mixing pulls in the
152 * opposite direction as the goal of parallelism. I did what I could.
153 * Rotates seem to cost as much as shifts on every machine I could lay
154 * my hands on, and rotates are much kinder to the top and bottom bits,
155 * so I used rotates.
156 *
157 * #define mix(a,b,c) \
158 * { \
159 * a -= c; a ^= rot(c, 4); c += b; \
160 * b -= a; b ^= rot(a, 6); a += c; \
161 * c -= b; c ^= rot(b, 8); b += a; \
162 * a -= c; a ^= rot(c,16); c += b; \
163 * b -= a; b ^= rot(a,19); a += c; \
164 * c -= b; c ^= rot(b, 4); b += a; \
165 * }
166 *
167 * mix(a,b,c);
168 */
169 a = (a - c) & INT_MASK; a ^= rot(c, 4); c = (c + b) & INT_MASK;
170 b = (b - a) & INT_MASK; b ^= rot(a, 6); a = (a + c) & INT_MASK;
171 c = (c - b) & INT_MASK; c ^= rot(b, 8); b = (b + a) & INT_MASK;
172 a = (a - c) & INT_MASK; a ^= rot(c,16); c = (c + b) & INT_MASK;
173 b = (b - a) & INT_MASK; b ^= rot(a,19); a = (a + c) & INT_MASK;
174 c = (c - b) & INT_MASK; c ^= rot(b, 4); b = (b + a) & INT_MASK;
175 }
176
177 //-------------------------------- last block: affect all 32 bits of (c)
178 switch (length) { // all the case statements fall through
179 case 12:
180 c = (c + (((key[offset + 11] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
181 case 11:
182 c = (c + (((key[offset + 10] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
183 case 10:
184 c = (c + (((key[offset + 9] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
185 case 9:
186 c = (c + (key[offset + 8] & BYTE_MASK)) & INT_MASK;
187 case 8:
188 b = (b + (((key[offset + 7] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
189 case 7:
190 b = (b + (((key[offset + 6] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
191 case 6:
192 b = (b + (((key[offset + 5] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
193 case 5:
194 b = (b + (key[offset + 4] & BYTE_MASK)) & INT_MASK;
195 case 4:
196 a = (a + (((key[offset + 3] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
197 case 3:
198 a = (a + (((key[offset + 2] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
199 case 2:
200 a = (a + (((key[offset + 1] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
201 case 1:
202 a = (a + (key[offset + 0] & BYTE_MASK)) & INT_MASK;
203 break;
204 case 0:
205 return (int)(c & INT_MASK);
206 }
207 /*
208 * final -- final mixing of 3 32-bit values (a,b,c) into c
209 *
210 * Pairs of (a,b,c) values differing in only a few bits will usually
211 * produce values of c that look totally different. This was tested for
212 * - pairs that differed by one bit, by two bits, in any combination
213 * of top bits of (a,b,c), or in any combination of bottom bits of
214 * (a,b,c).
215 *
216 * - "differ" is defined as +, -, ^, or ~^. For + and -, I transformed
217 * the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
218 * is commonly produced by subtraction) look like a single 1-bit
219 * difference.
220 *
221 * - the base values were pseudorandom, all zero but one bit set, or
222 * all zero plus a counter that starts at zero.
223 *
224 * These constants passed:
225 * 14 11 25 16 4 14 24
226 * 12 14 25 16 4 14 24
227 * and these came close:
228 * 4 8 15 26 3 22 24
229 * 10 8 15 26 3 22 24
230 * 11 8 15 26 3 22 24
231 *
232 * #define final(a,b,c) \
233 * {
234 * c ^= b; c -= rot(b,14); \
235 * a ^= c; a -= rot(c,11); \
236 * b ^= a; b -= rot(a,25); \
237 * c ^= b; c -= rot(b,16); \
238 * a ^= c; a -= rot(c,4); \
239 * b ^= a; b -= rot(a,14); \
240 * c ^= b; c -= rot(b,24); \
241 * }
242 *
243 */
244 c ^= b; c = (c - rot(b,14)) & INT_MASK;
245 a ^= c; a = (a - rot(c,11)) & INT_MASK;
246 b ^= a; b = (b - rot(a,25)) & INT_MASK;
247 c ^= b; c = (c - rot(b,16)) & INT_MASK;
248 a ^= c; a = (a - rot(c,4)) & INT_MASK;
249 b ^= a; b = (b - rot(a,14)) & INT_MASK;
250 c ^= b; c = (c - rot(b,24)) & INT_MASK;
251
252 return (int)(c & INT_MASK);
253 }
254
255 /**
256 * Compute the hash of the specified file
257 * @param args name of file to compute hash of.
258 * @throws IOException
259 */
260 public static void main(String[] args) throws IOException {
261 if (args.length != 1) {
262 System.err.println("Usage: JenkinsHash filename");
263 System.exit(-1);
264 }
265 FileInputStream in = new FileInputStream(args[0]);
266 byte[] bytes = new byte[512];
267 int value = 0;
268 for (int length = in.read(bytes); length > 0 ; length = in.read(bytes)) {
269 value = hash(bytes, length, value);
270 }
271 System.out.println(Math.abs(value));
272 }
273 }

apache@apache.org
ViewVC Help
Powered by ViewVC 1.1.2