hbase/util/JenkinsHash.java

/**
 * Copyright 2007 The Apache Software Foundation
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.hadoop.hbase.util;

import java.io.FileInputStream;
import java.io.IOException;

/**
 * lookup3.c, by Bob Jenkins, May 2006, Public Domain.
 * <a href="http://burtleburtle.net/bob/c/lookup3.c">lookup3.c</a>
 *
 * You can use this free for any purpose.  It's in the public domain.
 * It has no warranty.
 * 
 * Produces 32-bit hash for hash table lookup.
 */
public class JenkinsHash {
  private static long INT_MASK  = 0x00000000ffffffffL;
  private static long BYTE_MASK = 0x00000000000000ffL;

  private static long rot(long val, int pos) {
    return Long.valueOf(Integer.rotateLeft(
        Long.valueOf(val & INT_MASK).intValue(), pos)).longValue() & INT_MASK;
  }

  /**
   * Alternate form for hashing an entire byte array
   * 
   * @param bytes 
   * @param initval
   * @return hash value
   */
  public static int hash(byte[] bytes, int initval) {
    return hash(bytes, bytes.length, initval);
  }
  
  /**
   * taken from  hashlittle() -- hash a variable-length key into a 32-bit value
   * 
   * @param key the key (the unaligned variable-length array of bytes)
   * @param nbytes number of bytes to include in hash
   * @param initval can be any integer value
   * @return a 32-bit value.  Every bit of the key affects every bit of the
   * return value.  Two keys differing by one or two bits will have totally
   * different hash values.
   * 
   * The best hash table sizes are powers of 2.  There is no need to do mod a
   * prime (mod is sooo slow!).  If you need less than 32 bits, use a bitmask.
   * For example, if you need only 10 bits, do h = (h & hashmask(10));
   * In which case, the hash table should have hashsize(10) elements.
   * 
   * If you are hashing n strings byte[][] k, do it like this:
   * for (int i = 0, h = 0; i < n; ++i) h = hash( k[i], h);
   * 
   * By Bob Jenkins, 2006.  bob_jenkins@burtleburtle.net.  You may use this
   * code any way you wish, private, educational, or commercial.  It's free.
   * 
   * Use for hash table lookup, or anything where one collision in 2^^32 is
   * acceptable.  Do NOT use for cryptographic purposes.
  */
  public static int hash(byte[] key, int nbytes, int initval) {
    int length = nbytes;
    long a, b, c;       // We use longs because we don't have unsigned ints
    a = b = c = (0x00000000deadbeefL + length + initval) & INT_MASK;
    int offset = 0;
    for (; length > 12; offset += 12, length -= 12) {
      a = (a + (key[offset + 0]    & BYTE_MASK)) & INT_MASK;
      a = (a + (((key[offset + 1]  & BYTE_MASK) <<  8) & INT_MASK)) & INT_MASK;
      a = (a + (((key[offset + 2]  & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
      a = (a + (((key[offset + 3]  & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
      b = (b + (key[offset + 4]    & BYTE_MASK)) & INT_MASK;
      b = (b + (((key[offset + 5]  & BYTE_MASK) <<  8) & INT_MASK)) & INT_MASK;
      b = (b + (((key[offset + 6]  & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
      b = (b + (((key[offset + 7]  & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
      c = (c + (key[offset + 8]    & BYTE_MASK)) & INT_MASK;
      c = (c + (((key[offset + 9]  & BYTE_MASK) <<  8) & INT_MASK)) & INT_MASK;
      c = (c + (((key[offset + 10] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
      c = (c + (((key[offset + 11] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
      
      /*
       * mix -- mix 3 32-bit values reversibly.
       * This is reversible, so any information in (a,b,c) before mix() is
       * still in (a,b,c) after mix().
       * 
       * If four pairs of (a,b,c) inputs are run through mix(), or through
       * mix() in reverse, there are at least 32 bits of the output that
       * are sometimes the same for one pair and different for another pair.
       * 
       * This was tested for:
       * - pairs that differed by one bit, by two bits, in any combination
       *   of top bits of (a,b,c), or in any combination of bottom bits of
       *   (a,b,c).
       * - "differ" is defined as +, -, ^, or ~^.  For + and -, I transformed
       *   the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
       *    is commonly produced by subtraction) look like a single 1-bit
       *    difference.
       * - the base values were pseudorandom, all zero but one bit set, or
       *   all zero plus a counter that starts at zero.
       * 
       * Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that
       * satisfy this are
       *     4  6  8 16 19  4
       *     9 15  3 18 27 15
       *    14  9  3  7 17  3
       * Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing for 
       * "differ" defined as + with a one-bit base and a two-bit delta.  I
       * used http://burtleburtle.net/bob/hash/avalanche.html to choose
       * the operations, constants, and arrangements of the variables.
       * 
       * This does not achieve avalanche.  There are input bits of (a,b,c)
       * that fail to affect some output bits of (a,b,c), especially of a.
       * The most thoroughly mixed value is c, but it doesn't really even
       * achieve avalanche in c.
       * 
       * This allows some parallelism.  Read-after-writes are good at doubling
       * the number of bits affected, so the goal of mixing pulls in the
       * opposite direction as the goal of parallelism.  I did what I could.
       * Rotates seem to cost as much as shifts on every machine I could lay
       * my hands on, and rotates are much kinder to the top and bottom bits,
       * so I used rotates.
       *
       * #define mix(a,b,c) \
       * { \
       *   a -= c;  a ^= rot(c, 4);  c += b; \
       *   b -= a;  b ^= rot(a, 6);  a += c; \
       *   c -= b;  c ^= rot(b, 8);  b += a; \
       *   a -= c;  a ^= rot(c,16);  c += b; \
       *   b -= a;  b ^= rot(a,19);  a += c; \
       *   c -= b;  c ^= rot(b, 4);  b += a; \
       * }
       * 
       * mix(a,b,c);
       */
      a = (a - c) & INT_MASK;  a ^= rot(c, 4);  c = (c + b) & INT_MASK;
      b = (b - a) & INT_MASK;  b ^= rot(a, 6);  a = (a + c) & INT_MASK;
      c = (c - b) & INT_MASK;  c ^= rot(b, 8);  b = (b + a) & INT_MASK;
      a = (a - c) & INT_MASK;  a ^= rot(c,16);  c = (c + b) & INT_MASK;
      b = (b - a) & INT_MASK;  b ^= rot(a,19);  a = (a + c) & INT_MASK;
      c = (c - b) & INT_MASK;  c ^= rot(b, 4);  b = (b + a) & INT_MASK;
    }

    //-------------------------------- last block: affect all 32 bits of (c)
    switch (length) {                   // all the case statements fall through
    case 12:
      c = (c + (((key[offset + 11] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
    case 11:
      c = (c + (((key[offset + 10] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
    case 10:
      c = (c + (((key[offset + 9]  & BYTE_MASK) <<  8) & INT_MASK)) & INT_MASK;
    case  9:
      c = (c + (key[offset + 8]    & BYTE_MASK)) & INT_MASK;
    case  8:
      b = (b + (((key[offset + 7]  & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
    case  7:
      b = (b + (((key[offset + 6]  & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
    case  6:
      b = (b + (((key[offset + 5]  & BYTE_MASK) <<  8) & INT_MASK)) & INT_MASK;
    case  5:
      b = (b + (key[offset + 4]    & BYTE_MASK)) & INT_MASK;
    case  4:
      a = (a + (((key[offset + 3]  & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
    case  3:
      a = (a + (((key[offset + 2]  & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
    case  2:
      a = (a + (((key[offset + 1]  & BYTE_MASK) <<  8) & INT_MASK)) & INT_MASK;
    case  1:
      a = (a + (key[offset + 0]    & BYTE_MASK)) & INT_MASK;
      break;
    case  0:
      return Long.valueOf(c & INT_MASK).intValue();
    }
    /*
     * final -- final mixing of 3 32-bit values (a,b,c) into c
     * 
     * Pairs of (a,b,c) values differing in only a few bits will usually
     * produce values of c that look totally different.  This was tested for
     * - pairs that differed by one bit, by two bits, in any combination
     *   of top bits of (a,b,c), or in any combination of bottom bits of
     *   (a,b,c).
     * 
     * - "differ" is defined as +, -, ^, or ~^.  For + and -, I transformed
     *   the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
     *   is commonly produced by subtraction) look like a single 1-bit
     *   difference.
     * 
     * - the base values were pseudorandom, all zero but one bit set, or
     *   all zero plus a counter that starts at zero.
     * 
     * These constants passed:
     *   14 11 25 16 4 14 24
     *   12 14 25 16 4 14 24
     * and these came close:
     *    4  8 15 26 3 22 24
     *   10  8 15 26 3 22 24
     *   11  8 15 26 3 22 24
     * 
     * #define final(a,b,c) \
     * { 
     *   c ^= b; c -= rot(b,14); \
     *   a ^= c; a -= rot(c,11); \
     *   b ^= a; b -= rot(a,25); \
     *   c ^= b; c -= rot(b,16); \
     *   a ^= c; a -= rot(c,4);  \
     *   b ^= a; b -= rot(a,14); \
     *   c ^= b; c -= rot(b,24); \
     * }
     * 
     */
    c ^= b; c = (c - rot(b,14)) & INT_MASK;
    a ^= c; a = (a - rot(c,11)) & INT_MASK;
    b ^= a; b = (b - rot(a,25)) & INT_MASK;
    c ^= b; c = (c - rot(b,16)) & INT_MASK;
    a ^= c; a = (a - rot(c,4))  & INT_MASK;
    b ^= a; b = (b - rot(a,14)) & INT_MASK;
    c ^= b; c = (c - rot(b,24)) & INT_MASK;

    return Long.valueOf(c & INT_MASK).intValue();
  }
  
  /**
   * Compute the hash of the specified file
   * @param args name of file to compute hash of.
   * @throws IOException
   */
  public static void main(String[] args) throws IOException {
    if (args.length != 1) {
      System.err.println("Usage: JenkinsHash filename");
      System.exit(-1);
    }
    FileInputStream in = new FileInputStream(args[0]);
    byte[] bytes = new byte[512];
    int value = 0;
    for (int length = in.read(bytes); length > 0 ; length = in.read(bytes)) {
      value = hash(bytes, length, value);
    }
    System.out.println(Math.abs(value));
  }
}
1	jimk	590875	/**
2			* Copyright 2007 The Apache Software Foundation
3			*
4			* Licensed to the Apache Software Foundation (ASF) under one
5			* or more contributor license agreements. See the NOTICE file
6			* distributed with this work for additional information
7			* regarding copyright ownership. The ASF licenses this file
8			* to you under the Apache License, Version 2.0 (the
9			* "License"); you may not use this file except in compliance
10			* with the License. You may obtain a copy of the License at
11			*
12			* http://www.apache.org/licenses/LICENSE-2.0
13			*
14			* Unless required by applicable law or agreed to in writing, software
15			* distributed under the License is distributed on an "AS IS" BASIS,
16			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17			* See the License for the specific language governing permissions and
18			* limitations under the License.
19			*/
20
21			package org.apache.hadoop.hbase.util;
22
23	jimk	639775	import java.io.FileInputStream;
24			import java.io.IOException;
25
26	jimk	590875	/**
27			* lookup3.c, by Bob Jenkins, May 2006, Public Domain.
28			* <a href="http://burtleburtle.net/bob/c/lookup3.c">lookup3.c</a>
29			*
30			* You can use this free for any purpose. It's in the public domain.
31			* It has no warranty.
32			*
33			* Produces 32-bit hash for hash table lookup.
34			*/
35			public class JenkinsHash {
36			private static long INT_MASK = 0x00000000ffffffffL;
37			private static long BYTE_MASK = 0x00000000000000ffL;
38
39			private static long rot(long val, int pos) {
40			return Long.valueOf(Integer.rotateLeft(
41			Long.valueOf(val & INT_MASK).intValue(), pos)).longValue() & INT_MASK;
42			}
43
44			/**
45			* Alternate form for hashing an entire byte array
46			*
47			* @param bytes
48			* @param initval
49			* @return hash value
50			*/
51			public static int hash(byte[] bytes, int initval) {
52			return hash(bytes, bytes.length, initval);
53			}
54
55			/**
56			* taken from hashlittle() -- hash a variable-length key into a 32-bit value
57			*
58			* @param key the key (the unaligned variable-length array of bytes)
59			* @param nbytes number of bytes to include in hash
60			* @param initval can be any integer value
61			* @return a 32-bit value. Every bit of the key affects every bit of the
62			* return value. Two keys differing by one or two bits will have totally
63			* different hash values.
64			*
65			* The best hash table sizes are powers of 2. There is no need to do mod a
66			* prime (mod is sooo slow!). If you need less than 32 bits, use a bitmask.
67			* For example, if you need only 10 bits, do h = (h & hashmask(10));
68			* In which case, the hash table should have hashsize(10) elements.
69			*
70			* If you are hashing n strings byte[][] k, do it like this:
71			* for (int i = 0, h = 0; i < n; ++i) h = hash( k[i], h);
72			*
73			* By Bob Jenkins, 2006. bob_jenkins@burtleburtle.net. You may use this
74			* code any way you wish, private, educational, or commercial. It's free.
75			*
76			* Use for hash table lookup, or anything where one collision in 2^^32 is
77			* acceptable. Do NOT use for cryptographic purposes.
78			*/
79			public static int hash(byte[] key, int nbytes, int initval) {
80			int length = nbytes;
81			long a, b, c; // We use longs because we don't have unsigned ints
82			a = b = c = (0x00000000deadbeefL + length + initval) & INT_MASK;
83			int offset = 0;
84			for (; length > 12; offset += 12, length -= 12) {
85			a = (a + (key[offset + 0] & BYTE_MASK)) & INT_MASK;
86			a = (a + (((key[offset + 1] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
87			a = (a + (((key[offset + 2] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
88			a = (a + (((key[offset + 3] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
89			b = (b + (key[offset + 4] & BYTE_MASK)) & INT_MASK;
90			b = (b + (((key[offset + 5] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
91			b = (b + (((key[offset + 6] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
92			b = (b + (((key[offset + 7] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
93			c = (c + (key[offset + 8] & BYTE_MASK)) & INT_MASK;
94			c = (c + (((key[offset + 9] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
95			c = (c + (((key[offset + 10] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
96			c = (c + (((key[offset + 11] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
97
98			/*
99			* mix -- mix 3 32-bit values reversibly.
100			* This is reversible, so any information in (a,b,c) before mix() is
101			* still in (a,b,c) after mix().
102			*
103			* If four pairs of (a,b,c) inputs are run through mix(), or through
104			* mix() in reverse, there are at least 32 bits of the output that
105			* are sometimes the same for one pair and different for another pair.
106			*
107			* This was tested for:
108			* - pairs that differed by one bit, by two bits, in any combination
109			* of top bits of (a,b,c), or in any combination of bottom bits of
110			* (a,b,c).
111			* - "differ" is defined as +, -, ^, or ~^. For + and -, I transformed
112			* the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
113			* is commonly produced by subtraction) look like a single 1-bit
114			* difference.
115			* - the base values were pseudorandom, all zero but one bit set, or
116			* all zero plus a counter that starts at zero.
117			*
118			* Some k values for my "a-=c; a^=rot(c,k); c+=b;" arrangement that
119			* satisfy this are
120			* 4 6 8 16 19 4
121			* 9 15 3 18 27 15
122			* 14 9 3 7 17 3
123			* Well, "9 15 3 18 27 15" didn't quite get 32 bits diffing for
124			* "differ" defined as + with a one-bit base and a two-bit delta. I
125			* used http://burtleburtle.net/bob/hash/avalanche.html to choose
126			* the operations, constants, and arrangements of the variables.
127			*
128			* This does not achieve avalanche. There are input bits of (a,b,c)
129			* that fail to affect some output bits of (a,b,c), especially of a.
130			* The most thoroughly mixed value is c, but it doesn't really even
131			* achieve avalanche in c.
132			*
133			* This allows some parallelism. Read-after-writes are good at doubling
134			* the number of bits affected, so the goal of mixing pulls in the
135			* opposite direction as the goal of parallelism. I did what I could.
136			* Rotates seem to cost as much as shifts on every machine I could lay
137			* my hands on, and rotates are much kinder to the top and bottom bits,
138			* so I used rotates.
139			*
140			* #define mix(a,b,c) \
141			* { \
142			* a -= c; a ^= rot(c, 4); c += b; \
143			* b -= a; b ^= rot(a, 6); a += c; \
144			* c -= b; c ^= rot(b, 8); b += a; \
145			* a -= c; a ^= rot(c,16); c += b; \
146			* b -= a; b ^= rot(a,19); a += c; \
147			* c -= b; c ^= rot(b, 4); b += a; \
148			* }
149			*
150			* mix(a,b,c);
151			*/
152			a = (a - c) & INT_MASK; a ^= rot(c, 4); c = (c + b) & INT_MASK;
153			b = (b - a) & INT_MASK; b ^= rot(a, 6); a = (a + c) & INT_MASK;
154			c = (c - b) & INT_MASK; c ^= rot(b, 8); b = (b + a) & INT_MASK;
155			a = (a - c) & INT_MASK; a ^= rot(c,16); c = (c + b) & INT_MASK;
156			b = (b - a) & INT_MASK; b ^= rot(a,19); a = (a + c) & INT_MASK;
157			c = (c - b) & INT_MASK; c ^= rot(b, 4); b = (b + a) & INT_MASK;
158			}
159
160			//-------------------------------- last block: affect all 32 bits of (c)
161			switch (length) { // all the case statements fall through
162			case 12:
163			c = (c + (((key[offset + 11] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
164			case 11:
165			c = (c + (((key[offset + 10] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
166			case 10:
167			c = (c + (((key[offset + 9] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
168			case 9:
169			c = (c + (key[offset + 8] & BYTE_MASK)) & INT_MASK;
170			case 8:
171			b = (b + (((key[offset + 7] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
172			case 7:
173			b = (b + (((key[offset + 6] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
174			case 6:
175			b = (b + (((key[offset + 5] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
176			case 5:
177			b = (b + (key[offset + 4] & BYTE_MASK)) & INT_MASK;
178			case 4:
179			a = (a + (((key[offset + 3] & BYTE_MASK) << 24) & INT_MASK)) & INT_MASK;
180			case 3:
181			a = (a + (((key[offset + 2] & BYTE_MASK) << 16) & INT_MASK)) & INT_MASK;
182			case 2:
183			a = (a + (((key[offset + 1] & BYTE_MASK) << 8) & INT_MASK)) & INT_MASK;
184			case 1:
185			a = (a + (key[offset + 0] & BYTE_MASK)) & INT_MASK;
186			break;
187			case 0:
188			return Long.valueOf(c & INT_MASK).intValue();
189			}
190			/*
191			* final -- final mixing of 3 32-bit values (a,b,c) into c
192			*
193			* Pairs of (a,b,c) values differing in only a few bits will usually
194			* produce values of c that look totally different. This was tested for
195			* - pairs that differed by one bit, by two bits, in any combination
196			* of top bits of (a,b,c), or in any combination of bottom bits of
197			* (a,b,c).
198			*
199			* - "differ" is defined as +, -, ^, or ~^. For + and -, I transformed
200			* the output delta to a Gray code (a^(a>>1)) so a string of 1's (as
201			* is commonly produced by subtraction) look like a single 1-bit
202			* difference.
203			*
204			* - the base values were pseudorandom, all zero but one bit set, or
205			* all zero plus a counter that starts at zero.
206			*
207			* These constants passed:
208			* 14 11 25 16 4 14 24
209			* 12 14 25 16 4 14 24
210			* and these came close:
211			* 4 8 15 26 3 22 24
212			* 10 8 15 26 3 22 24
213			* 11 8 15 26 3 22 24
214			*
215			* #define final(a,b,c) \
216			* {
217			* c ^= b; c -= rot(b,14); \
218			* a ^= c; a -= rot(c,11); \
219			* b ^= a; b -= rot(a,25); \
220			* c ^= b; c -= rot(b,16); \
221			* a ^= c; a -= rot(c,4); \
222			* b ^= a; b -= rot(a,14); \
223			* c ^= b; c -= rot(b,24); \
224			* }
225			*
226			*/
227			c ^= b; c = (c - rot(b,14)) & INT_MASK;
228			a ^= c; a = (a - rot(c,11)) & INT_MASK;
229			b ^= a; b = (b - rot(a,25)) & INT_MASK;
230			c ^= b; c = (c - rot(b,16)) & INT_MASK;
231			a ^= c; a = (a - rot(c,4)) & INT_MASK;
232			b ^= a; b = (b - rot(a,14)) & INT_MASK;
233			c ^= b; c = (c - rot(b,24)) & INT_MASK;
234
235			return Long.valueOf(c & INT_MASK).intValue();
236			}
237	jimk	639775
238			/**
239			* Compute the hash of the specified file
240			* @param args name of file to compute hash of.
241			* @throws IOException
242			*/
243			public static void main(String[] args) throws IOException {
244			if (args.length != 1) {
245			System.err.println("Usage: JenkinsHash filename");
246			System.exit(-1);
247			}
248			FileInputStream in = new FileInputStream(args[0]);
249			byte[] bytes = new byte[512];
250			int value = 0;
251			for (int length = in.read(bytes); length > 0 ; length = in.read(bytes)) {
252			value = hash(bytes, length, value);
253			}
254			System.out.println(Math.abs(value));
255			}
256	jimk	590875	}