1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.DataInput;
22 import java.io.DataOutput;
23 import java.io.EOFException;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.io.OutputStream;
27 import java.nio.ByteBuffer;
28 import java.util.ArrayList;
29 import java.util.List;
30
31 import org.apache.hadoop.hbase.KeyValue.Type;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.hbase.io.util.StreamUtils;
34 import org.apache.hadoop.hbase.util.ByteBufferUtils;
35 import org.apache.hadoop.hbase.util.Bytes;
36 import org.apache.hadoop.hbase.util.IterableUtils;
37 import org.apache.hadoop.io.IOUtils;
38 import org.apache.hadoop.io.WritableUtils;
39
40 import com.google.common.base.Function;
41 import com.google.common.collect.Lists;
42
43
44
45
46 @InterfaceAudience.Private
47 public class KeyValueUtil {
48
49
50
51
52
53
54
55
56 public static int length(final Cell cell) {
57 return length(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength(),
58 cell.getValueLength(), cell.getTagsLength(), true);
59 }
60
61 public static int length(short rlen, byte flen, int qlen, int vlen, int tlen, boolean withTags) {
62 if (withTags) {
63 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen, tlen));
64 }
65 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen));
66 }
67
68
69
70
71
72
73
74 public static int keyLength(final Cell cell) {
75 return keyLength(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength());
76 }
77
78 private static int keyLength(short rlen, byte flen, int qlen) {
79 return (int) KeyValue.getKeyDataStructureSize(rlen, flen, qlen);
80 }
81
82 public static int lengthWithMvccVersion(final KeyValue kv, final boolean includeMvccVersion) {
83 int length = kv.getLength();
84 if (includeMvccVersion) {
85 length += WritableUtils.getVIntSize(kv.getSequenceId());
86 }
87 return length;
88 }
89
90 public static int totalLengthWithMvccVersion(final Iterable<? extends KeyValue> kvs,
91 final boolean includeMvccVersion) {
92 int length = 0;
93 for (KeyValue kv : IterableUtils.nullSafe(kvs)) {
94 length += lengthWithMvccVersion(kv, includeMvccVersion);
95 }
96 return length;
97 }
98
99
100
101
102 public static KeyValue copyToNewKeyValue(final Cell cell) {
103 byte[] bytes = copyToNewByteArray(cell);
104 KeyValue kvCell = new KeyValue(bytes, 0, bytes.length);
105 kvCell.setSequenceId(cell.getSequenceId());
106 return kvCell;
107 }
108
109
110
111
112
113
114 public static ByteBuffer copyKeyToNewByteBuffer(final Cell cell) {
115 byte[] bytes = new byte[keyLength(cell)];
116 appendKeyTo(cell, bytes, 0);
117 ByteBuffer buffer = ByteBuffer.wrap(bytes);
118 return buffer;
119 }
120
121 public static byte[] copyToNewByteArray(final Cell cell) {
122 int v1Length = length(cell);
123 byte[] backingBytes = new byte[v1Length];
124 appendToByteArray(cell, backingBytes, 0);
125 return backingBytes;
126 }
127
128 public static int appendKeyTo(final Cell cell, final byte[] output,
129 final int offset) {
130 int nextOffset = offset;
131 nextOffset = Bytes.putShort(output, nextOffset, cell.getRowLength());
132 nextOffset = CellUtil.copyRowTo(cell, output, nextOffset);
133 nextOffset = Bytes.putByte(output, nextOffset, cell.getFamilyLength());
134 nextOffset = CellUtil.copyFamilyTo(cell, output, nextOffset);
135 nextOffset = CellUtil.copyQualifierTo(cell, output, nextOffset);
136 nextOffset = Bytes.putLong(output, nextOffset, cell.getTimestamp());
137 nextOffset = Bytes.putByte(output, nextOffset, cell.getTypeByte());
138 return nextOffset;
139 }
140
141
142
143
144 public static int appendToByteArray(final Cell cell, final byte[] output, final int offset) {
145
146
147 int pos = offset;
148 pos = Bytes.putInt(output, pos, keyLength(cell));
149 pos = Bytes.putInt(output, pos, cell.getValueLength());
150 pos = appendKeyTo(cell, output, pos);
151 pos = CellUtil.copyValueTo(cell, output, pos);
152 if ((cell.getTagsLength() > 0)) {
153 pos = Bytes.putAsShort(output, pos, cell.getTagsLength());
154 pos = CellUtil.copyTagTo(cell, output, pos);
155 }
156 return pos;
157 }
158
159
160
161
162
163
164 public static ByteBuffer copyToNewByteBuffer(final Cell cell) {
165 byte[] bytes = new byte[length(cell)];
166 appendToByteArray(cell, bytes, 0);
167 ByteBuffer buffer = ByteBuffer.wrap(bytes);
168 return buffer;
169 }
170
171 public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv,
172 final boolean includeMvccVersion) {
173
174 bb.limit(bb.position() + kv.getLength());
175 bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
176 if (includeMvccVersion) {
177 int numMvccVersionBytes = WritableUtils.getVIntSize(kv.getSequenceId());
178 ByteBufferUtils.extendLimit(bb, numMvccVersionBytes);
179 ByteBufferUtils.writeVLong(bb, kv.getSequenceId());
180 }
181 }
182
183
184
185
186
187
188
189
190
191
192
193 public static KeyValue nextShallowCopy(final ByteBuffer bb, final boolean includesMvccVersion,
194 boolean includesTags) {
195 if (bb.isDirect()) {
196 throw new IllegalArgumentException("only supports heap buffers");
197 }
198 if (bb.remaining() < 1) {
199 return null;
200 }
201 KeyValue keyValue = null;
202 int underlyingArrayOffset = bb.arrayOffset() + bb.position();
203 int keyLength = bb.getInt();
204 int valueLength = bb.getInt();
205 ByteBufferUtils.skip(bb, keyLength + valueLength);
206 int tagsLength = 0;
207 if (includesTags) {
208
209 tagsLength = ((bb.get() & 0xff) << 8) ^ (bb.get() & 0xff);
210 ByteBufferUtils.skip(bb, tagsLength);
211 }
212 int kvLength = (int) KeyValue.getKeyValueDataStructureSize(keyLength, valueLength, tagsLength);
213 keyValue = new KeyValue(bb.array(), underlyingArrayOffset, kvLength);
214 if (includesMvccVersion) {
215 long mvccVersion = ByteBufferUtils.readVLong(bb);
216 keyValue.setSequenceId(mvccVersion);
217 }
218 return keyValue;
219 }
220
221
222
223
224
225
226
227
228
229
230
231 public static KeyValue previousKey(final KeyValue in) {
232 return createFirstOnRow(CellUtil.cloneRow(in), CellUtil.cloneFamily(in),
233 CellUtil.cloneQualifier(in), in.getTimestamp() - 1);
234 }
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262 public static KeyValue createLastOnRow(final byte[] row, final int roffset, final int rlength,
263 final byte[] family, final int foffset, final int flength, final byte[] qualifier,
264 final int qoffset, final int qlength) {
265 return new KeyValue(row, roffset, rlength, family, foffset, flength, qualifier, qoffset,
266 qlength, HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
267 }
268
269
270
271
272
273
274
275
276
277 public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) {
278 return new KeyValue(row, roffset, rlength,
279 null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
280 }
281
282
283
284
285
286
287
288
289 public static KeyValue createLastOnRow(final byte[] row) {
290 return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum);
291 }
292
293
294
295
296
297
298
299
300
301 public static KeyValue createFirstOnRow(final byte [] row) {
302 return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP);
303 }
304
305
306
307
308
309
310
311
312 public static KeyValue createFirstOnRow(final byte [] row,
313 final long ts) {
314 return new KeyValue(row, null, null, ts, Type.Maximum);
315 }
316
317
318
319
320
321
322
323
324
325
326 public static KeyValue createFirstOnRow(final byte [] row, final byte [] family,
327 final byte [] qualifier) {
328 return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
329 }
330
331
332
333
334
335
336
337
338 public static KeyValue createFirstOnRow(final byte [] row, final byte [] f,
339 final byte [] q, final long ts) {
340 return new KeyValue(row, f, q, ts, Type.Maximum);
341 }
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359 public static KeyValue createFirstOnRow(final byte [] row,
360 final int roffset, final int rlength, final byte [] family,
361 final int foffset, final int flength, final byte [] qualifier,
362 final int qoffset, final int qlength) {
363 return new KeyValue(row, roffset, rlength, family,
364 foffset, flength, qualifier, qoffset, qlength,
365 HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
366 }
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384 public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
385 final byte [] family, final byte [] qualifier)
386 throws IllegalArgumentException {
387 return createFirstOnRow(buffer, 0, row, 0, row.length,
388 family, 0, family.length,
389 qualifier, 0, qualifier.length);
390 }
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415 public static KeyValue createFirstOnRow(byte[] buffer, final int boffset, final byte[] row,
416 final int roffset, final int rlength, final byte[] family, final int foffset,
417 final int flength, final byte[] qualifier, final int qoffset, final int qlength)
418 throws IllegalArgumentException {
419
420 long lLength = KeyValue.getKeyValueDataStructureSize(rlength, flength, qlength, 0);
421
422 if (lLength > Integer.MAX_VALUE) {
423 throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
424 }
425 int iLength = (int) lLength;
426 if (buffer.length - boffset < iLength) {
427 throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < "
428 + iLength);
429 }
430
431 int len = KeyValue.writeByteArray(buffer, boffset, row, roffset, rlength, family, foffset,
432 flength, qualifier, qoffset, qlength, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
433 null, 0, 0, null);
434 return new KeyValue(buffer, boffset, len);
435 }
436
437
438
439
440
441
442
443
444
445
446
447 @Deprecated
448 public static KeyValue ensureKeyValue(final Cell cell) {
449 if (cell == null) return null;
450 if (cell instanceof KeyValue) {
451 if (cell.getClass().getName().equals(KeyValue.class.getName())) {
452 return (KeyValue) cell;
453 }
454
455
456 KeyValue kv = (KeyValue) cell;
457 KeyValue newKv = new KeyValue(kv.bytes, kv.offset, kv.length);
458 newKv.setSequenceId(kv.getSequenceId());
459 return newKv;
460 }
461 return copyToNewKeyValue(cell);
462 }
463
464 @Deprecated
465 public static List<KeyValue> ensureKeyValues(List<Cell> cells) {
466 List<KeyValue> lazyList = Lists.transform(cells, new Function<Cell, KeyValue>() {
467 @Override
468 public KeyValue apply(Cell arg0) {
469 return KeyValueUtil.ensureKeyValue(arg0);
470 }
471 });
472 return new ArrayList<KeyValue>(lazyList);
473 }
474
475
476
477
478
479
480
481
482
483
484 public static long write(final KeyValue kv, final DataOutput out) throws IOException {
485
486
487
488 int length = kv.getLength();
489 out.writeInt(length);
490 out.write(kv.getBuffer(), kv.getOffset(), length);
491 return length + Bytes.SIZEOF_INT;
492 }
493
494
495
496
497
498
499
500
501
502
503
504 public static KeyValue iscreate(final InputStream in, boolean withTags) throws IOException {
505 byte[] intBytes = new byte[Bytes.SIZEOF_INT];
506 int bytesRead = 0;
507 while (bytesRead < intBytes.length) {
508 int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead);
509 if (n < 0) {
510 if (bytesRead == 0) {
511 throw new EOFException();
512 }
513 throw new IOException("Failed read of int, read " + bytesRead + " bytes");
514 }
515 bytesRead += n;
516 }
517
518 byte[] bytes = new byte[Bytes.toInt(intBytes)];
519 IOUtils.readFully(in, bytes, 0, bytes.length);
520 if (withTags) {
521 return new KeyValue(bytes, 0, bytes.length);
522 } else {
523 return new NoTagsKeyValue(bytes, 0, bytes.length);
524 }
525 }
526
527
528
529
530
531
532 public static KeyValue createKeyValueFromKey(final byte[] b) {
533 return createKeyValueFromKey(b, 0, b.length);
534 }
535
536
537
538
539
540
541 public static KeyValue createKeyValueFromKey(final ByteBuffer bb) {
542 return createKeyValueFromKey(bb.array(), bb.arrayOffset(), bb.limit());
543 }
544
545
546
547
548
549
550
551
552 public static KeyValue createKeyValueFromKey(final byte[] b, final int o, final int l) {
553 byte[] newb = new byte[l + KeyValue.ROW_OFFSET];
554 System.arraycopy(b, o, newb, KeyValue.ROW_OFFSET, l);
555 Bytes.putInt(newb, 0, l);
556 Bytes.putInt(newb, Bytes.SIZEOF_INT, 0);
557 return new KeyValue(newb);
558 }
559
560
561
562
563
564
565
566
567
568
569 public static KeyValue create(final DataInput in) throws IOException {
570 return create(in.readInt(), in);
571 }
572
573
574
575
576
577
578
579
580
581
582 public static KeyValue create(int length, final DataInput in) throws IOException {
583
584 if (length <= 0) {
585 if (length == 0)
586 return null;
587 throw new IOException("Failed read " + length + " bytes, stream corrupt?");
588 }
589
590
591
592 byte[] bytes = new byte[length];
593 in.readFully(bytes);
594 return new KeyValue(bytes, 0, length);
595 }
596
597 public static void oswrite(final Cell cell, final OutputStream out, final boolean withTags)
598 throws IOException {
599 if (cell instanceof Streamable) {
600 ((Streamable)cell).write(out, withTags);
601 } else {
602 short rlen = cell.getRowLength();
603 byte flen = cell.getFamilyLength();
604 int qlen = cell.getQualifierLength();
605 int vlen = cell.getValueLength();
606 int tlen = cell.getTagsLength();
607
608
609 ByteBufferUtils.putInt(out, length(rlen, flen, qlen, vlen, tlen, withTags));
610
611 ByteBufferUtils.putInt(out, keyLength(rlen, flen, qlen));
612
613 ByteBufferUtils.putInt(out, vlen);
614
615 StreamUtils.writeShort(out, rlen);
616 out.write(cell.getRowArray(), cell.getRowOffset(), rlen);
617
618 out.write(flen);
619 out.write(cell.getFamilyArray(), cell.getFamilyOffset(), flen);
620
621 out.write(cell.getQualifierArray(), cell.getQualifierOffset(), qlen);
622
623 StreamUtils.writeLong(out, cell.getTimestamp());
624
625 out.write(cell.getTypeByte());
626
627 out.write(cell.getValueArray(), cell.getValueOffset(), vlen);
628
629 if (withTags && tlen > 0) {
630
631
632
633
634 out.write((byte) (0xff & (tlen >> 8)));
635 out.write((byte) (0xff & tlen));
636 out.write(cell.getTagsArray(), cell.getTagsOffset(), tlen);
637 }
638 }
639 }
640 }