1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.accumulo.core.data;
18
19 import java.io.DataInput;
20 import java.io.DataOutput;
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.Collections;
25 import java.util.List;
26
27 import org.apache.accumulo.core.data.thrift.TMutation;
28 import org.apache.accumulo.core.security.ColumnVisibility;
29 import org.apache.accumulo.core.util.ByteBufferUtil;
30 import org.apache.hadoop.io.Text;
31 import org.apache.hadoop.io.Writable;
32 import org.apache.hadoop.io.WritableUtils;
33
34 /**
35 * <p>
36 * Mutation represents an action that manipulates a row in a table. A mutation holds a list of column/value pairs that represent an atomic set of modifications
37 * to make to a row.
38 *
39 * <p>
40 * Convenience methods which takes columns and value as CharSequence (String implements CharSequence) are provided. CharSequence is converted to UTF-8 by
41 * constructing a new Text object.
42 *
43 * <p>
44 * When always passing in the same data as a CharSequence/String, its probably more efficient to call the Text put methods. This way the data is only encoded
45 * once and only one Text object is created.
46 *
47 * <p>
48 * All of the put methods append data to the mutation, they do not overwrite anything that was previously put. The mutation holds a list of all column/values
49 * that were put into it. The putDelete() methods do not remove something that was previously added to the mutation, rather they indicate that Accumulo should
50 * insert a delete marker for that row column.
51 *
52 */
53
54 public class Mutation implements Writable {
55
56 static final int VALUE_SIZE_COPY_CUTOFF = 1 << 15;
57
58 public static enum SERIALIZED_FORMAT {
59 VERSION1,
60 VERSION2
61 };
62
63 private boolean useOldDeserialize = false;
64 private byte[] row;
65 private byte[] data;
66 private int entries;
67 private List<byte[]> values;
68
69
70
71 private static class ByteBuffer {
72
73 int offset;
74 byte data[] = new byte[64];
75
76 private void reserve(int l) {
77 if (offset + l > data.length) {
78 int newSize = data.length * 2;
79 while (newSize <= offset + l)
80 newSize = newSize * 2;
81
82 byte[] newData = new byte[newSize];
83 System.arraycopy(data, 0, newData, 0, offset);
84 data = newData;
85 }
86
87 }
88
89 public void add(byte[] bytes, int off, int length) {
90 reserve(length);
91 System.arraycopy(bytes, off, data, offset, length);
92 offset += length;
93 }
94
95 void add(boolean b) {
96 reserve(1);
97 if (b)
98 data[offset++] = 1;
99 else
100 data[offset++] = 0;
101 }
102
103 public byte[] toArray() {
104 byte ret[] = new byte[offset];
105 System.arraycopy(data, 0, ret, 0, offset);
106 return ret;
107 }
108
109 public void writeVLong(long i) {
110 reserve(9);
111 if (i >= -112 && i <= 127) {
112 data[offset++] = (byte)i;
113 return;
114 }
115
116 int len = -112;
117 if (i < 0) {
118 i ^= -1L;
119 len = -120;
120 }
121
122 long tmp = i;
123 while (tmp != 0) {
124 tmp = tmp >> 8;
125 len--;
126 }
127
128 data[offset++] = (byte)len;
129
130 len = (len < -120) ? -(len + 120) : -(len + 112);
131
132 for (int idx = len; idx != 0; idx--) {
133 int shiftbits = (idx - 1) * 8;
134 long mask = 0xFFL << shiftbits;
135 data[offset++] = (byte)((i & mask) >> shiftbits);
136 }
137 }
138 }
139
140 private static class SimpleReader {
141 int offset;
142 byte data[];
143
144 SimpleReader(byte b[]) {
145 this.data = b;
146 }
147
148 int readInt() {
149 return (data[offset++] << 24) + ((data[offset++] & 255) << 16) + ((data[offset++] & 255) << 8) + ((data[offset++] & 255) << 0);
150 }
151
152 long readLong() {
153 return (((long) data[offset++] << 56) + ((long) (data[offset++] & 255) << 48) + ((long) (data[offset++] & 255) << 40)
154 + ((long) (data[offset++] & 255) << 32) + ((long) (data[offset++] & 255) << 24) + ((data[offset++] & 255) << 16) + ((data[offset++] & 255) << 8) + ((data[offset++] & 255) << 0));
155 }
156
157 void readBytes(byte b[]) {
158 System.arraycopy(data, offset, b, 0, b.length);
159 offset += b.length;
160 }
161
162 boolean readBoolean() {
163 return (data[offset++] == 1);
164 }
165
166 long readVLong() {
167 byte firstByte = data[offset++];
168 int len = WritableUtils.decodeVIntSize(firstByte);
169 if (len == 1) {
170 return firstByte;
171 }
172 long i = 0;
173 for (int idx = 0; idx < len-1; idx++) {
174 byte b = data[offset++];
175 i = i << 8;
176 i = i | (b & 0xFF);
177 }
178 return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
179 }
180 }
181
182 private ByteBuffer buffer;
183
184 private List<ColumnUpdate> updates;
185
186 private static final byte[] EMPTY_BYTES = new byte[0];
187
188 private void serialize() {
189 if (buffer != null) {
190 data = buffer.toArray();
191 buffer = null;
192 }
193 }
194
195 public Mutation(byte[] byteBuffer) {
196 this(byteBuffer, 0, byteBuffer.length);
197 }
198
199 public Mutation(byte[] byteBuffer, int start, int length) {
200 this.row = new byte[length];
201 System.arraycopy(byteBuffer, start, this.row, 0, length);
202 buffer = new ByteBuffer();
203 }
204
205 public Mutation(Text row) {
206 this(row.getBytes(), 0, row.getLength());
207 }
208
209 public Mutation(CharSequence row) {
210 this(new Text(row.toString()));
211 }
212
213 public Mutation() {}
214
215 public Mutation(TMutation tmutation) {
216 this.row = ByteBufferUtil.toBytes(tmutation.row);
217 this.data = ByteBufferUtil.toBytes(tmutation.data);
218 this.entries = tmutation.entries;
219 this.values = ByteBufferUtil.toBytesList(tmutation.values);
220 }
221
222 public Mutation(Mutation m) {
223 m.serialize();
224 this.row = m.row;
225 this.data = m.data;
226 this.entries = m.entries;
227 this.values = m.values;
228 }
229
230 public byte[] getRow() {
231 return row;
232 }
233
234 public static String toHexString(byte[] ba) {
235 StringBuilder str = new StringBuilder();
236 for (int i = 0; i < ba.length; i++)
237 str.append(String.format("%x", ba[i]));
238 return str.toString();
239 }
240
241 private void put(byte b[]) {
242 put(b, b.length);
243 }
244
245 private void put(byte b[], int length) {
246 buffer.writeVLong(length);
247 buffer.add(b, 0, length);
248 }
249
250 private void put(boolean b) {
251 buffer.add(b);
252 }
253
254 private void put(int i) {
255 buffer.writeVLong(i);
256 }
257
258 private void put(long l) {
259 buffer.writeVLong(l);
260 }
261
262 private void put(byte[] cf, byte[] cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
263 put(cf, cf.length, cq, cq.length, cv, hasts, ts, deleted, val, val.length);
264 }
265
266
267
268
269 private void put(Text cf, Text cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
270 put(cf.getBytes(), cf.getLength(), cq.getBytes(), cq.getLength(), cv, hasts, ts, deleted, val, val.length);
271 }
272
273 private void put(byte[] cf, int cfLength, byte[] cq, int cqLength, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val, int valLength) {
274 if (buffer == null) {
275 throw new IllegalStateException("Can not add to mutation after serializing it");
276 }
277 put(cf, cfLength);
278 put(cq, cqLength);
279 put(cv);
280 put(hasts);
281 if (hasts) {
282 put(ts);
283 }
284 put(deleted);
285
286 if (valLength < VALUE_SIZE_COPY_CUTOFF) {
287 put(val, valLength);
288 } else {
289 if (values == null) {
290 values = new ArrayList<byte[]>();
291 }
292 byte copy[] = new byte[valLength];
293 System.arraycopy(val, 0, copy, 0, valLength);
294 values.add(copy);
295 put(-1 * values.size());
296 }
297
298 entries++;
299 }
300
301 private void put(CharSequence cf, CharSequence cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
302 put(new Text(cf.toString()), new Text(cq.toString()), cv, hasts, ts, deleted, val);
303 }
304
305 private void put(Text cf, Text cq, byte[] cv, boolean hasts, long ts, boolean deleted, Text val) {
306 put(cf.getBytes(), cf.getLength(), cq.getBytes(), cq.getLength(), cv, hasts, ts, deleted, val.getBytes(), val.getLength());
307 }
308
309 private void put(CharSequence cf, CharSequence cq, byte[] cv, boolean hasts, long ts, boolean deleted, CharSequence val) {
310 put(new Text(cf.toString()), new Text(cq.toString()), cv, hasts, ts, deleted, new Text(val.toString()));
311 }
312
313 public void put(Text columnFamily, Text columnQualifier, Value value) {
314 put(columnFamily, columnQualifier, EMPTY_BYTES, false, 0l, false, value.get());
315 }
316
317 public void put(Text columnFamily, Text columnQualifier, ColumnVisibility columnVisibility, Value value) {
318 put(columnFamily, columnQualifier, columnVisibility.getExpression(), false, 0l, false, value.get());
319 }
320
321 public void put(Text columnFamily, Text columnQualifier, long timestamp, Value value) {
322 put(columnFamily, columnQualifier, EMPTY_BYTES, true, timestamp, false, value.get());
323 }
324
325 public void put(Text columnFamily, Text columnQualifier, ColumnVisibility columnVisibility, long timestamp, Value value) {
326 put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp, false, value.get());
327 }
328
329 public void putDelete(Text columnFamily, Text columnQualifier) {
330 put(columnFamily, columnQualifier, EMPTY_BYTES, false, 0l, true, EMPTY_BYTES);
331 }
332
333 public void putDelete(Text columnFamily, Text columnQualifier, ColumnVisibility columnVisibility) {
334 put(columnFamily, columnQualifier, columnVisibility.getExpression(), false, 0l, true, EMPTY_BYTES);
335 }
336
337 public void putDelete(Text columnFamily, Text columnQualifier, long timestamp) {
338 put(columnFamily, columnQualifier, EMPTY_BYTES, true, timestamp, true, EMPTY_BYTES);
339 }
340
341 public void putDelete(Text columnFamily, Text columnQualifier, ColumnVisibility columnVisibility, long timestamp) {
342 put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp, true, EMPTY_BYTES);
343 }
344
345 public void put(CharSequence columnFamily, CharSequence columnQualifier, Value value) {
346 put(columnFamily, columnQualifier, EMPTY_BYTES, false, 0l, false, value.get());
347 }
348
349 public void put(CharSequence columnFamily, CharSequence columnQualifier, ColumnVisibility columnVisibility, Value value) {
350 put(columnFamily, columnQualifier, columnVisibility.getExpression(), false, 0l, false, value.get());
351 }
352
353 public void put(CharSequence columnFamily, CharSequence columnQualifier, long timestamp, Value value) {
354 put(columnFamily, columnQualifier, EMPTY_BYTES, true, timestamp, false, value.get());
355 }
356
357 public void put(CharSequence columnFamily, CharSequence columnQualifier, ColumnVisibility columnVisibility, long timestamp, Value value) {
358 put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp, false, value.get());
359 }
360
361 public void putDelete(CharSequence columnFamily, CharSequence columnQualifier) {
362 put(columnFamily, columnQualifier, EMPTY_BYTES, false, 0l, true, EMPTY_BYTES);
363 }
364
365 public void putDelete(CharSequence columnFamily, CharSequence columnQualifier, ColumnVisibility columnVisibility) {
366 put(columnFamily, columnQualifier, columnVisibility.getExpression(), false, 0l, true, EMPTY_BYTES);
367 }
368
369 public void putDelete(CharSequence columnFamily, CharSequence columnQualifier, long timestamp) {
370 put(columnFamily, columnQualifier, EMPTY_BYTES, true, timestamp, true, EMPTY_BYTES);
371 }
372
373 public void putDelete(CharSequence columnFamily, CharSequence columnQualifier, ColumnVisibility columnVisibility, long timestamp) {
374 put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp, true, EMPTY_BYTES);
375 }
376
377 public void put(CharSequence columnFamily, CharSequence columnQualifier, CharSequence value) {
378 put(columnFamily, columnQualifier, EMPTY_BYTES, false, 0l, false, value);
379 }
380
381 public void put(CharSequence columnFamily, CharSequence columnQualifier, ColumnVisibility columnVisibility, CharSequence value) {
382 put(columnFamily, columnQualifier, columnVisibility.getExpression(), false, 0l, false, value);
383 }
384
385 public void put(CharSequence columnFamily, CharSequence columnQualifier, long timestamp, CharSequence value) {
386 put(columnFamily, columnQualifier, EMPTY_BYTES, true, timestamp, false, value);
387 }
388
389 public void put(CharSequence columnFamily, CharSequence columnQualifier, ColumnVisibility columnVisibility, long timestamp, CharSequence value) {
390 put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp, false, value);
391 }
392
393 public void put(byte[] columnFamily, byte[] columnQualifier, byte[] value) {
394 put(columnFamily, columnQualifier, EMPTY_BYTES, false, 0l, false, value);
395 }
396
397 public void put(byte[] columnFamily, byte[] columnQualifier, ColumnVisibility columnVisibility, byte[] value) {
398 put(columnFamily, columnQualifier, columnVisibility.getExpression(), false, 0l, false, value);
399 }
400
401 public void put(byte[] columnFamily, byte[] columnQualifier, long timestamp, byte[] value) {
402 put(columnFamily, columnQualifier, EMPTY_BYTES, true, timestamp, false, value);
403 }
404
405 public void put(byte[] columnFamily, byte[] columnQualifier, ColumnVisibility columnVisibility, long timestamp, byte[] value) {
406 put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp, false, value);
407 }
408
409 public void putDelete(byte[] columnFamily, byte[] columnQualifier) {
410 put(columnFamily, columnQualifier, EMPTY_BYTES, false, 0l, true, EMPTY_BYTES);
411 }
412
413 public void putDelete(byte[] columnFamily, byte[] columnQualifier, ColumnVisibility columnVisibility) {
414 put(columnFamily, columnQualifier, columnVisibility.getExpression(), false, 0l, true, EMPTY_BYTES);
415 }
416
417 public void putDelete(byte[] columnFamily, byte[] columnQualifier, long timestamp) {
418 put(columnFamily, columnQualifier, EMPTY_BYTES, true, timestamp, true, EMPTY_BYTES);
419 }
420
421 public void putDelete(byte[] columnFamily, byte[] columnQualifier, ColumnVisibility columnVisibility, long timestamp) {
422 put(columnFamily, columnQualifier, columnVisibility.getExpression(), true, timestamp, true, EMPTY_BYTES);
423 }
424
425 private byte[] oldReadBytes(SimpleReader in) {
426 int len = in.readInt();
427 if (len == 0)
428 return EMPTY_BYTES;
429
430 byte bytes[] = new byte[len];
431 in.readBytes(bytes);
432 return bytes;
433 }
434
435 private byte[] readBytes(SimpleReader in) {
436 int len = (int)in.readVLong();
437 if (len == 0)
438 return EMPTY_BYTES;
439
440 byte bytes[] = new byte[len];
441 in.readBytes(bytes);
442 return bytes;
443 }
444
445 public List<ColumnUpdate> getUpdates() {
446 serialize();
447
448 SimpleReader in = new SimpleReader(data);
449
450 if (updates == null) {
451 if (entries == 1) {
452 updates = Collections.singletonList(deserializeColumnUpdate(in));
453 } else {
454 ColumnUpdate[] tmpUpdates = new ColumnUpdate[entries];
455
456 for (int i = 0; i < entries; i++)
457 tmpUpdates[i] = deserializeColumnUpdate(in);
458
459 updates = Arrays.asList(tmpUpdates);
460 }
461 }
462
463 return updates;
464 }
465
466 protected ColumnUpdate newColumnUpdate(byte[] cf, byte[] cq, byte[] cv, boolean hasts, long ts, boolean deleted, byte[] val) {
467 return new ColumnUpdate(cf, cq, cv, hasts, ts, deleted, val);
468 }
469
470 private ColumnUpdate deserializeColumnUpdate(SimpleReader in) {
471 byte[] cf = readBytes(in);
472 byte[] cq = readBytes(in);
473 byte[] cv = readBytes(in);
474 boolean hasts = in.readBoolean();
475 long ts = 0;
476 if (hasts)
477 ts = in.readVLong();
478 boolean deleted = in.readBoolean();
479
480 byte[] val;
481 int valLen = (int)in.readVLong();
482
483 if (valLen < 0) {
484 val = values.get((-1 * valLen) - 1);
485 } else if (valLen == 0) {
486 val = EMPTY_BYTES;
487 } else {
488 val = new byte[valLen];
489 in.readBytes(val);
490 }
491
492 return newColumnUpdate(cf, cq, cv, hasts, ts, deleted, val);
493 }
494
495 private int cachedValLens = -1;
496
497 long getValueLengths() {
498 if (values == null)
499 return 0;
500
501 if (cachedValLens == -1) {
502 int tmpCVL = 0;
503 for (byte[] val : values)
504 tmpCVL += val.length;
505
506 cachedValLens = tmpCVL;
507 }
508
509 return cachedValLens;
510
511 }
512
513 public long numBytes() {
514 serialize();
515 return row.length + data.length + getValueLengths();
516 }
517
518 public long estimatedMemoryUsed() {
519 return numBytes() + 238;
520 }
521
522 /**
523 * @return the number of column value pairs added to the mutation
524 */
525 public int size() {
526 return entries;
527 }
528
529 @Override
530 public void readFields(DataInput in) throws IOException {
531
532
533
534
535 updates = null;
536 cachedValLens = -1;
537 buffer = null;
538 useOldDeserialize = false;
539
540 byte first = in.readByte();
541 if ((first & 0x80) != 0x80) {
542 oldReadFields(first, in);
543 useOldDeserialize = true;
544 return;
545 }
546
547 int len = WritableUtils.readVInt(in);
548 row = new byte[len];
549 in.readFully(row);
550 len = WritableUtils.readVInt(in);
551 data = new byte[len];
552 in.readFully(data);
553 entries = WritableUtils.readVInt(in);
554
555 boolean valuesPresent = (first & 0x01) == 0x01;
556 if (!valuesPresent) {
557 values = null;
558 } else {
559 values = new ArrayList<byte[]>();
560 int numValues = WritableUtils.readVInt(in);
561 for (int i = 0; i < numValues; i++) {
562 len = WritableUtils.readVInt(in);
563 byte val[] = new byte[len];
564 in.readFully(val);
565 values.add(val);
566 }
567 }
568 }
569
570 protected void droppingOldTimestamp(long ts) {}
571
572 private void oldReadFields(byte first, DataInput in) throws IOException {
573
574 byte b = (byte)in.readByte();
575 byte c = (byte)in.readByte();
576 byte d = (byte)in.readByte();
577
578 int len = (((first & 0xff) << 24) | ((b & 0xff) << 16) |
579 ((c & 0xff) << 8) | (d & 0xff));
580 row = new byte[len];
581 in.readFully(row);
582 len = in.readInt();
583 byte[] localData = new byte[len];
584 in.readFully(localData);
585 int localEntries = in.readInt();
586
587 List<byte[]> localValues;
588 boolean valuesPresent = in.readBoolean();
589 if (!valuesPresent) {
590 localValues = null;
591 } else {
592 localValues = new ArrayList<byte[]>();
593 int numValues = in.readInt();
594 for (int i = 0; i < numValues; i++) {
595 len = in.readInt();
596 byte val[] = new byte[len];
597 in.readFully(val);
598 localValues.add(val);
599 }
600 }
601
602
603 SimpleReader din = new SimpleReader(localData);
604 buffer = new ByteBuffer();
605 for (int i = 0; i < localEntries; i++) {
606 byte[] cf = oldReadBytes(din);
607 byte[] cq = oldReadBytes(din);
608 byte[] cv = oldReadBytes(din);
609 boolean hasts = din.readBoolean();
610 long ts = din.readLong();
611 boolean deleted = din.readBoolean();
612
613 byte[] val;
614 int valLen = din.readInt();
615
616 if (valLen < 0) {
617 val = localValues.get((-1 * valLen) - 1);
618 } else if (valLen == 0) {
619 val = EMPTY_BYTES;
620 } else {
621 val = new byte[valLen];
622 din.readBytes(val);
623 }
624
625 put(cf, cq, cv, hasts, ts, deleted, val);
626 if (!hasts)
627 droppingOldTimestamp(ts);
628 }
629
630 serialize();
631
632 }
633
634 @Override
635 public void write(DataOutput out) throws IOException {
636 serialize();
637 byte hasValues = (values == null) ? 0 : (byte)1;
638 out.write((byte)(0x80 | hasValues));
639
640 WritableUtils.writeVInt(out, row.length);
641 out.write(row);
642
643 WritableUtils.writeVInt(out, data.length);
644 out.write(data);
645 WritableUtils.writeVInt(out, entries);
646
647 if (hasValues > 0) {
648 WritableUtils.writeVInt(out, values.size());
649 for (int i = 0; i < values.size(); i++) {
650 byte val[] = values.get(i);
651 WritableUtils.writeVInt(out, val.length);
652 out.write(val);
653 }
654 }
655 }
656
657 @Override
658 public boolean equals(Object o) {
659 if (o instanceof Mutation)
660 return equals((Mutation) o);
661 return false;
662 }
663
664 @Override
665 public int hashCode() {
666 return toThrift().hashCode();
667 }
668
669 public boolean equals(Mutation m) {
670 serialize();
671 if (Arrays.equals(row, m.row) && entries == m.entries && Arrays.equals(data, m.data)) {
672 if (values == null && m.values == null)
673 return true;
674
675 if (values != null && m.values != null && values.size() == m.values.size()) {
676 for (int i = 0; i < values.size(); i++) {
677 if (!Arrays.equals(values.get(i), m.values.get(i)))
678 return false;
679 }
680
681 return true;
682 }
683
684 }
685
686 return false;
687 }
688
689 public TMutation toThrift() {
690 serialize();
691 return new TMutation(java.nio.ByteBuffer.wrap(row), java.nio.ByteBuffer.wrap(data), ByteBufferUtil.toByteBuffers(values), entries);
692 }
693
694 protected SERIALIZED_FORMAT getSerializedFormat() {
695 return this.useOldDeserialize ? SERIALIZED_FORMAT.VERSION1 : SERIALIZED_FORMAT.VERSION2;
696 }
697
698 }