1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase;
20
21 import java.io.DataInput;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.io.OutputStream;
25 import java.nio.ByteBuffer;
26 import java.util.ArrayList;
27 import java.util.List;
28
29 import org.apache.hadoop.hbase.classification.InterfaceAudience;
30 import org.apache.hadoop.hbase.KeyValue.Type;
31 import org.apache.hadoop.hbase.io.util.StreamUtils;
32 import org.apache.hadoop.hbase.util.ByteBufferUtils;
33 import org.apache.hadoop.hbase.util.Bytes;
34 import org.apache.hadoop.hbase.util.IterableUtils;
35 import org.apache.hadoop.hbase.util.SimpleMutableByteRange;
36 import org.apache.hadoop.io.IOUtils;
37 import org.apache.hadoop.io.WritableUtils;
38
39 import com.google.common.base.Function;
40 import com.google.common.collect.Lists;
41
42
43
44
45 @InterfaceAudience.Private
46 public class KeyValueUtil {
47
48
49
50
51
52
53
54
55 public static int length(final Cell cell) {
56 return length(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength(),
57 cell.getValueLength(), cell.getTagsLength(), true);
58 }
59
60 private static int length(short rlen, byte flen, int qlen, int vlen, int tlen, boolean withTags) {
61 if (withTags) {
62 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen, tlen));
63 }
64 return (int) (KeyValue.getKeyValueDataStructureSize(rlen, flen, qlen, vlen));
65 }
66
67
68
69
70
71
72
73 public static int keyLength(final Cell cell) {
74 return keyLength(cell.getRowLength(), cell.getFamilyLength(), cell.getQualifierLength());
75 }
76
77 private static int keyLength(short rlen, byte flen, int qlen) {
78 return (int) KeyValue.getKeyDataStructureSize(rlen, flen, qlen);
79 }
80
81 public static int lengthWithMvccVersion(final KeyValue kv, final boolean includeMvccVersion) {
82 int length = kv.getLength();
83 if (includeMvccVersion) {
84 length += WritableUtils.getVIntSize(kv.getMvccVersion());
85 }
86 return length;
87 }
88
89 public static int totalLengthWithMvccVersion(final Iterable<? extends KeyValue> kvs,
90 final boolean includeMvccVersion) {
91 int length = 0;
92 for (KeyValue kv : IterableUtils.nullSafe(kvs)) {
93 length += lengthWithMvccVersion(kv, includeMvccVersion);
94 }
95 return length;
96 }
97
98
99
100
101 public static KeyValue copyToNewKeyValue(final Cell cell) {
102 byte[] bytes = copyToNewByteArray(cell);
103 KeyValue kvCell = new KeyValue(bytes, 0, bytes.length);
104 kvCell.setSequenceId(cell.getMvccVersion());
105 return kvCell;
106 }
107
108
109
110
111
112
113 public static ByteBuffer copyKeyToNewByteBuffer(final Cell cell) {
114 byte[] bytes = new byte[keyLength(cell)];
115 appendKeyTo(cell, bytes, 0);
116 ByteBuffer buffer = ByteBuffer.wrap(bytes);
117 return buffer;
118 }
119
120 public static byte[] copyToNewByteArray(final Cell cell) {
121 int v1Length = length(cell);
122 byte[] backingBytes = new byte[v1Length];
123 appendToByteArray(cell, backingBytes, 0);
124 return backingBytes;
125 }
126
127 public static int appendKeyTo(final Cell cell, final byte[] output,
128 final int offset) {
129 int nextOffset = offset;
130 nextOffset = Bytes.putShort(output, nextOffset, cell.getRowLength());
131 nextOffset = CellUtil.copyRowTo(cell, output, nextOffset);
132 nextOffset = Bytes.putByte(output, nextOffset, cell.getFamilyLength());
133 nextOffset = CellUtil.copyFamilyTo(cell, output, nextOffset);
134 nextOffset = CellUtil.copyQualifierTo(cell, output, nextOffset);
135 nextOffset = Bytes.putLong(output, nextOffset, cell.getTimestamp());
136 nextOffset = Bytes.putByte(output, nextOffset, cell.getTypeByte());
137 return nextOffset;
138 }
139
140
141
142
143 public static int appendToByteArray(final Cell cell, final byte[] output, final int offset) {
144
145
146 int pos = offset;
147 pos = Bytes.putInt(output, pos, keyLength(cell));
148 pos = Bytes.putInt(output, pos, cell.getValueLength());
149 pos = appendKeyTo(cell, output, pos);
150 pos = CellUtil.copyValueTo(cell, output, pos);
151 if ((cell.getTagsLength() > 0)) {
152 pos = Bytes.putAsShort(output, pos, cell.getTagsLength());
153 pos = CellUtil.copyTagTo(cell, output, pos);
154 }
155 return pos;
156 }
157
158
159
160
161
162
163 public static ByteBuffer copyToNewByteBuffer(final Cell cell) {
164 byte[] bytes = new byte[length(cell)];
165 appendToByteArray(cell, bytes, 0);
166 ByteBuffer buffer = ByteBuffer.wrap(bytes);
167 return buffer;
168 }
169
170 public static void appendToByteBuffer(final ByteBuffer bb, final KeyValue kv,
171 final boolean includeMvccVersion) {
172
173 bb.limit(bb.position() + kv.getLength());
174 bb.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
175 if (includeMvccVersion) {
176 int numMvccVersionBytes = WritableUtils.getVIntSize(kv.getMvccVersion());
177 ByteBufferUtils.extendLimit(bb, numMvccVersionBytes);
178 ByteBufferUtils.writeVLong(bb, kv.getMvccVersion());
179 }
180 }
181
182
183
184
185
186
187
188
189
190
191
192 public static KeyValue nextShallowCopy(final ByteBuffer bb, final boolean includesMvccVersion,
193 boolean includesTags) {
194 if (bb.isDirect()) {
195 throw new IllegalArgumentException("only supports heap buffers");
196 }
197 if (bb.remaining() < 1) {
198 return null;
199 }
200 KeyValue keyValue = null;
201 int underlyingArrayOffset = bb.arrayOffset() + bb.position();
202 int keyLength = bb.getInt();
203 int valueLength = bb.getInt();
204 ByteBufferUtils.skip(bb, keyLength + valueLength);
205 int tagsLength = 0;
206 if (includesTags) {
207
208 tagsLength = ((bb.get() & 0xff) << 8) ^ (bb.get() & 0xff);
209 ByteBufferUtils.skip(bb, tagsLength);
210 }
211 int kvLength = (int) KeyValue.getKeyValueDataStructureSize(keyLength, valueLength, tagsLength);
212 keyValue = new KeyValue(bb.array(), underlyingArrayOffset, kvLength);
213 if (includesMvccVersion) {
214 long mvccVersion = ByteBufferUtils.readVLong(bb);
215 keyValue.setSequenceId(mvccVersion);
216 }
217 return keyValue;
218 }
219
220
221
222
223
224
225
226 public static KeyValue createFirstKeyInNextRow(final Cell in){
227 byte[] nextRow = new byte[in.getRowLength() + 1];
228 System.arraycopy(in.getRowArray(), in.getRowOffset(), nextRow, 0, in.getRowLength());
229 nextRow[nextRow.length - 1] = 0;
230 return createFirstOnRow(nextRow);
231 }
232
233
234
235
236 public static KeyValue createFirstKeyInIncrementedRow(final Cell in){
237 byte[] thisRow = new SimpleMutableByteRange(in.getRowArray(), in.getRowOffset(),
238 in.getRowLength()).deepCopyToNewArray();
239 byte[] nextRow = Bytes.unsignedCopyAndIncrement(thisRow);
240 return createFirstOnRow(nextRow);
241 }
242
243
244
245
246
247
248
249
250 public static KeyValue previousKey(final KeyValue in) {
251 return createFirstOnRow(CellUtil.cloneRow(in), CellUtil.cloneFamily(in),
252 CellUtil.cloneQualifier(in), in.getTimestamp() - 1);
253 }
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281 public static KeyValue createLastOnRow(final byte[] row, final int roffset, final int rlength,
282 final byte[] family, final int foffset, final int flength, final byte[] qualifier,
283 final int qoffset, final int qlength) {
284 return new KeyValue(row, roffset, rlength, family, foffset, flength, qualifier, qoffset,
285 qlength, HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0);
286 }
287
288
289
290
291
292
293
294 public static KeyValue createLastOnRow(Cell kv) {
295 return createLastOnRow(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), null, 0, 0,
296 null, 0, 0);
297 }
298
299
300
301
302
303
304
305
306
307
308
309
310 public static KeyValue createLastOnRowCol(Cell kv) {
311 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
312 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
313 kv.getQualifierOffset(), kv.getQualifierLength(), HConstants.OLDEST_TIMESTAMP,
314 Type.Minimum, null, 0, 0);
315 }
316
317
318
319
320
321
322
323
324
325
326
327 public static KeyValue createFirstOnRowColTS(Cell kv, long ts) {
328 return new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
329 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), kv.getQualifierArray(),
330 kv.getQualifierOffset(), kv.getQualifierLength(), ts, Type.Maximum, kv.getValueArray(),
331 kv.getValueOffset(), kv.getValueLength());
332 }
333
334
335
336
337
338
339
340
341
342 public static KeyValue createFirstOnRow(final byte [] row, int roffset, short rlength) {
343 return new KeyValue(row, roffset, rlength,
344 null, 0, 0, null, 0, 0, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
345 }
346
347
348
349
350
351
352
353
354
355 public static KeyValue createLastOnRow(final byte[] row) {
356 return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum);
357 }
358
359
360
361
362
363
364
365
366
367 public static KeyValue createFirstOnRow(final byte [] row) {
368 return createFirstOnRow(row, HConstants.LATEST_TIMESTAMP);
369 }
370
371
372
373
374
375
376
377
378 public static KeyValue createFirstOnRow(final byte [] row,
379 final long ts) {
380 return new KeyValue(row, null, null, ts, Type.Maximum);
381 }
382
383
384
385
386
387
388
389
390
391
392 public static KeyValue createFirstOnRow(final byte [] row, final byte [] family,
393 final byte [] qualifier) {
394 return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum);
395 }
396
397
398
399
400
401
402
403
404
405
406 public static KeyValue createFirstDeleteFamilyOnRow(final byte [] row,
407 final byte [] family) {
408 return new KeyValue(row, family, null, HConstants.LATEST_TIMESTAMP,
409 Type.DeleteFamily);
410 }
411
412
413
414
415
416
417
418
419 public static KeyValue createFirstOnRow(final byte [] row, final byte [] f,
420 final byte [] q, final long ts) {
421 return new KeyValue(row, f, q, ts, Type.Maximum);
422 }
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440 public static KeyValue createFirstOnRow(final byte [] row,
441 final int roffset, final int rlength, final byte [] family,
442 final int foffset, final int flength, final byte [] qualifier,
443 final int qoffset, final int qlength) {
444 return new KeyValue(row, roffset, rlength, family,
445 foffset, flength, qualifier, qoffset, qlength,
446 HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0);
447 }
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465 public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
466 final byte [] family, final byte [] qualifier)
467 throws IllegalArgumentException {
468 return createFirstOnRow(buffer, 0, row, 0, row.length,
469 family, 0, family.length,
470 qualifier, 0, qualifier.length);
471 }
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496 public static KeyValue createFirstOnRow(byte[] buffer, final int boffset, final byte[] row,
497 final int roffset, final int rlength, final byte[] family, final int foffset,
498 final int flength, final byte[] qualifier, final int qoffset, final int qlength)
499 throws IllegalArgumentException {
500
501 long lLength = KeyValue.getKeyValueDataStructureSize(rlength, flength, qlength, 0);
502
503 if (lLength > Integer.MAX_VALUE) {
504 throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
505 }
506 int iLength = (int) lLength;
507 if (buffer.length - boffset < iLength) {
508 throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < "
509 + iLength);
510 }
511
512 int len = KeyValue.writeByteArray(buffer, boffset, row, roffset, rlength, family, foffset,
513 flength, qualifier, qoffset, qlength, HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
514 null, 0, 0, null);
515 return new KeyValue(buffer, boffset, len);
516 }
517
518
519
520
521
522
523
524
525 public static KeyValue createFirstOnRowColTS(KeyValue kv, long ts) {
526 return new KeyValue(
527 kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
528 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
529 kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(),
530 ts, Type.Maximum, kv.getValueArray(), kv.getValueOffset(), kv.getValueLength());
531 }
532
533
534
535
536
537
538
539
540 @Deprecated
541 public static KeyValue ensureKeyValue(final Cell cell) {
542 if (cell == null) return null;
543 return cell instanceof KeyValue? (KeyValue)cell: copyToNewKeyValue(cell);
544 }
545
546 @Deprecated
547 public static List<KeyValue> ensureKeyValues(List<Cell> cells) {
548 List<KeyValue> lazyList = Lists.transform(cells, new Function<Cell, KeyValue>() {
549 @Override
550 public KeyValue apply(Cell arg0) {
551 return KeyValueUtil.ensureKeyValue(arg0);
552 }
553 });
554 return new ArrayList<KeyValue>(lazyList);
555 }
556
557
558
559
560
561
562
563
564
565
566
567
568 public static KeyValue iscreate(final InputStream in, boolean withTags) throws IOException {
569 byte[] intBytes = new byte[Bytes.SIZEOF_INT];
570 int bytesRead = 0;
571 while (bytesRead < intBytes.length) {
572 int n = in.read(intBytes, bytesRead, intBytes.length - bytesRead);
573 if (n < 0) {
574 if (bytesRead == 0)
575 return null;
576 throw new IOException("Failed read of int, read " + bytesRead + " bytes");
577 }
578 bytesRead += n;
579 }
580
581 byte[] bytes = new byte[Bytes.toInt(intBytes)];
582 IOUtils.readFully(in, bytes, 0, bytes.length);
583 if (withTags) {
584 return new KeyValue(bytes, 0, bytes.length);
585 } else {
586 return new NoTagsKeyValue(bytes, 0, bytes.length);
587 }
588 }
589
590 public static void oswrite(final Cell cell, final OutputStream out, final boolean withTags)
591 throws IOException {
592 if (cell instanceof KeyValue) {
593 KeyValue.oswrite((KeyValue) cell, out, withTags);
594 } else {
595 short rlen = cell.getRowLength();
596 byte flen = cell.getFamilyLength();
597 int qlen = cell.getQualifierLength();
598 int vlen = cell.getValueLength();
599 int tlen = cell.getTagsLength();
600
601
602 StreamUtils.writeInt(out, length(rlen, flen, qlen, vlen, tlen, withTags));
603
604 StreamUtils.writeInt(out, keyLength(rlen, flen, qlen));
605
606 StreamUtils.writeInt(out, vlen);
607
608 StreamUtils.writeShort(out, rlen);
609 out.write(cell.getRowArray(), cell.getRowOffset(), rlen);
610
611 out.write(flen);
612 out.write(cell.getFamilyArray(), cell.getFamilyOffset(), flen);
613
614 out.write(cell.getQualifierArray(), cell.getQualifierOffset(), qlen);
615
616 StreamUtils.writeLong(out, cell.getTimestamp());
617
618 out.write(cell.getTypeByte());
619
620 out.write(cell.getValueArray(), cell.getValueOffset(), vlen);
621
622 if (withTags && tlen > 0) {
623
624
625
626 out.write((byte) (0xff & (tlen >> 8)));
627 out.write((byte) (0xff & tlen));
628 out.write(cell.getTagsArray(), cell.getTagsOffset(), tlen);
629 }
630 }
631 }
632 }