1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import org.apache.hadoop.conf.Configuration;
22 import org.apache.hadoop.fs.FileSystem;
23 import org.apache.hadoop.fs.Path;
24 import org.apache.hadoop.hbase.Cell;
25 import org.apache.hadoop.hbase.KeyValue;
26 import org.apache.hadoop.hbase.Tag;
27 import org.apache.hadoop.hbase.TagType;
28 import org.apache.hadoop.hbase.client.Result;
29 import org.apache.hadoop.hbase.client.ResultScanner;
30 import org.apache.hadoop.hbase.client.Scan;
31 import org.apache.hadoop.hbase.client.Table;
32 import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
33 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
34 import org.apache.hadoop.hbase.io.hfile.HFile;
35 import org.apache.hadoop.hbase.io.hfile.HFileContext;
36 import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder;
37 import org.apache.hadoop.hbase.regionserver.StoreFile;
38
39 import java.io.IOException;
40 import java.util.ArrayList;
41 import java.util.List;
42
43 import static org.junit.Assert.assertArrayEquals;
44 import static org.junit.Assert.fail;
45
46
47
48
49 public class HFileTestUtil {
50
51
52
53
54
55
56 public static void createHFile(
57 Configuration configuration,
58 FileSystem fs, Path path,
59 byte[] family, byte[] qualifier,
60 byte[] startKey, byte[] endKey, int numRows) throws IOException {
61 createHFile(configuration, fs, path, DataBlockEncoding.NONE, family, qualifier,
62 startKey, endKey, numRows, false);
63 }
64
65
66
67
68
69
70 public static void createHFileWithDataBlockEncoding(
71 Configuration configuration,
72 FileSystem fs, Path path, DataBlockEncoding encoding,
73 byte[] family, byte[] qualifier,
74 byte[] startKey, byte[] endKey, int numRows) throws IOException {
75 createHFile(configuration, fs, path, encoding, family, qualifier, startKey, endKey,
76 numRows, false);
77 }
78
79
80
81
82
83
84 public static void createHFileWithTags(
85 Configuration configuration,
86 FileSystem fs, Path path,
87 byte[] family, byte[] qualifier,
88 byte[] startKey, byte[] endKey, int numRows) throws IOException {
89 createHFile(configuration, fs, path, DataBlockEncoding.NONE, family, qualifier,
90 startKey, endKey, numRows, true);
91 }
92
93
94
95
96
97
98
99 public static void createHFile(
100 Configuration configuration,
101 FileSystem fs, Path path, DataBlockEncoding encoding,
102 byte[] family, byte[] qualifier,
103 byte[] startKey, byte[] endKey, int numRows, boolean withTag) throws IOException {
104 HFileContext meta = new HFileContextBuilder()
105 .withIncludesTags(withTag)
106 .withDataBlockEncoding(encoding)
107 .build();
108 HFile.Writer writer = HFile.getWriterFactory(configuration, new CacheConfig(configuration))
109 .withPath(fs, path)
110 .withFileContext(meta)
111 .create();
112 long now = System.currentTimeMillis();
113 try {
114
115 for (byte[] key : Bytes.iterateOnSplits(startKey, endKey, numRows - 2)) {
116 KeyValue kv = new KeyValue(key, family, qualifier, now, key);
117 if (withTag) {
118
119 List<Tag> tags = new ArrayList<Tag>();
120 tags.add(new Tag(TagType.ACL_TAG_TYPE, key));
121 kv = new KeyValue(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(),
122 kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(),
123 kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(),
124 kv.getTimestamp(), KeyValue.Type.Put, kv.getValueArray(), kv.getValueOffset(),
125 kv.getValueLength(), tags);
126
127
128 byte[] ta = kv.getTagsArray();
129 int toff = kv.getTagsOffset();
130 int tlen = kv.getTagsLength();
131 Tag t = Tag.getTag(ta, toff, tlen, TagType.ACL_TAG_TYPE);
132 if (t == null) {
133 throw new IllegalStateException("Tag didn't stick to KV " + kv.toString());
134 }
135 }
136 writer.append(kv);
137 }
138 } finally {
139 writer.appendFileInfo(StoreFile.BULKLOAD_TIME_KEY,
140 Bytes.toBytes(System.currentTimeMillis()));
141 writer.close();
142 }
143 }
144
145
146
147
148
149
150
151
152 public static void verifyTags(Table table) throws IOException {
153 ResultScanner s = table.getScanner(new Scan());
154 for (Result r : s) {
155 for (Cell c : r.listCells()) {
156 byte[] ta = c.getTagsArray();
157 int toff = c.getTagsOffset();
158 int tlen = c.getTagsLength();
159 Tag t = Tag.getTag(ta, toff, tlen, TagType.ACL_TAG_TYPE);
160 if (t == null) {
161 fail(c.toString() + " has null tag");
162 continue;
163 }
164 byte[] tval = t.getValue();
165 assertArrayEquals(c.toString() + " has tag" + Bytes.toString(tval),
166 r.getRow(), tval);
167 }
168 }
169 }
170 }