1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver.compactions;
19
20 import static org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.STRIPE_END_KEY;
21 import static org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.STRIPE_START_KEY;
22 import static org.junit.Assert.assertArrayEquals;
23 import static org.junit.Assert.assertEquals;
24 import static org.junit.Assert.assertFalse;
25 import static org.junit.Assert.assertNotNull;
26 import static org.junit.Assert.assertNull;
27 import static org.junit.Assert.assertTrue;
28 import static org.mockito.Matchers.any;
29 import static org.mockito.Matchers.anyBoolean;
30 import static org.mockito.Matchers.anyLong;
31 import static org.mockito.Mockito.doAnswer;
32 import static org.mockito.Mockito.mock;
33 import static org.mockito.Mockito.when;
34
35 import java.io.IOException;
36 import java.util.ArrayList;
37 import java.util.Arrays;
38 import java.util.List;
39 import java.util.TreeMap;
40
41 import org.apache.hadoop.fs.Path;
42 import org.apache.hadoop.hbase.Cell;
43 import org.apache.hadoop.hbase.KeyValue;
44 import org.apache.hadoop.hbase.io.hfile.HFile;
45 import org.apache.hadoop.hbase.regionserver.BloomType;
46 import org.apache.hadoop.hbase.regionserver.InternalScanner;
47 import org.apache.hadoop.hbase.regionserver.ScannerContext;
48 import org.apache.hadoop.hbase.regionserver.StoreFile;
49 import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
50 import org.apache.hadoop.hbase.regionserver.StripeMultiFileWriter;
51 import org.apache.hadoop.hbase.util.Bytes;
52 import org.mockito.invocation.InvocationOnMock;
53 import org.mockito.stubbing.Answer;
54
55 public class TestCompactor {
56
57 public static StoreFile createDummyStoreFile(long maxSequenceId) throws Exception {
58
59
60 StoreFile sf = mock(StoreFile.class);
61 StoreFile.Reader r = mock(StoreFile.Reader.class);
62 when(r.length()).thenReturn(1L);
63 when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
64 when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
65 when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong()))
66 .thenReturn(mock(StoreFileScanner.class));
67 when(sf.getReader()).thenReturn(r);
68 when(sf.createReader()).thenReturn(r);
69 when(sf.createReader()).thenReturn(r);
70 when(sf.cloneForReader()).thenReturn(sf);
71 when(sf.getMaxSequenceId()).thenReturn(maxSequenceId);
72 return sf;
73 }
74
75 public static CompactionRequest createDummyRequest() throws Exception {
76 return new CompactionRequest(Arrays.asList(createDummyStoreFile(1L)));
77 }
78
79
80 public static class StoreFileWritersCapture
81 implements Answer<StoreFile.Writer>, StripeMultiFileWriter.WriterFactory {
82 public static class Writer {
83 public ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
84 public TreeMap<byte[], byte[]> data = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
85 public boolean hasMetadata;
86 }
87
88 private List<Writer> writers = new ArrayList<Writer>();
89
90 @Override
91 public StoreFile.Writer createWriter() throws IOException {
92 final Writer realWriter = new Writer();
93 writers.add(realWriter);
94 StoreFile.Writer writer = mock(StoreFile.Writer.class);
95 doAnswer(new Answer<Object>() {
96 public Object answer(InvocationOnMock invocation) {
97 return realWriter.kvs.add((KeyValue) invocation.getArguments()[0]);
98 }
99 }).when(writer).append(any(KeyValue.class));
100 doAnswer(new Answer<Object>() {
101 public Object answer(InvocationOnMock invocation) {
102 Object[] args = invocation.getArguments();
103 return realWriter.data.put((byte[]) args[0], (byte[]) args[1]);
104 }
105 }).when(writer).appendFileInfo(any(byte[].class), any(byte[].class));
106 doAnswer(new Answer<Void>() {
107 @Override
108 public Void answer(InvocationOnMock invocation) throws Throwable {
109 realWriter.hasMetadata = true;
110 return null;
111 }
112 }).when(writer).appendMetadata(any(long.class), any(boolean.class));
113 doAnswer(new Answer<Path>() {
114 @Override
115 public Path answer(InvocationOnMock invocation) throws Throwable {
116 return new Path("foo");
117 }
118 }).when(writer).getPath();
119 return writer;
120 }
121
122 @Override
123 public StoreFile.Writer answer(InvocationOnMock invocation) throws Throwable {
124 return createWriter();
125 }
126
127 public void verifyKvs(KeyValue[][] kvss, boolean allFiles, boolean requireMetadata) {
128 if (allFiles) {
129 assertEquals(kvss.length, writers.size());
130 }
131 int skippedWriters = 0;
132 for (int i = 0; i < kvss.length; ++i) {
133 KeyValue[] kvs = kvss[i];
134 if (kvs != null) {
135 Writer w = writers.get(i - skippedWriters);
136 if (requireMetadata) {
137 assertNotNull(w.data.get(STRIPE_START_KEY));
138 assertNotNull(w.data.get(STRIPE_END_KEY));
139 } else {
140 assertNull(w.data.get(STRIPE_START_KEY));
141 assertNull(w.data.get(STRIPE_END_KEY));
142 }
143 assertEquals(kvs.length, w.kvs.size());
144 for (int j = 0; j < kvs.length; ++j) {
145 assertEquals(kvs[j], w.kvs.get(j));
146 }
147 } else {
148 assertFalse(allFiles);
149 ++skippedWriters;
150 }
151 }
152 }
153
154 public void verifyBoundaries(byte[][] boundaries) {
155 assertEquals(boundaries.length - 1, writers.size());
156 for (int i = 0; i < writers.size(); ++i) {
157 assertArrayEquals("i = " + i, boundaries[i], writers.get(i).data.get(STRIPE_START_KEY));
158 assertArrayEquals("i = " + i, boundaries[i + 1], writers.get(i).data.get(STRIPE_END_KEY));
159 }
160 }
161
162 public void verifyKvs(KeyValue[][] kvss, boolean allFiles, List<Long> boundaries) {
163 if (allFiles) {
164 assertEquals(kvss.length, writers.size());
165 }
166 int skippedWriters = 0;
167 for (int i = 0; i < kvss.length; ++i) {
168 KeyValue[] kvs = kvss[i];
169 if (kvs != null) {
170 Writer w = writers.get(i - skippedWriters);
171 assertEquals(kvs.length, w.kvs.size());
172 for (int j = 0; j < kvs.length; ++j) {
173 assertTrue(kvs[j].getTimestamp() >= boundaries.get(i));
174 assertTrue(kvs[j].getTimestamp() < boundaries.get(i + 1));
175 assertEquals(kvs[j], w.kvs.get(j));
176 }
177 } else {
178 assertFalse(allFiles);
179 ++skippedWriters;
180 }
181 }
182 }
183
184 public List<Writer> getWriters() {
185 return writers;
186 }
187 }
188
189 public static class Scanner implements InternalScanner {
190 private final ArrayList<KeyValue> kvs;
191
192 public Scanner(KeyValue... kvs) {
193 this.kvs = new ArrayList<KeyValue>(Arrays.asList(kvs));
194 }
195
196 @Override
197 public boolean next(List<Cell> results) throws IOException {
198 if (kvs.isEmpty()) return false;
199 results.add(kvs.remove(0));
200 return !kvs.isEmpty();
201 }
202
203 @Override
204 public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
205 return next(result);
206 }
207
208 @Override
209 public void close() throws IOException {
210 }
211 }
212 }