View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver.compactions;
19  
20  import static org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.STRIPE_END_KEY;
21  import static org.apache.hadoop.hbase.regionserver.StripeStoreFileManager.STRIPE_START_KEY;
22  import static org.junit.Assert.assertArrayEquals;
23  import static org.junit.Assert.assertEquals;
24  import static org.junit.Assert.assertFalse;
25  import static org.junit.Assert.assertNotNull;
26  import static org.junit.Assert.assertNull;
27  import static org.junit.Assert.assertTrue;
28  import static org.mockito.Matchers.any;
29  import static org.mockito.Matchers.anyBoolean;
30  import static org.mockito.Matchers.anyLong;
31  import static org.mockito.Mockito.doAnswer;
32  import static org.mockito.Mockito.mock;
33  import static org.mockito.Mockito.when;
34  
35  import java.io.IOException;
36  import java.util.ArrayList;
37  import java.util.Arrays;
38  import java.util.List;
39  import java.util.TreeMap;
40  
41  import org.apache.hadoop.fs.Path;
42  import org.apache.hadoop.hbase.Cell;
43  import org.apache.hadoop.hbase.KeyValue;
44  import org.apache.hadoop.hbase.io.hfile.HFile;
45  import org.apache.hadoop.hbase.regionserver.BloomType;
46  import org.apache.hadoop.hbase.regionserver.InternalScanner;
47  import org.apache.hadoop.hbase.regionserver.ScannerContext;
48  import org.apache.hadoop.hbase.regionserver.StoreFile;
49  import org.apache.hadoop.hbase.regionserver.StoreFileScanner;
50  import org.apache.hadoop.hbase.regionserver.StripeMultiFileWriter;
51  import org.apache.hadoop.hbase.util.Bytes;
52  import org.mockito.invocation.InvocationOnMock;
53  import org.mockito.stubbing.Answer;
54  
55  public class TestCompactor {
56  
57    public static StoreFile createDummyStoreFile(long maxSequenceId) throws Exception {
58      // "Files" are totally unused, it's Scanner class below that gives compactor fake KVs.
59      // But compaction depends on everything under the sun, so stub everything with dummies.
60      StoreFile sf = mock(StoreFile.class);
61      StoreFile.Reader r = mock(StoreFile.Reader.class);
62      when(r.length()).thenReturn(1L);
63      when(r.getBloomFilterType()).thenReturn(BloomType.NONE);
64      when(r.getHFileReader()).thenReturn(mock(HFile.Reader.class));
65      when(r.getStoreFileScanner(anyBoolean(), anyBoolean(), anyBoolean(), anyLong()))
66          .thenReturn(mock(StoreFileScanner.class));
67      when(sf.getReader()).thenReturn(r);
68      when(sf.createReader()).thenReturn(r);
69      when(sf.createReader()).thenReturn(r);
70      when(sf.cloneForReader()).thenReturn(sf);
71      when(sf.getMaxSequenceId()).thenReturn(maxSequenceId);
72      return sf;
73    }
74  
75    public static CompactionRequest createDummyRequest() throws Exception {
76      return new CompactionRequest(Arrays.asList(createDummyStoreFile(1L)));
77    }
78  
79    // StoreFile.Writer has private ctor and is unwieldy, so this has to be convoluted.
80    public static class StoreFileWritersCapture
81        implements Answer<StoreFile.Writer>, StripeMultiFileWriter.WriterFactory {
82      public static class Writer {
83        public ArrayList<KeyValue> kvs = new ArrayList<KeyValue>();
84        public TreeMap<byte[], byte[]> data = new TreeMap<byte[], byte[]>(Bytes.BYTES_COMPARATOR);
85        public boolean hasMetadata;
86      }
87  
88      private List<Writer> writers = new ArrayList<Writer>();
89  
90      @Override
91      public StoreFile.Writer createWriter() throws IOException {
92        final Writer realWriter = new Writer();
93        writers.add(realWriter);
94        StoreFile.Writer writer = mock(StoreFile.Writer.class);
95        doAnswer(new Answer<Object>() {
96          public Object answer(InvocationOnMock invocation) {
97            return realWriter.kvs.add((KeyValue) invocation.getArguments()[0]);
98          }
99        }).when(writer).append(any(KeyValue.class));
100       doAnswer(new Answer<Object>() {
101         public Object answer(InvocationOnMock invocation) {
102           Object[] args = invocation.getArguments();
103           return realWriter.data.put((byte[]) args[0], (byte[]) args[1]);
104         }
105       }).when(writer).appendFileInfo(any(byte[].class), any(byte[].class));
106       doAnswer(new Answer<Void>() {
107         @Override
108         public Void answer(InvocationOnMock invocation) throws Throwable {
109           realWriter.hasMetadata = true;
110           return null;
111         }
112       }).when(writer).appendMetadata(any(long.class), any(boolean.class));
113       doAnswer(new Answer<Path>() {
114         @Override
115         public Path answer(InvocationOnMock invocation) throws Throwable {
116           return new Path("foo");
117         }
118       }).when(writer).getPath();
119       return writer;
120     }
121 
122     @Override
123     public StoreFile.Writer answer(InvocationOnMock invocation) throws Throwable {
124       return createWriter();
125     }
126 
127     public void verifyKvs(KeyValue[][] kvss, boolean allFiles, boolean requireMetadata) {
128       if (allFiles) {
129         assertEquals(kvss.length, writers.size());
130       }
131       int skippedWriters = 0;
132       for (int i = 0; i < kvss.length; ++i) {
133         KeyValue[] kvs = kvss[i];
134         if (kvs != null) {
135           Writer w = writers.get(i - skippedWriters);
136           if (requireMetadata) {
137             assertNotNull(w.data.get(STRIPE_START_KEY));
138             assertNotNull(w.data.get(STRIPE_END_KEY));
139           } else {
140             assertNull(w.data.get(STRIPE_START_KEY));
141             assertNull(w.data.get(STRIPE_END_KEY));
142           }
143           assertEquals(kvs.length, w.kvs.size());
144           for (int j = 0; j < kvs.length; ++j) {
145             assertEquals(kvs[j], w.kvs.get(j));
146           }
147         } else {
148           assertFalse(allFiles);
149           ++skippedWriters;
150         }
151       }
152     }
153 
154     public void verifyBoundaries(byte[][] boundaries) {
155       assertEquals(boundaries.length - 1, writers.size());
156       for (int i = 0; i < writers.size(); ++i) {
157         assertArrayEquals("i = " + i, boundaries[i], writers.get(i).data.get(STRIPE_START_KEY));
158         assertArrayEquals("i = " + i, boundaries[i + 1], writers.get(i).data.get(STRIPE_END_KEY));
159       }
160     }
161 
162     public void verifyKvs(KeyValue[][] kvss, boolean allFiles, List<Long> boundaries) {
163       if (allFiles) {
164         assertEquals(kvss.length, writers.size());
165       }
166       int skippedWriters = 0;
167       for (int i = 0; i < kvss.length; ++i) {
168         KeyValue[] kvs = kvss[i];
169         if (kvs != null) {
170           Writer w = writers.get(i - skippedWriters);
171           assertEquals(kvs.length, w.kvs.size());
172           for (int j = 0; j < kvs.length; ++j) {
173             assertTrue(kvs[j].getTimestamp() >= boundaries.get(i));
174             assertTrue(kvs[j].getTimestamp() < boundaries.get(i + 1));
175             assertEquals(kvs[j], w.kvs.get(j));
176           }
177         } else {
178           assertFalse(allFiles);
179           ++skippedWriters;
180         }
181       }
182     }
183 
184     public List<Writer> getWriters() {
185       return writers;
186     }
187   }
188 
189   public static class Scanner implements InternalScanner {
190     private final ArrayList<KeyValue> kvs;
191 
192     public Scanner(KeyValue... kvs) {
193       this.kvs = new ArrayList<KeyValue>(Arrays.asList(kvs));
194     }
195 
196     @Override
197     public boolean next(List<Cell> results) throws IOException {
198       if (kvs.isEmpty()) return false;
199       results.add(kvs.remove(0));
200       return !kvs.isEmpty();
201     }
202 
203     @Override
204     public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
205       return next(result);
206     }
207 
208     @Override
209     public void close() throws IOException {
210     }
211   }
212 }