View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.mockito.Matchers.anyObject;
22  import static org.mockito.Mockito.doReturn;
23  import static org.mockito.Mockito.mock;
24  import static org.mockito.Mockito.reset;
25  import static org.mockito.Mockito.times;
26  import static org.mockito.Mockito.verify;
27  
28  import java.math.BigDecimal;
29  import java.util.List;
30  import java.util.NavigableMap;
31  import java.util.Random;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.hbase.HBaseTestingUtility;
36  import org.apache.hadoop.hbase.HConstants;
37  import org.apache.hadoop.hbase.HRegionInfo;
38  import org.apache.hadoop.hbase.MetaTableAccessor;
39  import org.apache.hadoop.hbase.ServerName;
40  import org.apache.hadoop.hbase.TableName;
41  import org.apache.hadoop.hbase.testclassification.MediumTests;
42  import org.apache.hadoop.hbase.util.Bytes;
43  import org.apache.hadoop.hbase.util.StoppableImplementation;
44  import org.apache.hadoop.hbase.util.Threads;
45  import org.apache.hadoop.util.StringUtils;
46  import org.junit.After;
47  import org.junit.Assert;
48  import org.junit.Test;
49  import org.junit.experimental.categories.Category;
50  
51  @Category(MediumTests.class)
52  public class TestMetaScanner {
53    final Log LOG = LogFactory.getLog(getClass());
54    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
55    private Connection connection;
56  
57    public void setUp() throws Exception {
58      TEST_UTIL.startMiniCluster(1);
59      this.connection = TEST_UTIL.getConnection();
60    }
61  
62    @After
63    public void tearDown() throws Exception {
64      TEST_UTIL.shutdownMiniCluster();
65    }
66  
67    @Test
68    public void testMetaScanner() throws Exception {
69      LOG.info("Starting testMetaScanner");
70  
71      setUp();
72      final TableName TABLENAME = TableName.valueOf("testMetaScanner");
73      final byte[] FAMILY = Bytes.toBytes("family");
74      final byte[][] SPLIT_KEYS =
75          new byte[][] { Bytes.toBytes("region_a"), Bytes.toBytes("region_b") };
76  
77      TEST_UTIL.createTable(TABLENAME, FAMILY, SPLIT_KEYS);
78      HTable table = (HTable) connection.getTable(TABLENAME);
79      // Make sure all the regions are deployed
80      TEST_UTIL.countRows(table);
81  
82      MetaScanner.MetaScannerVisitor visitor =
83        mock(MetaScanner.MetaScannerVisitor.class);
84      doReturn(true).when(visitor).processRow((Result)anyObject());
85  
86      // Scanning the entire table should give us three rows
87      MetaScanner.metaScan(connection, visitor, TABLENAME);
88      verify(visitor, times(3)).processRow((Result)anyObject());
89  
90      // Scanning the table with a specified empty start row should also
91      // give us three hbase:meta rows
92      reset(visitor);
93      doReturn(true).when(visitor).processRow((Result)anyObject());
94      MetaScanner.metaScan(connection, visitor, TABLENAME, HConstants.EMPTY_BYTE_ARRAY, 1000);
95      verify(visitor, times(3)).processRow((Result)anyObject());
96  
97      // Scanning the table starting in the middle should give us two rows:
98      // region_a and region_b
99      reset(visitor);
100     doReturn(true).when(visitor).processRow((Result)anyObject());
101     MetaScanner.metaScan(connection, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1000);
102     verify(visitor, times(2)).processRow((Result)anyObject());
103 
104     // Scanning with a limit of 1 should only give us one row
105     reset(visitor);
106     doReturn(true).when(visitor).processRow((Result) anyObject());
107     MetaScanner.metaScan(connection, visitor, TABLENAME, Bytes.toBytes("region_ac"), 1);
108     verify(visitor, times(1)).processRow((Result) anyObject());
109     table.close();
110   }
111 
112   @Test
113   public void testConcurrentMetaScannerAndCatalogJanitor() throws Throwable {
114     /* TEST PLAN: start with only one region in a table. Have a splitter
115      * thread  and metascanner threads that continously scan the meta table for regions.
116      * CatalogJanitor from master will run frequently to clean things up
117      */
118     TEST_UTIL.getConfiguration().setLong("hbase.catalogjanitor.interval", 500);
119     setUp();
120 
121     final long runtime = 30 * 1000; //30 sec
122     LOG.info("Starting testConcurrentMetaScannerAndCatalogJanitor");
123     final TableName TABLENAME =
124         TableName.valueOf("testConcurrentMetaScannerAndCatalogJanitor");
125     final byte[] FAMILY = Bytes.toBytes("family");
126     TEST_UTIL.createTable(TABLENAME, FAMILY);
127 
128     class RegionMetaSplitter extends StoppableImplementation implements Runnable {
129       Random random = new Random();
130       Throwable ex = null;
131       @Override
132       public void run() {
133         while (!isStopped()) {
134           try {
135             List<HRegionInfo> regions = MetaScanner.listAllRegions(TEST_UTIL.getConfiguration(),
136                 connection, false);
137 
138             //select a random region
139             HRegionInfo parent = regions.get(random.nextInt(regions.size()));
140             if (parent == null || !TABLENAME.equals(parent.getTable())) {
141               continue;
142             }
143 
144             long startKey = 0, endKey = Long.MAX_VALUE;
145             byte[] start = parent.getStartKey();
146             byte[] end = parent.getEndKey();
147             if (!Bytes.equals(HConstants.EMPTY_START_ROW, parent.getStartKey())) {
148               startKey = Bytes.toLong(parent.getStartKey());
149             }
150             if (!Bytes.equals(HConstants.EMPTY_END_ROW, parent.getEndKey())) {
151               endKey = Bytes.toLong(parent.getEndKey());
152             }
153             if (startKey == endKey) {
154               continue;
155             }
156 
157             long midKey = BigDecimal.valueOf(startKey).add(BigDecimal.valueOf(endKey))
158                 .divideToIntegralValue(BigDecimal.valueOf(2)).longValue();
159 
160             HRegionInfo splita = new HRegionInfo(TABLENAME,
161               start,
162               Bytes.toBytes(midKey));
163             HRegionInfo splitb = new HRegionInfo(TABLENAME,
164               Bytes.toBytes(midKey),
165               end);
166 
167             MetaTableAccessor.splitRegion(connection,
168               parent, splita, splitb, ServerName.valueOf("fooserver", 1, 0), 1);
169 
170             Threads.sleep(random.nextInt(200));
171           } catch (Throwable e) {
172             ex = e;
173             Assert.fail(StringUtils.stringifyException(e));
174           }
175         }
176       }
177       void rethrowExceptionIfAny() throws Throwable {
178         if (ex != null) { throw ex; }
179       }
180     }
181 
182     class MetaScannerVerifier extends StoppableImplementation implements Runnable {
183       Random random = new Random();
184       Throwable ex = null;
185       @Override
186       public void run() {
187          while(!isStopped()) {
188            try {
189             NavigableMap<HRegionInfo, ServerName> regions =
190                 MetaScanner.allTableRegions(connection, TABLENAME);
191 
192             LOG.info("-------");
193             byte[] lastEndKey = HConstants.EMPTY_START_ROW;
194             for (HRegionInfo hri: regions.navigableKeySet()) {
195               long startKey = 0, endKey = Long.MAX_VALUE;
196               if (!Bytes.equals(HConstants.EMPTY_START_ROW, hri.getStartKey())) {
197                 startKey = Bytes.toLong(hri.getStartKey());
198               }
199               if (!Bytes.equals(HConstants.EMPTY_END_ROW, hri.getEndKey())) {
200                 endKey = Bytes.toLong(hri.getEndKey());
201               }
202               LOG.info("start:" + startKey + " end:" + endKey + " hri:" + hri);
203               Assert.assertTrue("lastEndKey=" + Bytes.toString(lastEndKey) + ", startKey=" +
204                 Bytes.toString(hri.getStartKey()), Bytes.equals(lastEndKey, hri.getStartKey()));
205               lastEndKey = hri.getEndKey();
206             }
207             Assert.assertTrue(Bytes.equals(lastEndKey, HConstants.EMPTY_END_ROW));
208             LOG.info("-------");
209             Threads.sleep(10 + random.nextInt(50));
210           } catch (Throwable e) {
211             ex = e;
212             Assert.fail(StringUtils.stringifyException(e));
213           }
214          }
215       }
216       void rethrowExceptionIfAny() throws Throwable {
217         if (ex != null) { throw ex; }
218       }
219     }
220 
221     RegionMetaSplitter regionMetaSplitter = new RegionMetaSplitter();
222     MetaScannerVerifier metaScannerVerifier = new MetaScannerVerifier();
223 
224     Thread regionMetaSplitterThread = new Thread(regionMetaSplitter);
225     Thread metaScannerVerifierThread = new Thread(metaScannerVerifier);
226 
227     regionMetaSplitterThread.start();
228     metaScannerVerifierThread.start();
229 
230     Threads.sleep(runtime);
231 
232     regionMetaSplitter.stop("test finished");
233     metaScannerVerifier.stop("test finished");
234 
235     regionMetaSplitterThread.join();
236     metaScannerVerifierThread.join();
237 
238     regionMetaSplitter.rethrowExceptionIfAny();
239     metaScannerVerifier.rethrowExceptionIfAny();
240   }
241 
242 }