View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertTrue;
22  import static org.junit.Assert.fail;
23  
24  import java.io.IOException;
25  import java.util.Collections;
26  import java.util.Comparator;
27  import java.util.HashMap;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.concurrent.CountDownLatch;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FileSystem;
36  import org.apache.hadoop.fs.Path;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HRegionInfo;
40  import org.apache.hadoop.hbase.client.HTable;
41  import org.apache.hadoop.hbase.testclassification.LargeTests;
42  import org.apache.hadoop.hbase.TableName;
43  import org.apache.hadoop.hbase.TableNotFoundException;
44  import org.apache.hadoop.hbase.client.Admin;
45  import org.apache.hadoop.hbase.client.Table;
46  import org.apache.hadoop.hbase.master.HMaster;
47  import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
48  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
49  import org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.FSUtils;
52  import org.junit.After;
53  import org.junit.AfterClass;
54  import org.junit.Before;
55  import org.junit.BeforeClass;
56  import org.junit.Test;
57  import org.junit.experimental.categories.Category;
58  
59  /**
60   * Test creating/using/deleting snapshots from the client
61   * <p>
62   * This is an end-to-end test for the snapshot utility
63   *
64   * TODO This is essentially a clone of TestSnapshotFromClient.  This is worth refactoring this
65   * because there will be a few more flavors of snapshots that need to run these tests.
66   */
67  @Category(LargeTests.class)
68  public class TestFlushSnapshotFromClient {
69    private static final Log LOG = LogFactory.getLog(TestFlushSnapshotFromClient.class);
70  
71    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
72    protected static final int NUM_RS = 2;
73    protected static final byte[] TEST_FAM = Bytes.toBytes("fam");
74    protected static final TableName TABLE_NAME = TableName.valueOf("test");
75    protected final int DEFAULT_NUM_ROWS = 100;
76  
77    @BeforeClass
78    public static void setupCluster() throws Exception {
79      setupConf(UTIL.getConfiguration());
80      UTIL.startMiniCluster(NUM_RS);
81    }
82  
83    protected static void setupConf(Configuration conf) {
84      // disable the ui
85      conf.setInt("hbase.regionsever.info.port", -1);
86      // change the flush size to a small amount, regulating number of store files
87      conf.setInt("hbase.hregion.memstore.flush.size", 25000);
88      // so make sure we get a compaction when doing a load, but keep around some
89      // files in the store
90      conf.setInt("hbase.hstore.compaction.min", 10);
91      conf.setInt("hbase.hstore.compactionThreshold", 10);
92      // block writes if we get to 12 store files
93      conf.setInt("hbase.hstore.blockingStoreFiles", 12);
94      // Enable snapshot
95      conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
96      conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
97        ConstantSizeRegionSplitPolicy.class.getName());
98    }
99  
100   @Before
101   public void setup() throws Exception {
102     createTable();
103   }
104 
105   protected void createTable() throws Exception {
106     SnapshotTestingUtils.createTable(UTIL, TABLE_NAME, TEST_FAM);
107   }
108 
109   @After
110   public void tearDown() throws Exception {
111     UTIL.deleteTable(TABLE_NAME);
112 
113     SnapshotTestingUtils.deleteAllSnapshots(UTIL.getHBaseAdmin());
114     SnapshotTestingUtils.deleteArchiveDirectory(UTIL);
115   }
116 
117   @AfterClass
118   public static void cleanupTest() throws Exception {
119     try {
120       UTIL.shutdownMiniCluster();
121     } catch (Exception e) {
122       LOG.warn("failure shutting down cluster", e);
123     }
124   }
125 
126   /**
127    * Test simple flush snapshotting a table that is online
128    * @throws Exception
129    */
130   @Test (timeout=300000)
131   public void testFlushTableSnapshot() throws Exception {
132     Admin admin = UTIL.getHBaseAdmin();
133     // make sure we don't fail on listing snapshots
134     SnapshotTestingUtils.assertNoSnapshots(admin);
135 
136     // put some stuff in the table
137     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
138 
139     LOG.debug("FS state before snapshot:");
140     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
141         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
142 
143     // take a snapshot of the enabled table
144     String snapshotString = "offlineTableSnapshot";
145     byte[] snapshot = Bytes.toBytes(snapshotString);
146     admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.FLUSH);
147     LOG.debug("Snapshot completed.");
148 
149     // make sure we have the snapshot
150     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
151       snapshot, TABLE_NAME);
152 
153     // make sure its a valid snapshot
154     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
155     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
156     LOG.debug("FS state after snapshot:");
157     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
158         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
159 
160     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
161         admin, fs);
162   }
163 
164    /**
165    * Test snapshotting a table that is online without flushing
166    * @throws Exception
167    */
168   @Test(timeout=30000)
169   public void testSkipFlushTableSnapshot() throws Exception {
170     Admin admin = UTIL.getHBaseAdmin();
171     // make sure we don't fail on listing snapshots
172     SnapshotTestingUtils.assertNoSnapshots(admin);
173 
174     // put some stuff in the table
175     try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) {
176       UTIL.loadTable(table, TEST_FAM);
177     }
178 
179     LOG.debug("FS state before snapshot:");
180     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
181         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
182 
183     // take a snapshot of the enabled table
184     String snapshotString = "skipFlushTableSnapshot";
185     byte[] snapshot = Bytes.toBytes(snapshotString);
186     admin.snapshot(snapshotString, TABLE_NAME, SnapshotDescription.Type.SKIPFLUSH);
187     LOG.debug("Snapshot completed.");
188 
189     // make sure we have the snapshot
190     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
191         snapshot, TABLE_NAME);
192 
193     // make sure its a valid snapshot
194     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
195     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
196     LOG.debug("FS state after snapshot:");
197     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
198         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
199 
200     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
201         admin, fs);
202 
203     admin.deleteSnapshot(snapshot);
204     snapshots = admin.listSnapshots();
205     SnapshotTestingUtils.assertNoSnapshots(admin);
206   }
207 
208 
209   /**
210    * Test simple flush snapshotting a table that is online
211    * @throws Exception
212    */
213   @Test (timeout=300000)
214   public void testFlushTableSnapshotWithProcedure() throws Exception {
215     Admin admin = UTIL.getHBaseAdmin();
216     // make sure we don't fail on listing snapshots
217     SnapshotTestingUtils.assertNoSnapshots(admin);
218 
219     // put some stuff in the table
220     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
221 
222     LOG.debug("FS state before snapshot:");
223     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
224         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
225 
226     // take a snapshot of the enabled table
227     String snapshotString = "offlineTableSnapshot";
228     byte[] snapshot = Bytes.toBytes(snapshotString);
229     Map<String, String> props = new HashMap<String, String>();
230     props.put("table", TABLE_NAME.getNameAsString());
231     admin.execProcedure(SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION,
232         snapshotString, props);
233 
234 
235     LOG.debug("Snapshot completed.");
236 
237     // make sure we have the snapshot
238     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertOneSnapshotThatMatches(admin,
239       snapshot, TABLE_NAME);
240 
241     // make sure its a valid snapshot
242     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
243     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
244     LOG.debug("FS state after snapshot:");
245     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
246         FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
247 
248     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), TABLE_NAME, TEST_FAM, rootDir,
249         admin, fs);
250   }
251 
252   @Test (timeout=300000)
253   public void testSnapshotFailsOnNonExistantTable() throws Exception {
254     Admin admin = UTIL.getHBaseAdmin();
255     // make sure we don't fail on listing snapshots
256     SnapshotTestingUtils.assertNoSnapshots(admin);
257     TableName tableName = TableName.valueOf("_not_a_table");
258 
259     // make sure the table doesn't exist
260     boolean fail = false;
261     do {
262     try {
263       admin.getTableDescriptor(tableName);
264       fail = true;
265       LOG.error("Table:" + tableName + " already exists, checking a new name");
266       tableName = TableName.valueOf(tableName+"!");
267     } catch (TableNotFoundException e) {
268       fail = false;
269       }
270     } while (fail);
271 
272     // snapshot the non-existant table
273     try {
274       admin.snapshot("fail", tableName, SnapshotDescription.Type.FLUSH);
275       fail("Snapshot succeeded even though there is not table.");
276     } catch (SnapshotCreationException e) {
277       LOG.info("Correctly failed to snapshot a non-existant table:" + e.getMessage());
278     }
279   }
280 
281   @Test(timeout = 300000)
282   public void testAsyncFlushSnapshot() throws Exception {
283     Admin admin = UTIL.getHBaseAdmin();
284     SnapshotDescription snapshot = SnapshotDescription.newBuilder().setName("asyncSnapshot")
285         .setTable(TABLE_NAME.getNameAsString())
286         .setType(SnapshotDescription.Type.FLUSH)
287         .build();
288 
289     // take the snapshot async
290     admin.takeSnapshotAsync(snapshot);
291 
292     // constantly loop, looking for the snapshot to complete
293     HMaster master = UTIL.getMiniHBaseCluster().getMaster();
294     SnapshotTestingUtils.waitForSnapshotToComplete(master, snapshot, 200);
295     LOG.info(" === Async Snapshot Completed ===");
296     FSUtils.logFileSystemState(UTIL.getTestFileSystem(),
297       FSUtils.getRootDir(UTIL.getConfiguration()), LOG);
298     // make sure we get the snapshot
299     SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshot);
300   }
301 
302   @Test (timeout=300000)
303   public void testSnapshotStateAfterMerge() throws Exception {
304     int numRows = DEFAULT_NUM_ROWS;
305     Admin admin = UTIL.getHBaseAdmin();
306     // make sure we don't fail on listing snapshots
307     SnapshotTestingUtils.assertNoSnapshots(admin);
308     // load the table so we have some data
309     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
310 
311     // Take a snapshot
312     String snapshotBeforeMergeName = "snapshotBeforeMerge";
313     admin.snapshot(snapshotBeforeMergeName, TABLE_NAME, SnapshotDescription.Type.FLUSH);
314 
315     // Clone the table
316     TableName cloneBeforeMergeName = TableName.valueOf("cloneBeforeMerge");
317     admin.cloneSnapshot(snapshotBeforeMergeName, cloneBeforeMergeName);
318     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneBeforeMergeName);
319 
320     // Merge two regions
321     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
322     Collections.sort(regions, new Comparator<HRegionInfo>() {
323       public int compare(HRegionInfo r1, HRegionInfo r2) {
324         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
325       }
326     });
327 
328     int numRegions = admin.getTableRegions(TABLE_NAME).size();
329     int numRegionsAfterMerge = numRegions - 2;
330     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
331         regions.get(2).getEncodedNameAsBytes(), true);
332     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
333         regions.get(6).getEncodedNameAsBytes(), true);
334 
335     // Verify that there's one region less
336     waitRegionsAfterMerge(numRegionsAfterMerge);
337     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
338 
339     // Clone the table
340     TableName cloneAfterMergeName = TableName.valueOf("cloneAfterMerge");
341     admin.cloneSnapshot(snapshotBeforeMergeName, cloneAfterMergeName);
342     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneAfterMergeName);
343 
344     verifyRowCount(UTIL, TABLE_NAME, numRows);
345     verifyRowCount(UTIL, cloneBeforeMergeName, numRows);
346     verifyRowCount(UTIL, cloneAfterMergeName, numRows);
347 
348     // test that we can delete the snapshot
349     UTIL.deleteTable(cloneAfterMergeName);
350     UTIL.deleteTable(cloneBeforeMergeName);
351   }
352 
353   @Test (timeout=300000)
354   public void testTakeSnapshotAfterMerge() throws Exception {
355     int numRows = DEFAULT_NUM_ROWS;
356     Admin admin = UTIL.getHBaseAdmin();
357     // make sure we don't fail on listing snapshots
358     SnapshotTestingUtils.assertNoSnapshots(admin);
359     // load the table so we have some data
360     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, numRows, TEST_FAM);
361 
362     // Merge two regions
363     List<HRegionInfo> regions = admin.getTableRegions(TABLE_NAME);
364     Collections.sort(regions, new Comparator<HRegionInfo>() {
365       public int compare(HRegionInfo r1, HRegionInfo r2) {
366         return Bytes.compareTo(r1.getStartKey(), r2.getStartKey());
367       }
368     });
369 
370     int numRegions = admin.getTableRegions(TABLE_NAME).size();
371     int numRegionsAfterMerge = numRegions - 2;
372     admin.mergeRegions(regions.get(1).getEncodedNameAsBytes(),
373         regions.get(2).getEncodedNameAsBytes(), true);
374     admin.mergeRegions(regions.get(5).getEncodedNameAsBytes(),
375         regions.get(6).getEncodedNameAsBytes(), true);
376 
377     waitRegionsAfterMerge(numRegionsAfterMerge);
378     assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
379 
380     // Take a snapshot
381     String snapshotName = "snapshotAfterMerge";
382     SnapshotTestingUtils.snapshot(admin, snapshotName, TABLE_NAME.getNameAsString(),
383       SnapshotDescription.Type.FLUSH, 3);
384 
385     // Clone the table
386     TableName cloneName = TableName.valueOf("cloneMerge");
387     admin.cloneSnapshot(snapshotName, cloneName);
388     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, cloneName);
389 
390     verifyRowCount(UTIL, TABLE_NAME, numRows);
391     verifyRowCount(UTIL, cloneName, numRows);
392 
393     // test that we can delete the snapshot
394     UTIL.deleteTable(cloneName);
395   }
396 
397   /**
398    * Basic end-to-end test of simple-flush-based snapshots
399    */
400   @Test (timeout=300000)
401   public void testFlushCreateListDestroy() throws Exception {
402     LOG.debug("------- Starting Snapshot test -------------");
403     Admin admin = UTIL.getHBaseAdmin();
404     // make sure we don't fail on listing snapshots
405     SnapshotTestingUtils.assertNoSnapshots(admin);
406     // load the table so we have some data
407     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
408 
409     String snapshotName = "flushSnapshotCreateListDestroy";
410     FileSystem fs = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getFileSystem();
411     Path rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
412     SnapshotTestingUtils.createSnapshotAndValidate(admin, TABLE_NAME, Bytes.toString(TEST_FAM),
413       snapshotName, rootDir, fs, true);
414   }
415 
416   /**
417    * Demonstrate that we reject snapshot requests if there is a snapshot already running on the
418    * same table currently running and that concurrent snapshots on different tables can both
419    * succeed concurretly.
420    */
421   @Test(timeout=300000)
422   public void testConcurrentSnapshottingAttempts() throws IOException, InterruptedException {
423     final TableName TABLE2_NAME = TableName.valueOf(TABLE_NAME + "2");
424 
425     int ssNum = 20;
426     Admin admin = UTIL.getHBaseAdmin();
427     // make sure we don't fail on listing snapshots
428     SnapshotTestingUtils.assertNoSnapshots(admin);
429     // create second testing table
430     SnapshotTestingUtils.createTable(UTIL, TABLE2_NAME, TEST_FAM);
431     // load the table so we have some data
432     SnapshotTestingUtils.loadData(UTIL, TABLE_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
433     SnapshotTestingUtils.loadData(UTIL, TABLE2_NAME, DEFAULT_NUM_ROWS, TEST_FAM);
434 
435     final CountDownLatch toBeSubmitted = new CountDownLatch(ssNum);
436     // We'll have one of these per thread
437     class SSRunnable implements Runnable {
438       SnapshotDescription ss;
439       SSRunnable(SnapshotDescription ss) {
440         this.ss = ss;
441       }
442 
443       @Override
444       public void run() {
445         try {
446           Admin admin = UTIL.getHBaseAdmin();
447           LOG.info("Submitting snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
448           admin.takeSnapshotAsync(ss);
449         } catch (Exception e) {
450           LOG.info("Exception during snapshot request: " + ClientSnapshotDescriptionUtils.toString(
451               ss)
452               + ".  This is ok, we expect some", e);
453         }
454         LOG.info("Submitted snapshot request: " + ClientSnapshotDescriptionUtils.toString(ss));
455         toBeSubmitted.countDown();
456       }
457     };
458 
459     // build descriptions
460     SnapshotDescription[] descs = new SnapshotDescription[ssNum];
461     for (int i = 0; i < ssNum; i++) {
462       SnapshotDescription.Builder builder = SnapshotDescription.newBuilder();
463       builder.setTable(((i % 2) == 0 ? TABLE_NAME : TABLE2_NAME).getNameAsString());
464       builder.setName("ss"+i);
465       builder.setType(SnapshotDescription.Type.FLUSH);
466       descs[i] = builder.build();
467     }
468 
469     // kick each off its own thread
470     for (int i=0 ; i < ssNum; i++) {
471       new Thread(new SSRunnable(descs[i])).start();
472     }
473 
474     // wait until all have been submitted
475     toBeSubmitted.await();
476 
477     // loop until all are done.
478     while (true) {
479       int doneCount = 0;
480       for (SnapshotDescription ss : descs) {
481         try {
482           if (admin.isSnapshotFinished(ss)) {
483             doneCount++;
484           }
485         } catch (Exception e) {
486           LOG.warn("Got an exception when checking for snapshot " + ss.getName(), e);
487           doneCount++;
488         }
489       }
490       if (doneCount == descs.length) {
491         break;
492       }
493       Thread.sleep(100);
494     }
495 
496     // dump for debugging
497     logFSTree(FSUtils.getRootDir(UTIL.getConfiguration()));
498 
499     List<SnapshotDescription> taken = admin.listSnapshots();
500     int takenSize = taken.size();
501     LOG.info("Taken " + takenSize + " snapshots:  " + taken);
502     assertTrue("We expect at least 1 request to be rejected because of we concurrently" +
503         " issued many requests", takenSize < ssNum && takenSize > 0);
504 
505     // Verify that there's at least one snapshot per table
506     int t1SnapshotsCount = 0;
507     int t2SnapshotsCount = 0;
508     for (SnapshotDescription ss : taken) {
509       if (TableName.valueOf(ss.getTable()).equals(TABLE_NAME)) {
510         t1SnapshotsCount++;
511       } else if (TableName.valueOf(ss.getTable()).equals(TABLE2_NAME)) {
512         t2SnapshotsCount++;
513       }
514     }
515     assertTrue("We expect at least 1 snapshot of table1 ", t1SnapshotsCount > 0);
516     assertTrue("We expect at least 1 snapshot of table2 ", t2SnapshotsCount > 0);
517 
518     UTIL.deleteTable(TABLE2_NAME);
519   }
520 
521   private void logFSTree(Path root) throws IOException {
522     FSUtils.logFileSystemState(UTIL.getDFSCluster().getFileSystem(), root, LOG);
523   }
524 
525   private void waitRegionsAfterMerge(final long numRegionsAfterMerge)
526       throws IOException, InterruptedException {
527     Admin admin = UTIL.getHBaseAdmin();
528     // Verify that there's one region less
529     long startTime = System.currentTimeMillis();
530     while (admin.getTableRegions(TABLE_NAME).size() != numRegionsAfterMerge) {
531       // This may be flaky... if after 15sec the merge is not complete give up
532       // it will fail in the assertEquals(numRegionsAfterMerge).
533       if ((System.currentTimeMillis() - startTime) > 15000)
534         break;
535       Thread.sleep(100);
536     }
537     SnapshotTestingUtils.waitForTableToBeOnline(UTIL, TABLE_NAME);
538   }
539 
540 
541   protected void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
542       long expectedRows) throws IOException {
543     SnapshotTestingUtils.verifyRowCount(util, tableName, expectedRows);
544   }
545 
546   protected int countRows(final Table table, final byte[]... families) throws IOException {
547     return UTIL.countRows(table, families);
548   }
549 }