View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import static org.junit.Assert.assertEquals;
21  import static org.junit.Assert.assertFalse;
22  import static org.junit.Assert.assertTrue;
23  
24  import java.io.IOException;
25  import java.util.Arrays;
26  import java.util.ArrayList;
27  import java.util.HashSet;
28  import java.util.List;
29  import java.util.Map;
30  import java.util.Set;
31  import java.util.TreeSet;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.fs.FSDataOutputStream;
37  import org.apache.hadoop.fs.FileSystem;
38  import org.apache.hadoop.fs.FileStatus;
39  import org.apache.hadoop.fs.FSDataInputStream;
40  import org.apache.hadoop.fs.Path;
41  import org.apache.hadoop.fs.PathFilter;
42  import org.apache.hadoop.hbase.HBaseTestingUtility;
43  import org.apache.hadoop.hbase.HColumnDescriptor;
44  import org.apache.hadoop.hbase.HConstants;
45  import org.apache.hadoop.hbase.HRegionInfo;
46  import org.apache.hadoop.hbase.HTableDescriptor;
47  import org.apache.hadoop.hbase.TableName;
48  import org.apache.hadoop.hbase.TableNotEnabledException;
49  import org.apache.hadoop.hbase.Waiter;
50  import org.apache.hadoop.hbase.client.Admin;
51  import org.apache.hadoop.hbase.client.BufferedMutator;
52  import org.apache.hadoop.hbase.client.Durability;
53  import org.apache.hadoop.hbase.client.HBaseAdmin;
54  import org.apache.hadoop.hbase.client.HTable;
55  import org.apache.hadoop.hbase.client.Put;
56  import org.apache.hadoop.hbase.client.Table;
57  import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
58  import org.apache.hadoop.hbase.client.RegionReplicaUtil;
59  import org.apache.hadoop.hbase.io.HFileLink;
60  import org.apache.hadoop.hbase.master.HMaster;
61  import org.apache.hadoop.hbase.master.MasterFileSystem;
62  import org.apache.hadoop.hbase.mob.MobUtils;
63  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
64  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
65  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
66  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
67  import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
68  import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
69  import org.apache.hadoop.hbase.regionserver.HRegionServer;
70  import org.apache.hadoop.hbase.regionserver.Region;
71  import org.apache.hadoop.hbase.util.Bytes;
72  import org.apache.hadoop.hbase.util.FSTableDescriptors;
73  import org.apache.hadoop.hbase.util.FSVisitor;
74  import org.apache.hadoop.hbase.util.FSUtils;
75  import org.apache.hadoop.hbase.util.MD5Hash;
76  import org.junit.Assert;
77  
78  import com.google.protobuf.ServiceException;
79  
80  /**
81   * Utilities class for snapshots
82   */
83  public class SnapshotTestingUtils {
84  
85    private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
86    private static byte[] KEYS = Bytes.toBytes("0123456789");
87  
88    /**
89     * Assert that we don't have any snapshots lists
90     *
91     * @throws IOException
92     *           if the admin operation fails
93     */
94    public static void assertNoSnapshots(Admin admin) throws IOException {
95      assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
96          .size());
97    }
98  
99    /**
100    * Make sure that there is only one snapshot returned from the master and its
101    * name and table match the passed in parameters.
102    */
103   public static List<SnapshotDescription> assertExistsMatchingSnapshot(
104       Admin admin, String snapshotName, TableName tableName)
105       throws IOException {
106     // list the snapshot
107     List<SnapshotDescription> snapshots = admin.listSnapshots();
108 
109     List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
110     for (SnapshotDescription sd : snapshots) {
111       if (snapshotName.equals(sd.getName()) &&
112           tableName.equals(TableName.valueOf(sd.getTable()))) {
113         returnedSnapshots.add(sd);
114       }
115     }
116 
117     Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
118     return returnedSnapshots;
119   }
120 
121   /**
122    * Make sure that there is only one snapshot returned from the master
123    */
124   public static void assertOneSnapshotThatMatches(Admin admin,
125       SnapshotDescription snapshot) throws IOException {
126     assertOneSnapshotThatMatches(admin, snapshot.getName(),
127         TableName.valueOf(snapshot.getTable()));
128   }
129 
130   /**
131    * Make sure that there is only one snapshot returned from the master and its
132    * name and table match the passed in parameters.
133    */
134   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
135       Admin admin, String snapshotName, TableName tableName)
136       throws IOException {
137     // list the snapshot
138     List<SnapshotDescription> snapshots = admin.listSnapshots();
139 
140     assertEquals("Should only have 1 snapshot", 1, snapshots.size());
141     assertEquals(snapshotName, snapshots.get(0).getName());
142     assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
143 
144     return snapshots;
145   }
146 
147   /**
148    * Make sure that there is only one snapshot returned from the master and its
149    * name and table match the passed in parameters.
150    */
151   public static List<SnapshotDescription> assertOneSnapshotThatMatches(
152       Admin admin, byte[] snapshot, TableName tableName) throws IOException {
153     return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
154         tableName);
155   }
156 
157   /**
158    * Confirm that the snapshot contains references to all the files that should
159    * be in the snapshot.
160    */
161   public static void confirmSnapshotValid(
162       SnapshotDescription snapshotDescriptor, TableName tableName,
163       byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
164       throws IOException {
165     ArrayList nonEmptyTestFamilies = new ArrayList(1);
166     nonEmptyTestFamilies.add(testFamily);
167     confirmSnapshotValid(snapshotDescriptor, tableName,
168       nonEmptyTestFamilies, null, rootDir, admin, fs);
169   }
170 
171   /**
172    * Confirm that the snapshot has no references files but only metadata.
173    */
174   public static void confirmEmptySnapshotValid(
175       SnapshotDescription snapshotDescriptor, TableName tableName,
176       byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
177       throws IOException {
178     ArrayList emptyTestFamilies = new ArrayList(1);
179     emptyTestFamilies.add(testFamily);
180     confirmSnapshotValid(snapshotDescriptor, tableName,
181       null, emptyTestFamilies, rootDir, admin, fs);
182   }
183 
184   /**
185    * Confirm that the snapshot contains references to all the files that should
186    * be in the snapshot. This method also perform some redundant check like
187    * the existence of the snapshotinfo or the regioninfo which are done always
188    * by the MasterSnapshotVerifier, at the end of the snapshot operation.
189    */
190   public static void confirmSnapshotValid(
191       SnapshotDescription snapshotDescriptor, TableName tableName,
192       List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
193       Path rootDir, Admin admin, FileSystem fs) throws IOException {
194     final Configuration conf = admin.getConfiguration();
195 
196     // check snapshot dir
197     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
198         snapshotDescriptor, rootDir);
199     assertTrue(fs.exists(snapshotDir));
200 
201     SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
202 
203     // Extract regions and families with store files
204     final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
205 
206     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
207     Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
208     for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
209       SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
210           new SnapshotReferenceUtil.StoreFileVisitor() {
211         @Override
212         public void storeFile(final HRegionInfo regionInfo, final String family,
213               final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
214           snapshotFamilies.add(Bytes.toBytes(family));
215         }
216       });
217     }
218 
219     // Verify that there are store files in the specified families
220     if (nonEmptyTestFamilies != null) {
221       for (final byte[] familyName: nonEmptyTestFamilies) {
222         assertTrue(snapshotFamilies.contains(familyName));
223       }
224     }
225 
226     // Verify that there are no store files in the specified families
227     if (emptyTestFamilies != null) {
228       for (final byte[] familyName: emptyTestFamilies) {
229         assertFalse(snapshotFamilies.contains(familyName));
230       }
231     }
232 
233     // check the region snapshot for all the regions
234     List<HRegionInfo> regions = admin.getTableRegions(tableName);
235     // remove the non-default regions
236     RegionReplicaUtil.removeNonDefaultRegions(regions);
237     boolean hasMob = regionManifests.containsKey(MobUtils.getMobRegionInfo(tableName)
238         .getEncodedName());
239     if (hasMob) {
240       assertEquals(regions.size(), regionManifests.size() - 1);
241     } else {
242       assertEquals(regions.size(), regionManifests.size());
243     }
244 
245     // Verify Regions (redundant check, see MasterSnapshotVerifier)
246     for (HRegionInfo info : regions) {
247       String regionName = info.getEncodedName();
248       assertTrue(regionManifests.containsKey(regionName));
249     }
250   }
251 
252   /**
253    * Helper method for testing async snapshot operations. Just waits for the
254    * given snapshot to complete on the server by repeatedly checking the master.
255    *
256    * @param master: the master running the snapshot
257    * @param snapshot: the snapshot to check
258    * @param sleep: amount to sleep between checks to see if the snapshot is done
259    * @throws ServiceException if the snapshot fails
260    */
261   public static void waitForSnapshotToComplete(HMaster master,
262       SnapshotDescription snapshot, long sleep) throws ServiceException {
263     final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
264         .setSnapshot(snapshot).build();
265     IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
266         .buildPartial();
267     while (!done.getDone()) {
268       done = master.getMasterRpcServices().isSnapshotDone(null, request);
269       try {
270         Thread.sleep(sleep);
271       } catch (InterruptedException e) {
272         throw new ServiceException(e);
273       }
274     }
275   }
276 
277   /*
278    * Take snapshot with maximum of numTries attempts, ignoring CorruptedSnapshotException
279    * except for the last CorruptedSnapshotException
280    */
281   public static void snapshot(Admin admin,
282       final String snapshotName, final String tableName,
283       SnapshotDescription.Type type, int numTries) throws IOException {
284     int tries = 0;
285     CorruptedSnapshotException lastEx = null;
286     while (tries++ < numTries) {
287       try {
288         admin.snapshot(snapshotName, TableName.valueOf(tableName), type);
289         return;
290       } catch (CorruptedSnapshotException cse) {
291         LOG.warn("Got CorruptedSnapshotException", cse);
292         lastEx = cse;
293       }
294     }
295     throw lastEx;
296   }
297 
298   public static void cleanupSnapshot(Admin admin, byte[] tableName)
299       throws IOException {
300     SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
301   }
302 
303   public static void cleanupSnapshot(Admin admin, String snapshotName)
304       throws IOException {
305     // delete the taken snapshot
306     admin.deleteSnapshot(snapshotName);
307     assertNoSnapshots(admin);
308   }
309 
310   /**
311    * Expect the snapshot to throw an error when checking if the snapshot is
312    * complete
313    *
314    * @param master master to check
315    * @param snapshot the {@link SnapshotDescription} request to pass to the master
316    * @param clazz expected exception from the master
317    */
318   public static void expectSnapshotDoneException(HMaster master,
319       IsSnapshotDoneRequest snapshot,
320       Class<? extends HBaseSnapshotException> clazz) {
321     try {
322       master.getMasterRpcServices().isSnapshotDone(null, snapshot);
323       Assert.fail("didn't fail to lookup a snapshot");
324     } catch (ServiceException se) {
325       try {
326         throw ProtobufUtil.getRemoteException(se);
327       } catch (HBaseSnapshotException e) {
328         assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
329       } catch (Throwable t) {
330         Assert.fail("Threw an unexpected exception:" + t);
331       }
332     }
333   }
334 
335   /**
336    * List all the HFiles in the given table
337    *
338    * @param fs: FileSystem where the table lives
339    * @param tableDir directory of the table
340    * @return array of the current HFiles in the table (could be a zero-length array)
341    * @throws IOException on unexecpted error reading the FS
342    */
343   public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
344       throws IOException {
345     final ArrayList<Path> hfiles = new ArrayList<Path>();
346     FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
347       @Override
348       public void storeFile(final String region, final String family, final String hfileName)
349           throws IOException {
350         hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
351       }
352     });
353     return hfiles.toArray(new Path[hfiles.size()]);
354   }
355 
356   public static String[] listHFileNames(final FileSystem fs, final Path tableDir)
357       throws IOException {
358     Path[] files = listHFiles(fs, tableDir);
359     String[] names = new String[files.length];
360     for (int i = 0; i < files.length; ++i) {
361       names[i] = files[i].getName();
362     }
363     Arrays.sort(names);
364     return names;
365   }
366 
367   /**
368    * Take a snapshot of the specified table and verify that the given family is
369    * not empty. Note that this will leave the table disabled
370    * in the case of an offline snapshot.
371    */
372   public static void createSnapshotAndValidate(Admin admin,
373       TableName tableName, String familyName, String snapshotNameString,
374       Path rootDir, FileSystem fs, boolean onlineSnapshot)
375       throws Exception {
376     ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
377     nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
378     createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames, /* emptyFamilyNames= */ null,
379                               snapshotNameString, rootDir, fs, onlineSnapshot);
380   }
381 
382   /**
383    * Take a snapshot of the specified table and verify the given families.
384    * Note that this will leave the table disabled in the case of an offline snapshot.
385    */
386   public static void createSnapshotAndValidate(Admin admin,
387       TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
388       String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
389         throws Exception {
390     if (!onlineSnapshot) {
391       try {
392         admin.disableTable(tableName);
393       } catch (TableNotEnabledException tne) {
394         LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
395             "already disabled.");
396       }
397     }
398     admin.snapshot(snapshotNameString, tableName);
399 
400     List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
401       snapshotNameString, tableName);
402     if (snapshots == null || snapshots.size() != 1) {
403       Assert.fail("Incorrect number of snapshots for table " + tableName);
404     }
405 
406     SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
407       emptyFamilyNames, rootDir, admin, fs);
408   }
409 
410   /**
411    * Corrupt the specified snapshot by deleting some files.
412    *
413    * @param util {@link HBaseTestingUtility}
414    * @param snapshotName name of the snapshot to corrupt
415    * @return array of the corrupted HFiles
416    * @throws IOException on unexecpted error reading the FS
417    */
418   public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
419       throws IOException {
420     final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
421     final FileSystem fs = mfs.getFileSystem();
422 
423     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
424                                                                         mfs.getRootDir());
425     SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
426     final TableName table = TableName.valueOf(snapshotDesc.getTable());
427 
428     final ArrayList corruptedFiles = new ArrayList();
429     final Configuration conf = util.getConfiguration();
430     SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
431         new SnapshotReferenceUtil.StoreFileVisitor() {
432       @Override
433       public void storeFile(final HRegionInfo regionInfo, final String family,
434             final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
435         String region = regionInfo.getEncodedName();
436         String hfile = storeFile.getName();
437         HFileLink link = HFileLink.build(conf, table, region, family, hfile);
438         if (corruptedFiles.size() % 2 == 0) {
439           fs.delete(link.getAvailablePath(fs), true);
440           corruptedFiles.add(hfile);
441         }
442       }
443     });
444 
445     assertTrue(corruptedFiles.size() > 0);
446     return corruptedFiles;
447   }
448 
449   // ==========================================================================
450   //  Snapshot Mock
451   // ==========================================================================
452   public static class SnapshotMock {
453     protected final static String TEST_FAMILY = "cf";
454     public final static int TEST_NUM_REGIONS = 4;
455 
456     private final Configuration conf;
457     private final FileSystem fs;
458     private final Path rootDir;
459 
460     static class RegionData {
461       public HRegionInfo hri;
462       public Path tableDir;
463       public Path[] files;
464 
465       public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
466         this.tableDir = tableDir;
467         this.hri = hri;
468         this.files = new Path[nfiles];
469       }
470     }
471 
472     public static class SnapshotBuilder {
473       private final RegionData[] tableRegions;
474       private final SnapshotDescription desc;
475       private final HTableDescriptor htd;
476       private final Configuration conf;
477       private final FileSystem fs;
478       private final Path rootDir;
479       private Path snapshotDir;
480       private int snapshotted = 0;
481 
482       public SnapshotBuilder(final Configuration conf, final FileSystem fs,
483           final Path rootDir, final HTableDescriptor htd,
484           final SnapshotDescription desc, final RegionData[] tableRegions)
485           throws IOException {
486         this.fs = fs;
487         this.conf = conf;
488         this.rootDir = rootDir;
489         this.htd = htd;
490         this.desc = desc;
491         this.tableRegions = tableRegions;
492         this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
493         new FSTableDescriptors(conf)
494           .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
495       }
496 
497       public HTableDescriptor getTableDescriptor() {
498         return this.htd;
499       }
500 
501       public SnapshotDescription getSnapshotDescription() {
502         return this.desc;
503       }
504 
505       public Path getSnapshotsDir() {
506         return this.snapshotDir;
507       }
508 
509       public Path[] addRegion() throws IOException {
510         return addRegion(desc);
511       }
512 
513       public Path[] addRegionV1() throws IOException {
514         return addRegion(desc.toBuilder()
515                           .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
516                           .build());
517       }
518 
519       public Path[] addRegionV2() throws IOException {
520         return addRegion(desc.toBuilder()
521                           .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
522                           .build());
523       }
524 
525       private Path[] addRegion(final SnapshotDescription desc) throws IOException {
526         if (this.snapshotted == tableRegions.length) {
527           throw new UnsupportedOperationException("No more regions in the table");
528         }
529 
530         RegionData regionData = tableRegions[this.snapshotted++];
531         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
532         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
533         manifest.addRegion(regionData.tableDir, regionData.hri);
534         return regionData.files;
535       }
536 
537       private void corruptFile(Path p) throws IOException {
538         String manifestName = p.getName();
539 
540         // Rename the original region-manifest file
541         Path newP = new Path(p.getParent(), manifestName + "1");
542         fs.rename(p, newP);
543 
544         // Create a new region-manifest file
545         FSDataOutputStream out = fs.create(p);
546 
547         //Copy the first 25 bytes of the original region-manifest into the new one,
548         //make it a corrupted region-manifest file.
549         FSDataInputStream input = fs.open(newP);
550         byte[] buffer = new byte[25];
551         int len = input.read(0, buffer, 0, 25);
552         if (len > 1) {
553           out.write(buffer, 0, len - 1);
554         }
555         out.close();
556 
557         // Delete the original region-manifest
558         fs.delete(newP);
559       }
560 
561       /**
562        * Corrupt one region-manifest file
563        *
564        * @throws IOException on unexecpted error from the FS
565        */
566       public void corruptOneRegionManifest() throws IOException {
567         FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
568           @Override public boolean accept(Path path) {
569             return path.getName().startsWith(SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX);
570           }
571         });
572 
573         if (manifestFiles.length == 0) return;
574 
575         // Just choose the first one
576         Path p = manifestFiles[0].getPath();
577         corruptFile(p);
578       }
579 
580       /**
581        * Corrupt data-manifest file
582        *
583        * @throws IOException on unexecpted error from the FS
584        */
585       public void corruptDataManifest() throws IOException {
586         FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
587           @Override
588           public boolean accept(Path path) {
589             return path.getName().startsWith(SnapshotManifest.DATA_MANIFEST_NAME);
590           }
591         });
592 
593         if (manifestFiles.length == 0) return;
594 
595         // Just choose the first one
596         Path p = manifestFiles[0].getPath();
597         corruptFile(p);
598       }
599 
600       public Path commit() throws IOException {
601         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
602         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
603         manifest.addTableDescriptor(htd);
604         manifest.consolidate();
605         SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
606         snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
607         return snapshotDir;
608       }
609 
610       public void consolidate() throws IOException {
611         ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
612         SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
613         manifest.addTableDescriptor(htd);
614         manifest.consolidate();
615       }
616     }
617 
618     public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
619       this.fs = fs;
620       this.conf = conf;
621       this.rootDir = rootDir;
622     }
623 
624     public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName)
625         throws IOException {
626       return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION);
627     }
628 
629     public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName,
630         final int numRegions) throws IOException {
631       return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION);
632     }
633 
634     public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName)
635         throws IOException {
636       return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION);
637     }
638 
639     public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
640         final int numRegions) throws IOException {
641       return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION);
642     }
643 
644     private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
645         final int version) throws IOException {
646       return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version);
647     }
648 
649     private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
650         final int numRegions, final int version) throws IOException {
651       HTableDescriptor htd = createHtd(tableName);
652       RegionData[] regions = createTable(htd, numRegions);
653 
654       SnapshotDescription desc = SnapshotDescription.newBuilder()
655         .setTable(htd.getNameAsString())
656         .setName(snapshotName)
657         .setVersion(version)
658         .build();
659 
660       Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
661       SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
662       return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
663     }
664 
665     public HTableDescriptor createHtd(final String tableName) {
666       HTableDescriptor htd = new HTableDescriptor(tableName);
667       htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
668       return htd;
669     }
670 
671     private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
672         throws IOException {
673       Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
674       new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
675 
676       assertTrue(nregions % 2 == 0);
677       RegionData[] regions = new RegionData[nregions];
678       for (int i = 0; i < regions.length; i += 2) {
679         byte[] startKey = Bytes.toBytes(0 + i * 2);
680         byte[] endKey = Bytes.toBytes(1 + i * 2);
681 
682         // First region, simple with one plain hfile.
683         HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
684         HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
685         regions[i] = new RegionData(tableDir, hri, 3);
686         for (int j = 0; j < regions[i].files.length; ++j) {
687           Path storeFile = createStoreFile(rfs.createTempName());
688           regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
689         }
690 
691         // Second region, used to test the split case.
692         // This region contains a reference to the hfile in the first region.
693         startKey = Bytes.toBytes(2 + i * 2);
694         endKey = Bytes.toBytes(3 + i * 2);
695         hri = new HRegionInfo(htd.getTableName());
696         rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
697         regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
698         for (int j = 0; j < regions[i].files.length; ++j) {
699           String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
700           Path refFile = createStoreFile(new Path(rootDir, refName));
701           regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
702         }
703       }
704       return regions;
705     }
706 
707     private Path createStoreFile(final Path storeFile)
708         throws IOException {
709       FSDataOutputStream out = fs.create(storeFile);
710       try {
711         out.write(Bytes.toBytes(storeFile.toString()));
712       } finally {
713         out.close();
714       }
715       return storeFile;
716     }
717   }
718 
719   // ==========================================================================
720   //  Table Helpers
721   // ==========================================================================
722   public static void waitForTableToBeOnline(final HBaseTestingUtility util,
723                                             final TableName tableName)
724       throws IOException, InterruptedException {
725     HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
726     List<Region> onlineRegions = rs.getOnlineRegions(tableName);
727     for (Region region : onlineRegions) {
728       region.waitForFlushesAndCompactions();
729     }
730     // Wait up to 60 seconds for a table to be available.
731     util.waitFor(60000, util.predicateTableAvailable(tableName));
732   }
733 
734   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
735       int regionReplication, final byte[]... families) throws IOException, InterruptedException {
736     HTableDescriptor htd = new HTableDescriptor(tableName);
737     htd.setRegionReplication(regionReplication);
738     for (byte[] family : families) {
739       HColumnDescriptor hcd = new HColumnDescriptor(family);
740       htd.addFamily(hcd);
741     }
742     byte[][] splitKeys = getSplitKeys();
743     util.createTable(htd, splitKeys);
744     assertEquals((splitKeys.length + 1) * regionReplication,
745         util.getHBaseAdmin().getTableRegions(tableName).size());
746   }
747 
748   public static byte[][] getSplitKeys() {
749     byte[][] splitKeys = new byte[KEYS.length-2][];
750     for (int i = 0; i < splitKeys.length; ++i) {
751       splitKeys[i] = new byte[] { KEYS[i+1] };
752     }
753     return splitKeys;
754   }
755 
756   public static void createTable(final HBaseTestingUtility util, final TableName tableName,
757       final byte[]... families) throws IOException, InterruptedException {
758     createTable(util, tableName, 1, families);
759   }
760 
761   public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
762       byte[]... families) throws IOException, InterruptedException {
763     try (BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName)) {
764       loadData(util, mutator, rows, families);
765     }
766   }
767 
768   public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows,
769       byte[]... families) throws IOException, InterruptedException {
770     // Ensure one row per region
771     assertTrue(rows >= KEYS.length);
772     for (byte k0: KEYS) {
773       byte[] k = new byte[] { k0 };
774       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
775       byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
776       final byte[][] families1 = families;
777       final byte[] key1 = key;
778       final byte[] value1 = value;
779       mutator.mutate(createPut(families1, key1, value1));
780       rows--;
781     }
782 
783     // Add other extra rows. more rows, more files
784     while (rows-- > 0) {
785       byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
786       byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
787       final byte[][] families1 = families;
788       final byte[] key1 = key;
789       final byte[] value1 = value;
790       mutator.mutate(createPut(families1, key1, value1));
791     }
792     mutator.flush();
793 
794     waitForTableToBeOnline(util, mutator.getName());
795   }
796 
797   private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
798     byte[] q = Bytes.toBytes("q");
799     Put put = new Put(key);
800     put.setDurability(Durability.SKIP_WAL);
801     for (byte[] family: families) {
802       put.add(family, q, value);
803     }
804     return put;
805   }
806 
807   public static void deleteAllSnapshots(final Admin admin)
808       throws IOException {
809     // Delete all the snapshots
810     for (SnapshotDescription snapshot: admin.listSnapshots()) {
811       admin.deleteSnapshot(snapshot.getName());
812     }
813     SnapshotTestingUtils.assertNoSnapshots(admin);
814   }
815 
816   public static void deleteArchiveDirectory(final HBaseTestingUtility util)
817       throws IOException {
818     // Ensure the archiver to be empty
819     MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
820     Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
821     mfs.getFileSystem().delete(archiveDir, true);
822   }
823 
824   public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
825       long expectedRows) throws IOException {
826     Table table = new HTable(util.getConfiguration(), tableName);
827     try {
828       assertEquals(expectedRows, util.countRows(table));
829     } finally {
830       table.close();
831     }
832   }
833 
834   public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
835       int regionReplication) throws IOException {
836     List<HRegionInfo> regions = admin.getTableRegions(tableName);
837     HashSet<HRegionInfo> set = new HashSet<HRegionInfo>();
838     for (HRegionInfo hri : regions) {
839       set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
840       for (int i = 0; i < regionReplication; i++) {
841         HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
842         if (!regions.contains(replica)) {
843           Assert.fail(replica + " is not contained in the list of online regions");
844         }
845       }
846     }
847     assert(set.size() == getSplitKeys().length + 1);
848   }
849 }