1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import com.google.protobuf.InvalidProtocolBufferException;
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.fs.FSDataOutputStream;
25 import org.apache.hadoop.fs.FileSystem;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.hbase.HColumnDescriptor;
28 import org.apache.hadoop.hbase.HRegionInfo;
29 import org.apache.hadoop.hbase.TableName;
30 import org.apache.hadoop.hbase.HBaseTestingUtility;
31 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
32 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
33 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
34 import org.apache.hadoop.hbase.testclassification.SmallTests;
35 import org.apache.hadoop.hbase.util.ByteStringer;
36 import org.apache.hadoop.hbase.util.Bytes;
37 import org.junit.After;
38 import org.junit.Before;
39 import org.junit.Test;
40 import org.junit.experimental.categories.Category;
41
42 import java.io.IOException;
43
44 import static org.junit.Assert.fail;
45
46 @Category({SmallTests.class})
47 public class TestSnapshotManifest {
48 private final Log LOG = LogFactory.getLog(getClass());
49
50 private static final String TABLE_NAME_STR = "testSnapshotManifest";
51 private static final TableName TABLE_NAME = TableName.valueOf(TABLE_NAME_STR);
52 private static final int TEST_NUM_REGIONS = 16000;
53
54 private static HBaseTestingUtility TEST_UTIL;
55 private Configuration conf;
56 private FileSystem fs;
57 private Path rootDir;
58 private Path snapshotDir;
59 private SnapshotDescription snapshotDesc;
60
61 @Before
62 public void setup() throws Exception {
63 TEST_UTIL = HBaseTestingUtility.createLocalHTU();
64
65 rootDir = TEST_UTIL.getDataTestDir(TABLE_NAME_STR);
66 fs = TEST_UTIL.getTestFileSystem();
67 conf = TEST_UTIL.getConfiguration();
68
69 SnapshotTestingUtils.SnapshotMock snapshotMock =
70 new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
71 SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder =
72 snapshotMock.createSnapshotV2("snapshot", TABLE_NAME_STR, 0);
73 snapshotDir = builder.commit();
74 snapshotDesc = builder.getSnapshotDescription();
75
76 SnapshotDataManifest.Builder dataManifestBuilder =
77 SnapshotDataManifest.newBuilder();
78 byte[] startKey = null;
79 byte[] stopKey = null;
80 for (int i = 1; i <= TEST_NUM_REGIONS; i++) {
81 stopKey = Bytes.toBytes(String.format("%016d", i));
82 HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false);
83 SnapshotRegionManifest.Builder dataRegionManifestBuilder =
84 SnapshotRegionManifest.newBuilder();
85
86 for (HColumnDescriptor hcd: builder.getTableDescriptor().getFamilies()) {
87 SnapshotRegionManifest.FamilyFiles.Builder family =
88 SnapshotRegionManifest.FamilyFiles.newBuilder();
89 family.setFamilyName(ByteStringer.wrap(hcd.getName()));
90 for (int j = 0; j < 100; ++j) {
91 SnapshotRegionManifest.StoreFile.Builder sfManifest =
92 SnapshotRegionManifest.StoreFile.newBuilder();
93 sfManifest.setName(String.format("%032d", i));
94 sfManifest.setFileSize((1 + i) * (1 + i) * 1024);
95 family.addStoreFiles(sfManifest.build());
96 }
97 dataRegionManifestBuilder.addFamilyFiles(family.build());
98 }
99
100 dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo));
101 dataManifestBuilder.addRegionManifests(dataRegionManifestBuilder.build());
102
103 startKey = stopKey;
104 }
105
106 dataManifestBuilder.setTableSchema(builder.getTableDescriptor().convert());
107
108 SnapshotDataManifest dataManifest = dataManifestBuilder.build();
109 writeDataManifest(dataManifest);
110 }
111
112 @After
113 public void tearDown() throws Exception {
114 fs.delete(rootDir,true);
115 }
116
117 @Test
118 public void testReadSnapshotManifest() throws IOException {
119 try {
120 SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
121 fail("fail to test snapshot manifest because message size is too small.");
122 } catch (CorruptedSnapshotException cse) {
123 try {
124 conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
125 SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
126 LOG.info("open snapshot manifest succeed.");
127 } catch (CorruptedSnapshotException cse2) {
128 fail("fail to take snapshot because Manifest proto-message too large.");
129 }
130 }
131 }
132
133 private void writeDataManifest(final SnapshotDataManifest manifest)
134 throws IOException {
135 FSDataOutputStream stream = fs.create(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME));
136 try {
137 manifest.writeTo(stream);
138 } finally {
139 stream.close();
140 }
141 }
142 }