View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.snapshot;
19  
20  import com.google.protobuf.InvalidProtocolBufferException;
21  import org.apache.commons.logging.Log;
22  import org.apache.commons.logging.LogFactory;
23  import org.apache.hadoop.conf.Configuration;
24  import org.apache.hadoop.fs.FSDataOutputStream;
25  import org.apache.hadoop.fs.FileSystem;
26  import org.apache.hadoop.fs.Path;
27  import org.apache.hadoop.hbase.HColumnDescriptor;
28  import org.apache.hadoop.hbase.HRegionInfo;
29  import org.apache.hadoop.hbase.TableName;
30  import org.apache.hadoop.hbase.HBaseTestingUtility;
31  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
32  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
33  import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
34  import org.apache.hadoop.hbase.testclassification.SmallTests;
35  import org.apache.hadoop.hbase.util.ByteStringer;
36  import org.apache.hadoop.hbase.util.Bytes;
37  import org.junit.After;
38  import org.junit.Before;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  
42  import java.io.IOException;
43  
44  import static org.junit.Assert.fail;
45  
46  @Category({SmallTests.class})
47  public class TestSnapshotManifest {
48    private final Log LOG = LogFactory.getLog(getClass());
49  
50    private static final String TABLE_NAME_STR = "testSnapshotManifest";
51    private static final TableName TABLE_NAME = TableName.valueOf(TABLE_NAME_STR);
52    private static final int TEST_NUM_REGIONS = 16000;
53  
54    private static HBaseTestingUtility TEST_UTIL;
55    private Configuration conf;
56    private FileSystem fs;
57    private Path rootDir;
58    private Path snapshotDir;
59    private SnapshotDescription snapshotDesc;
60  
61    @Before
62    public void setup() throws Exception {
63      TEST_UTIL = HBaseTestingUtility.createLocalHTU();
64  
65      rootDir = TEST_UTIL.getDataTestDir(TABLE_NAME_STR);
66      fs = TEST_UTIL.getTestFileSystem();
67      conf = TEST_UTIL.getConfiguration();
68  
69      SnapshotTestingUtils.SnapshotMock snapshotMock =
70        new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
71      SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder =
72        snapshotMock.createSnapshotV2("snapshot", TABLE_NAME_STR, 0);
73      snapshotDir = builder.commit();
74      snapshotDesc = builder.getSnapshotDescription();
75  
76      SnapshotDataManifest.Builder dataManifestBuilder =
77        SnapshotDataManifest.newBuilder();
78      byte[] startKey = null;
79      byte[] stopKey = null;
80      for (int i = 1; i <= TEST_NUM_REGIONS; i++) {
81        stopKey = Bytes.toBytes(String.format("%016d", i));
82        HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false);
83        SnapshotRegionManifest.Builder dataRegionManifestBuilder =
84          SnapshotRegionManifest.newBuilder();
85  
86        for (HColumnDescriptor hcd: builder.getTableDescriptor().getFamilies()) {
87          SnapshotRegionManifest.FamilyFiles.Builder family =
88              SnapshotRegionManifest.FamilyFiles.newBuilder();
89          family.setFamilyName(ByteStringer.wrap(hcd.getName()));
90          for (int j = 0; j < 100; ++j) {
91            SnapshotRegionManifest.StoreFile.Builder sfManifest =
92              SnapshotRegionManifest.StoreFile.newBuilder();
93            sfManifest.setName(String.format("%032d", i));
94            sfManifest.setFileSize((1 + i) * (1 + i) * 1024);
95            family.addStoreFiles(sfManifest.build());
96          }
97          dataRegionManifestBuilder.addFamilyFiles(family.build());
98        }
99  
100       dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo));
101       dataManifestBuilder.addRegionManifests(dataRegionManifestBuilder.build());
102 
103       startKey = stopKey;
104     }
105 
106     dataManifestBuilder.setTableSchema(builder.getTableDescriptor().convert());
107 
108     SnapshotDataManifest dataManifest = dataManifestBuilder.build();
109     writeDataManifest(dataManifest);
110   }
111 
112   @After
113   public void tearDown() throws Exception {
114     fs.delete(rootDir,true);
115   }
116 
117   @Test
118   public void testReadSnapshotManifest() throws IOException {
119     try {
120       SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
121       fail("fail to test snapshot manifest because message size is too small.");
122     } catch (CorruptedSnapshotException cse) {
123       try {
124         conf.setInt(SnapshotManifest.SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 128 * 1024 * 1024);
125         SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
126         LOG.info("open snapshot manifest succeed.");
127       } catch (CorruptedSnapshotException cse2) {
128         fail("fail to take snapshot because Manifest proto-message too large.");
129       }
130     }
131   }
132 
133   private void writeDataManifest(final SnapshotDataManifest manifest)
134       throws IOException {
135     FSDataOutputStream stream = fs.create(new Path(snapshotDir, SnapshotManifest.DATA_MANIFEST_NAME));
136     try {
137       manifest.writeTo(stream);
138     } finally {
139       stream.close();
140     }
141   }
142 }