1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.migration;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22
23 import java.io.File;
24 import java.io.IOException;
25
26 import org.apache.commons.logging.Log;
27 import org.apache.commons.logging.LogFactory;
28 import org.apache.hadoop.fs.FileStatus;
29 import org.apache.hadoop.fs.FileSystem;
30 import org.apache.hadoop.fs.FsShell;
31 import org.apache.hadoop.fs.Path;
32 import org.apache.hadoop.hbase.HBaseTestingUtility;
33 import org.apache.hadoop.hbase.testclassification.MediumTests;
34 import org.apache.hadoop.hbase.io.FileLink;
35 import org.apache.hadoop.hbase.io.HFileLink;
36 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
37 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
38 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.ReplicationPeer;
39 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.Table.State;
40 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
41 import org.apache.hadoop.hbase.util.Bytes;
42 import org.apache.hadoop.hbase.util.FSUtils;
43 import org.apache.hadoop.hbase.util.HFileV1Detector;
44 import org.apache.hadoop.hbase.zookeeper.ZKUtil;
45 import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
46 import org.apache.hadoop.util.ToolRunner;
47 import org.apache.zookeeper.KeeperException;
48 import org.junit.AfterClass;
49 import org.junit.BeforeClass;
50 import org.junit.Test;
51 import org.junit.experimental.categories.Category;
52
53 import com.google.protobuf.InvalidProtocolBufferException;
54
55
56
57
58
59
60 @Category(MediumTests.class)
61 public class TestUpgradeTo96 {
62
63 static final Log LOG = LogFactory.getLog(TestUpgradeTo96.class);
64 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
65
66
67
68
69 private static FileSystem fs;
70
71
72
73 private static Path hbaseRootDir;
74 private static ZooKeeperWatcher zkw;
75
76
77
78 private static String replicationPeerZnode;
79
80
81
82 private static String tableAZnode;
83 private static ReplicationPeer peer1;
84
85
86
87 private static String peer1Znode;
88
89 @BeforeClass
90 public static void setUpBeforeClass() throws Exception {
91
92
93 TEST_UTIL.startMiniZKCluster();
94 TEST_UTIL.startMiniDFSCluster(1);
95
96 hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
97 fs = FileSystem.get(TEST_UTIL.getConfiguration());
98 FSUtils.setRootDir(TEST_UTIL.getConfiguration(), hbaseRootDir);
99 zkw = TEST_UTIL.getZooKeeperWatcher();
100
101 Path testdir = TEST_UTIL.getDataTestDir("TestUpgradeTo96");
102
103
104 set94FSLayout(testdir);
105 setUp94Znodes();
106 }
107
108
109
110
111
112
113
114 private static void set94FSLayout(Path testdir) throws IOException, Exception {
115 File untar = TestNamespaceUpgrade.untar(new File(testdir.toString()));
116 if (!fs.exists(hbaseRootDir.getParent())) {
117
118 fs.mkdirs(hbaseRootDir.getParent());
119 }
120 FsShell shell = new FsShell(TEST_UTIL.getConfiguration());
121 shell.run(new String[] { "-put", untar.toURI().toString(), hbaseRootDir.toString() });
122
123 shell.run(new String[] { "-lsr", "/" });
124 }
125
126
127
128
129
130
131 private static void setUp94Znodes() throws IOException, KeeperException {
132
133 String rootRegionServerZnode = ZKUtil.joinZNode(zkw.baseZNode, "root-region-server");
134 ZKUtil.createWithParents(zkw, rootRegionServerZnode);
135 ZKUtil.createWithParents(zkw, zkw.backupMasterAddressesZNode);
136
137 tableAZnode = ZKUtil.joinZNode(zkw.tableZNode, "a");
138 ZKUtil.createWithParents(zkw, tableAZnode,
139 Bytes.toBytes(ZooKeeperProtos.Table.State.ENABLED.toString()));
140
141 String replicationZnode = ZKUtil.joinZNode(zkw.baseZNode, "replication");
142 replicationPeerZnode = ZKUtil.joinZNode(replicationZnode, "peers");
143 peer1Znode = ZKUtil.joinZNode(replicationPeerZnode, "1");
144 peer1 = ReplicationPeer.newBuilder().setClusterkey("abc:123:/hbase").build();
145 ZKUtil.createWithParents(zkw, peer1Znode, Bytes.toBytes(peer1.getClusterkey()));
146 }
147
148
149
150
151
152 @Test
153 public void testHFileV1Detector() throws Exception {
154 assertEquals(0, ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), null));
155 }
156
157
158
159
160
161 @Test
162 public void testHFileV1DetectorWithCorruptFiles() throws Exception {
163
164 Path tablePath = new Path(hbaseRootDir, "foo");
165 FileStatus[] regionsDir = fs.listStatus(tablePath);
166 if (regionsDir == null) throw new IOException("No Regions found for table " + "foo");
167 Path columnFamilyDir = null;
168 Path targetRegion = null;
169 for (FileStatus s : regionsDir) {
170 if (fs.exists(new Path(s.getPath(), HRegionFileSystem.REGION_INFO_FILE))) {
171 targetRegion = s.getPath();
172 break;
173 }
174 }
175 FileStatus[] cfs = fs.listStatus(targetRegion);
176 for (FileStatus f : cfs) {
177 if (f.isDirectory()) {
178 columnFamilyDir = f.getPath();
179 break;
180 }
181 }
182 LOG.debug("target columnFamilyDir: " + columnFamilyDir);
183
184 Path corruptFile = new Path(columnFamilyDir, "corrupt_file");
185 if (!fs.createNewFile(corruptFile)) throw new IOException("Couldn't create corrupt file: "
186 + corruptFile);
187 assertEquals(1, ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), null));
188
189 FileSystem.get(TEST_UTIL.getConfiguration()).delete(corruptFile, false);
190 }
191
192 @Test
193 public void testHFileLink() throws Exception {
194
195 Path rootDir = FSUtils.getRootDir(TEST_UTIL.getConfiguration());
196 Path aFileLink = new Path(rootDir, "table/2086db948c48/cf/table=21212abcdc33-0906db948c48");
197 Path preNamespaceTablePath = new Path(rootDir, "table/21212abcdc33/cf/0906db948c48");
198 Path preNamespaceArchivePath =
199 new Path(rootDir, ".archive/table/21212abcdc33/cf/0906db948c48");
200 Path preNamespaceTempPath = new Path(rootDir, ".tmp/table/21212abcdc33/cf/0906db948c48");
201 boolean preNSTablePathExists = false;
202 boolean preNSArchivePathExists = false;
203 boolean preNSTempPathExists = false;
204 assertTrue(HFileLink.isHFileLink(aFileLink));
205 HFileLink hFileLink =
206 HFileLink.buildFromHFileLinkPattern(TEST_UTIL.getConfiguration(), aFileLink);
207 assertTrue(hFileLink.getArchivePath().toString().startsWith(rootDir.toString()));
208
209 HFileV1Detector t = new HFileV1Detector();
210 t.setConf(TEST_UTIL.getConfiguration());
211 FileLink fileLink = t.getFileLinkWithPreNSPath(aFileLink);
212
213 Path[] paths = fileLink.getLocations();
214 assertTrue(paths.length == 7);
215 for (Path p : fileLink.getLocations()) {
216 if (p.equals(preNamespaceArchivePath)) preNSArchivePathExists = true;
217 if (p.equals(preNamespaceTablePath)) preNSTablePathExists = true;
218 if (p.equals(preNamespaceTempPath)) preNSTempPathExists = true;
219 }
220 assertTrue(preNSArchivePathExists & preNSTablePathExists & preNSTempPathExists);
221 }
222
223 @Test
224 public void testADirForHFileV1() throws Exception {
225 Path tablePath = new Path(hbaseRootDir, "foo");
226 System.out.println("testADirForHFileV1: " + tablePath.makeQualified(fs));
227 System.out.println("Passed: " + hbaseRootDir + "/foo");
228 assertEquals(0,
229 ToolRunner.run(TEST_UTIL.getConfiguration(), new HFileV1Detector(), new String[] { "-p"
230 + "foo" }));
231 }
232
233 @Test
234 public void testZnodeMigration() throws Exception {
235 String rootRSZnode = ZKUtil.joinZNode(zkw.baseZNode, "root-region-server");
236 assertTrue(ZKUtil.checkExists(zkw, rootRSZnode) > -1);
237 ToolRunner.run(TEST_UTIL.getConfiguration(), new UpgradeTo96(), new String[] { "-execute" });
238 assertEquals(-1, ZKUtil.checkExists(zkw, rootRSZnode));
239 byte[] data = ZKUtil.getData(zkw, tableAZnode);
240 assertTrue(ProtobufUtil.isPBMagicPrefix(data));
241 checkTableState(data, ZooKeeperProtos.Table.State.ENABLED);
242
243 data = ZKUtil.getData(zkw, peer1Znode);
244 assertTrue(ProtobufUtil.isPBMagicPrefix(data));
245 checkReplicationPeerData(data, peer1);
246 }
247
248 private void checkTableState(byte[] data, State expectedState)
249 throws InvalidProtocolBufferException {
250 ZooKeeperProtos.Table.Builder builder = ZooKeeperProtos.Table.newBuilder();
251 int magicLen = ProtobufUtil.lengthOfPBMagic();
252 ZooKeeperProtos.Table t = builder.mergeFrom(data, magicLen, data.length - magicLen).build();
253 assertTrue(t.getState() == expectedState);
254 }
255
256 private void checkReplicationPeerData(byte[] data, ReplicationPeer peer)
257 throws InvalidProtocolBufferException {
258 int magicLen = ProtobufUtil.lengthOfPBMagic();
259 ZooKeeperProtos.ReplicationPeer.Builder builder = ZooKeeperProtos.ReplicationPeer.newBuilder();
260 assertEquals(builder.mergeFrom(data, magicLen, data.length - magicLen).build().getClusterkey(),
261 peer.getClusterkey());
262
263 }
264
265 @AfterClass
266 public static void tearDownAfterClass() throws Exception {
267 TEST_UTIL.shutdownMiniHBaseCluster();
268 TEST_UTIL.shutdownMiniDFSCluster();
269 TEST_UTIL.shutdownMiniZKCluster();
270 }
271
272 }