View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.hbase.HBaseTestingUtility;
29  import org.apache.hadoop.hbase.HColumnDescriptor;
30  import org.apache.hadoop.hbase.HRegionInfo;
31  import org.apache.hadoop.hbase.HTableDescriptor;
32  import org.apache.hadoop.hbase.TableName;
33  import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
34  import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
35  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ModifyTableState;
36  import org.apache.hadoop.hbase.testclassification.MediumTests;
37  import org.junit.After;
38  import org.junit.AfterClass;
39  import org.junit.Before;
40  import org.junit.BeforeClass;
41  import org.junit.Test;
42  import org.junit.experimental.categories.Category;
43  
44  @Category(MediumTests.class)
45  public class TestModifyTableProcedure {
46    private static final Log LOG = LogFactory.getLog(TestModifyTableProcedure.class);
47  
48    protected static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
49  
50    private static void setupConf(Configuration conf) {
51      conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
52    }
53  
54    @BeforeClass
55    public static void setupCluster() throws Exception {
56      setupConf(UTIL.getConfiguration());
57      UTIL.startMiniCluster(1);
58    }
59  
60    @AfterClass
61    public static void cleanupTest() throws Exception {
62      try {
63        UTIL.shutdownMiniCluster();
64      } catch (Exception e) {
65        LOG.warn("failure shutting down cluster", e);
66      }
67    }
68  
69    @Before
70    public void setup() throws Exception {
71      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
72    }
73  
74    @After
75    public void tearDown() throws Exception {
76      ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
77      for (HTableDescriptor htd: UTIL.getHBaseAdmin().listTables()) {
78        LOG.info("Tear down, remove table=" + htd.getTableName());
79        UTIL.deleteTable(htd.getTableName());
80      }
81    }
82  
83    @Test(timeout=60000)
84    public void testModifyTable() throws Exception {
85      final TableName tableName = TableName.valueOf("testModifyTable");
86      final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
87  
88      MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf");
89      UTIL.getHBaseAdmin().disableTable(tableName);
90  
91      // Modify the table descriptor
92      HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
93  
94      // Test 1: Modify 1 property
95      long newMaxFileSize = htd.getMaxFileSize() * 2;
96      htd.setMaxFileSize(newMaxFileSize);
97      htd.setRegionReplication(3);
98  
99      long procId1 = ProcedureTestingUtility.submitAndWait(
100         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
101     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId1));
102 
103     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
104     assertEquals(newMaxFileSize, currentHtd.getMaxFileSize());
105 
106     // Test 2: Modify multiple properties
107     boolean newReadOnlyOption = htd.isReadOnly() ? false : true;
108     long newMemStoreFlushSize = htd.getMemStoreFlushSize() * 2;
109     htd.setReadOnly(newReadOnlyOption);
110     htd.setMemStoreFlushSize(newMemStoreFlushSize);
111 
112     long procId2 = ProcedureTestingUtility.submitAndWait(
113         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
114     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
115 
116     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
117     assertEquals(newReadOnlyOption, currentHtd.isReadOnly());
118     assertEquals(newMemStoreFlushSize, currentHtd.getMemStoreFlushSize());
119   }
120 
121   @Test(timeout = 60000)
122   public void testModifyTableAddCF() throws Exception {
123     final TableName tableName = TableName.valueOf("testModifyTableAddCF");
124     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
125 
126     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1");
127     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
128     assertEquals(1, currentHtd.getFamiliesKeys().size());
129 
130     // Test 1: Modify the table descriptor online
131     String cf2 = "cf2";
132     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
133     htd.addFamily(new HColumnDescriptor(cf2));
134 
135     long procId = ProcedureTestingUtility.submitAndWait(
136         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
137     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
138 
139     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
140     assertEquals(2, currentHtd.getFamiliesKeys().size());
141     assertTrue(currentHtd.hasFamily(cf2.getBytes()));
142 
143     // Test 2: Modify the table descriptor offline
144     UTIL.getHBaseAdmin().disableTable(tableName);
145     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
146     String cf3 = "cf3";
147     HTableDescriptor htd2 =
148         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
149     htd2.addFamily(new HColumnDescriptor(cf3));
150 
151     long procId2 =
152         ProcedureTestingUtility.submitAndWait(procExec,
153           new ModifyTableProcedure(procExec.getEnvironment(), htd2));
154     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
155 
156     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
157     assertTrue(currentHtd.hasFamily(cf3.getBytes()));
158     assertEquals(3, currentHtd.getFamiliesKeys().size());
159   }
160 
161   @Test(timeout = 60000)
162   public void testModifyTableDeleteCF() throws Exception {
163     final TableName tableName = TableName.valueOf("testModifyTableAddCF");
164     final String cf2 = "cf2";
165     final String cf3 = "cf3";
166     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
167 
168     MasterProcedureTestingUtility.createTable(procExec, tableName, null, "cf1", cf2, cf3);
169     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
170     assertEquals(3, currentHtd.getFamiliesKeys().size());
171 
172     // Test 1: Modify the table descriptor
173     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
174     htd.removeFamily(cf2.getBytes());
175 
176     long procId = ProcedureTestingUtility.submitAndWait(
177         procExec, new ModifyTableProcedure(procExec.getEnvironment(), htd));
178     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId));
179 
180     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
181     assertEquals(2, currentHtd.getFamiliesKeys().size());
182     assertFalse(currentHtd.hasFamily(cf2.getBytes()));
183 
184     // Test 2: Modify the table descriptor offline
185     UTIL.getHBaseAdmin().disableTable(tableName);
186     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
187 
188     HTableDescriptor htd2 =
189         new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
190     htd2.removeFamily(cf3.getBytes());
191 
192     long procId2 =
193         ProcedureTestingUtility.submitAndWait(procExec,
194           new ModifyTableProcedure(procExec.getEnvironment(), htd2));
195     ProcedureTestingUtility.assertProcNotFailed(procExec.getResult(procId2));
196 
197     currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
198     assertEquals(1, currentHtd.getFamiliesKeys().size());
199     assertFalse(currentHtd.hasFamily(cf3.getBytes()));
200   }
201 
202   @Test(timeout=60000)
203   public void testRecoveryAndDoubleExecutionOffline() throws Exception {
204     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOffline");
205     final String cf2 = "cf2";
206     final String cf3 = "cf3";
207     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
208 
209     // create the table
210     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
211       procExec, tableName, null, "cf1", cf3);
212     UTIL.getHBaseAdmin().disableTable(tableName);
213 
214     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
215     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
216 
217     // Modify multiple properties of the table.
218     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
219     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
220     htd.setCompactionEnabled(newCompactionEnableOption);
221     htd.addFamily(new HColumnDescriptor(cf2));
222     htd.removeFamily(cf3.getBytes());
223     htd.setRegionReplication(3);
224 
225     // Start the Modify procedure && kill the executor
226     long procId = procExec.submitProcedure(
227       new ModifyTableProcedure(procExec.getEnvironment(), htd));
228 
229     // Restart the executor and execute the step twice
230     int numberOfSteps = ModifyTableState.values().length;
231     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(
232       procExec,
233       procId,
234       numberOfSteps,
235       ModifyTableState.values());
236 
237     // Validate descriptor
238     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
239     assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
240     assertEquals(2, currentHtd.getFamiliesKeys().size());
241 
242     // cf2 should be added cf3 should be removed
243     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
244       tableName, regions, false, "cf1", cf2);
245   }
246 
247   @Test(timeout = 60000)
248   public void testRecoveryAndDoubleExecutionOnline() throws Exception {
249     final TableName tableName = TableName.valueOf("testRecoveryAndDoubleExecutionOnline");
250     final String cf2 = "cf2";
251     final String cf3 = "cf3";
252     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
253 
254     // create the table
255     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
256       procExec, tableName, null, "cf1", cf3);
257 
258     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
259 
260     // Modify multiple properties of the table.
261     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
262     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
263     htd.setCompactionEnabled(newCompactionEnableOption);
264     htd.addFamily(new HColumnDescriptor(cf2));
265     htd.removeFamily(cf3.getBytes());
266 
267     // Start the Modify procedure && kill the executor
268     long procId = procExec.submitProcedure(
269       new ModifyTableProcedure(procExec.getEnvironment(), htd));
270 
271     // Restart the executor and execute the step twice
272     int numberOfSteps = ModifyTableState.values().length;
273     MasterProcedureTestingUtility.testRecoveryAndDoubleExecution(procExec, procId, numberOfSteps,
274       ModifyTableState.values());
275 
276     // Validate descriptor
277     HTableDescriptor currentHtd = UTIL.getHBaseAdmin().getTableDescriptor(tableName);
278     assertEquals(newCompactionEnableOption, currentHtd.isCompactionEnabled());
279     assertEquals(2, currentHtd.getFamiliesKeys().size());
280     assertTrue(currentHtd.hasFamily(cf2.getBytes()));
281     assertFalse(currentHtd.hasFamily(cf3.getBytes()));
282 
283     // cf2 should be added cf3 should be removed
284     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
285       tableName, regions, "cf1", cf2);
286   }
287 
288   @Test(timeout = 60000)
289   public void testRollbackAndDoubleExecutionOnline() throws Exception {
290     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
291     final String familyName = "cf2";
292     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
293 
294     // create the table
295     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
296       procExec, tableName, null, "cf1");
297 
298     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
299 
300     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
301     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
302     htd.setCompactionEnabled(newCompactionEnableOption);
303     htd.addFamily(new HColumnDescriptor(familyName));
304 
305     // Start the Modify procedure && kill the executor
306     long procId = procExec.submitProcedure(
307       new ModifyTableProcedure(procExec.getEnvironment(), htd));
308 
309     // Restart the executor and rollback the step twice
310     int numberOfSteps = ModifyTableState.values().length - 4; // failing in the middle of proc
311     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
312       procExec,
313       procId,
314       numberOfSteps,
315       ModifyTableState.values());
316 
317     // cf2 should not be present
318     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
319       tableName, regions, "cf1");
320   }
321 
322   @Test(timeout = 60000)
323   public void testRollbackAndDoubleExecutionOffline() throws Exception {
324     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecution");
325     final String familyName = "cf2";
326     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
327 
328     // create the table
329     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
330       procExec, tableName, null, "cf1");
331     UTIL.getHBaseAdmin().disableTable(tableName);
332 
333     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
334     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
335 
336     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
337     boolean newCompactionEnableOption = htd.isCompactionEnabled() ? false : true;
338     htd.setCompactionEnabled(newCompactionEnableOption);
339     htd.addFamily(new HColumnDescriptor(familyName));
340     htd.setRegionReplication(3);
341 
342     // Start the Modify procedure && kill the executor
343     long procId = procExec.submitProcedure(
344       new ModifyTableProcedure(procExec.getEnvironment(), htd));
345 
346     // Restart the executor and rollback the step twice
347     int numberOfSteps = ModifyTableState.values().length - 4; // failing in the middle of proc
348     MasterProcedureTestingUtility.testRollbackAndDoubleExecution(
349       procExec,
350       procId,
351       numberOfSteps,
352       ModifyTableState.values());
353 
354     // cf2 should not be present
355     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
356       tableName, regions, "cf1");
357   }
358 
359   @Test(timeout = 60000)
360   public void testRollbackAndDoubleExecutionAfterPONR() throws Exception {
361     final TableName tableName = TableName.valueOf("testRollbackAndDoubleExecutionAfterPONR");
362     final String familyToAddName = "cf2";
363     final String familyToRemove = "cf1";
364     final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
365 
366     // create the table
367     HRegionInfo[] regions = MasterProcedureTestingUtility.createTable(
368       procExec, tableName, null, familyToRemove);
369     UTIL.getHBaseAdmin().disableTable(tableName);
370 
371     ProcedureTestingUtility.waitNoProcedureRunning(procExec);
372     ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);
373 
374     HTableDescriptor htd = new HTableDescriptor(UTIL.getHBaseAdmin().getTableDescriptor(tableName));
375     htd.setCompactionEnabled(!htd.isCompactionEnabled());
376     htd.addFamily(new HColumnDescriptor(familyToAddName));
377     htd.removeFamily(familyToRemove.getBytes());
378     htd.setRegionReplication(3);
379 
380     // Start the Modify procedure && kill the executor
381     long procId = procExec.submitProcedure(
382       new ModifyTableProcedure(procExec.getEnvironment(), htd));
383 
384     // Failing after MODIFY_TABLE_DELETE_FS_LAYOUT we should not trigger the rollback.
385     // NOTE: the 5 (number of MODIFY_TABLE_DELETE_FS_LAYOUT + 1 step) is hardcoded,
386     //       so you have to look at this test at least once when you add a new step.
387     int numberOfSteps = 5;
388     MasterProcedureTestingUtility.testRollbackAndDoubleExecutionAfterPONR(
389       procExec,
390       procId,
391       numberOfSteps,
392       ModifyTableState.values());
393 
394     // "cf2" should be added and "cf1" should be removed
395     MasterProcedureTestingUtility.validateTableCreation(UTIL.getHBaseCluster().getMaster(),
396       tableName, regions, false, familyToAddName);
397   }
398 
399   private ProcedureExecutor<MasterProcedureEnv> getMasterProcedureExecutor() {
400     return UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor();
401   }
402 }