View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.master.procedure;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.OutputStream;
24  import java.security.PrivilegedExceptionAction;
25  import java.util.HashSet;
26  import java.util.List;
27  import java.util.Set;
28  import java.util.concurrent.atomic.AtomicBoolean;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.HRegionInfo;
34  import org.apache.hadoop.hbase.HTableDescriptor;
35  import org.apache.hadoop.hbase.MetaTableAccessor;
36  import org.apache.hadoop.hbase.TableName;
37  import org.apache.hadoop.hbase.TableNotDisabledException;
38  import org.apache.hadoop.hbase.TableNotFoundException;
39  import org.apache.hadoop.hbase.classification.InterfaceAudience;
40  import org.apache.hadoop.hbase.client.Connection;
41  import org.apache.hadoop.hbase.client.Result;
42  import org.apache.hadoop.hbase.client.ResultScanner;
43  import org.apache.hadoop.hbase.client.Scan;
44  import org.apache.hadoop.hbase.client.Table;
45  import org.apache.hadoop.hbase.executor.EventType;
46  import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
47  import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
48  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos;
49  import org.apache.hadoop.hbase.protobuf.generated.MasterProcedureProtos.ModifyTableState;
50  import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos;
51  import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil;
52  import org.apache.hadoop.security.UserGroupInformation;
53  
54  @InterfaceAudience.Private
55  public class ModifyTableProcedure
56      extends StateMachineProcedure<MasterProcedureEnv, ModifyTableState>
57      implements TableProcedureInterface {
58    private static final Log LOG = LogFactory.getLog(ModifyTableProcedure.class);
59  
60    private final AtomicBoolean aborted = new AtomicBoolean(false);
61  
62    private HTableDescriptor unmodifiedHTableDescriptor = null;
63    private HTableDescriptor modifiedHTableDescriptor;
64    private UserGroupInformation user;
65    private boolean deleteColumnFamilyInModify;
66  
67    private List<HRegionInfo> regionInfoList;
68    private Boolean traceEnabled = null;
69  
70    public ModifyTableProcedure() {
71      initilize();
72    }
73  
74    public ModifyTableProcedure(
75      final MasterProcedureEnv env,
76      final HTableDescriptor htd) throws IOException {
77      initilize();
78      this.modifiedHTableDescriptor = htd;
79      this.user = env.getRequestUser().getUGI();
80      this.setOwner(this.user.getShortUserName());
81    }
82  
83    private void initilize() {
84      this.unmodifiedHTableDescriptor = null;
85      this.regionInfoList = null;
86      this.traceEnabled = null;
87      this.deleteColumnFamilyInModify = false;
88    }
89  
90    @Override
91    protected Flow executeFromState(final MasterProcedureEnv env, final ModifyTableState state) {
92      if (isTraceEnabled()) {
93        LOG.trace(this + " execute state=" + state);
94      }
95  
96      try {
97        switch (state) {
98        case MODIFY_TABLE_PREPARE:
99          prepareModify(env);
100         setNextState(ModifyTableState.MODIFY_TABLE_PRE_OPERATION);
101         break;
102       case MODIFY_TABLE_PRE_OPERATION:
103         preModify(env, state);
104         setNextState(ModifyTableState.MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR);
105         break;
106       case MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR:
107         updateTableDescriptor(env);
108         setNextState(ModifyTableState.MODIFY_TABLE_REMOVE_REPLICA_COLUMN);
109         break;
110       case MODIFY_TABLE_REMOVE_REPLICA_COLUMN:
111         updateReplicaColumnsIfNeeded(env, unmodifiedHTableDescriptor, modifiedHTableDescriptor);
112         if (deleteColumnFamilyInModify) {
113           setNextState(ModifyTableState.MODIFY_TABLE_DELETE_FS_LAYOUT);
114         } else {
115           setNextState(ModifyTableState.MODIFY_TABLE_POST_OPERATION);
116         }
117         break;
118       case MODIFY_TABLE_DELETE_FS_LAYOUT:
119         deleteFromFs(env, unmodifiedHTableDescriptor, modifiedHTableDescriptor);
120         setNextState(ModifyTableState.MODIFY_TABLE_POST_OPERATION);
121         break;
122       case MODIFY_TABLE_POST_OPERATION:
123         postModify(env, state);
124         setNextState(ModifyTableState.MODIFY_TABLE_REOPEN_ALL_REGIONS);
125         break;
126       case MODIFY_TABLE_REOPEN_ALL_REGIONS:
127         reOpenAllRegionsIfTableIsOnline(env);
128         return Flow.NO_MORE_STATE;
129       default:
130         throw new UnsupportedOperationException("unhandled state=" + state);
131       }
132     } catch (InterruptedException|IOException e) {
133       if (!isRollbackSupported(state)) {
134         // We reach a state that cannot be rolled back. We just need to keep retry.
135         LOG.warn("Error trying to modify table=" + getTableName() + " state=" + state, e);
136       } else {
137         LOG.error("Error trying to modify table=" + getTableName() + " state=" + state, e);
138         setFailure("master-modify-table", e);
139       }
140     }
141     return Flow.HAS_MORE_STATE;
142   }
143 
144   @Override
145   protected void rollbackState(final MasterProcedureEnv env, final ModifyTableState state)
146       throws IOException {
147     if (isTraceEnabled()) {
148       LOG.trace(this + " rollback state=" + state);
149     }
150     try {
151       switch (state) {
152       case MODIFY_TABLE_REOPEN_ALL_REGIONS:
153         break; // Nothing to undo.
154       case MODIFY_TABLE_POST_OPERATION:
155         // TODO-MAYBE: call the coprocessor event to un-modify?
156         break;
157       case MODIFY_TABLE_DELETE_FS_LAYOUT:
158         // Once we reach to this state - we could NOT rollback - as it is tricky to undelete
159         // the deleted files. We are not suppose to reach here, throw exception so that we know
160         // there is a code bug to investigate.
161         assert deleteColumnFamilyInModify;
162         throw new UnsupportedOperationException(this + " rollback of state=" + state
163             + " is unsupported.");
164       case MODIFY_TABLE_REMOVE_REPLICA_COLUMN:
165         // Undo the replica column update.
166         updateReplicaColumnsIfNeeded(env, modifiedHTableDescriptor, unmodifiedHTableDescriptor);
167         break;
168       case MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR:
169         restoreTableDescriptor(env);
170         break;
171       case MODIFY_TABLE_PRE_OPERATION:
172         // TODO-MAYBE: call the coprocessor event to un-modify?
173         break;
174       case MODIFY_TABLE_PREPARE:
175         break; // Nothing to undo.
176       default:
177         throw new UnsupportedOperationException("unhandled state=" + state);
178       }
179     } catch (IOException e) {
180       LOG.warn("Fail trying to rollback modify table=" + getTableName() + " state=" + state, e);
181       throw e;
182     }
183   }
184 
185   @Override
186   protected ModifyTableState getState(final int stateId) {
187     return ModifyTableState.valueOf(stateId);
188   }
189 
190   @Override
191   protected int getStateId(final ModifyTableState state) {
192     return state.getNumber();
193   }
194 
195   @Override
196   protected ModifyTableState getInitialState() {
197     return ModifyTableState.MODIFY_TABLE_PREPARE;
198   }
199 
200   @Override
201   protected void setNextState(final ModifyTableState state) {
202     if (aborted.get() && isRollbackSupported(state)) {
203       setAbortFailure("modify-table", "abort requested");
204     } else {
205       super.setNextState(state);
206     }
207   }
208 
209   @Override
210   public boolean abort(final MasterProcedureEnv env) {
211     aborted.set(true);
212     return true;
213   }
214 
215   @Override
216   protected boolean acquireLock(final MasterProcedureEnv env) {
217     if (!env.isInitialized()) return false;
218     return env.getProcedureQueue().tryAcquireTableWrite(
219       getTableName(),
220       EventType.C_M_MODIFY_TABLE.toString());
221   }
222 
223   @Override
224   protected void releaseLock(final MasterProcedureEnv env) {
225     env.getProcedureQueue().releaseTableWrite(getTableName());
226   }
227 
228   @Override
229   public void serializeStateData(final OutputStream stream) throws IOException {
230     super.serializeStateData(stream);
231 
232     MasterProcedureProtos.ModifyTableStateData.Builder modifyTableMsg =
233         MasterProcedureProtos.ModifyTableStateData.newBuilder()
234             .setUserInfo(MasterProcedureUtil.toProtoUserInfo(user))
235             .setModifiedTableSchema(modifiedHTableDescriptor.convert())
236             .setDeleteColumnFamilyInModify(deleteColumnFamilyInModify);
237 
238     if (unmodifiedHTableDescriptor != null) {
239       modifyTableMsg.setUnmodifiedTableSchema(unmodifiedHTableDescriptor.convert());
240     }
241 
242     modifyTableMsg.build().writeDelimitedTo(stream);
243   }
244 
245   @Override
246   public void deserializeStateData(final InputStream stream) throws IOException {
247     super.deserializeStateData(stream);
248 
249     MasterProcedureProtos.ModifyTableStateData modifyTableMsg =
250         MasterProcedureProtos.ModifyTableStateData.parseDelimitedFrom(stream);
251     user = MasterProcedureUtil.toUserInfo(modifyTableMsg.getUserInfo());
252     modifiedHTableDescriptor = HTableDescriptor.convert(modifyTableMsg.getModifiedTableSchema());
253     deleteColumnFamilyInModify = modifyTableMsg.getDeleteColumnFamilyInModify();
254 
255     if (modifyTableMsg.hasUnmodifiedTableSchema()) {
256       unmodifiedHTableDescriptor =
257           HTableDescriptor.convert(modifyTableMsg.getUnmodifiedTableSchema());
258     }
259   }
260 
261   @Override
262   public void toStringClassDetails(StringBuilder sb) {
263     sb.append(getClass().getSimpleName());
264     sb.append(" (table=");
265     sb.append(getTableName());
266     sb.append(")");
267   }
268 
269   @Override
270   public TableName getTableName() {
271     return modifiedHTableDescriptor.getTableName();
272   }
273 
274   @Override
275   public TableOperationType getTableOperationType() {
276     return TableOperationType.EDIT;
277   }
278 
279   /**
280    * Check conditions before any real action of modifying a table.
281    * @param env MasterProcedureEnv
282    * @throws IOException
283    */
284   private void prepareModify(final MasterProcedureEnv env) throws IOException {
285     // Checks whether the table exists
286     if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), getTableName())) {
287       throw new TableNotFoundException(getTableName());
288     }
289 
290     // In order to update the descriptor, we need to retrieve the old descriptor for comparison.
291     this.unmodifiedHTableDescriptor =
292         env.getMasterServices().getTableDescriptors().get(getTableName());
293 
294     if (env.getMasterServices().getAssignmentManager().getTableStateManager()
295         .isTableState(getTableName(), ZooKeeperProtos.Table.State.ENABLED)) {
296       // We only execute this procedure with table online if online schema change config is set.
297       if (!MasterDDLOperationHelper.isOnlineSchemaChangeAllowed(env)) {
298         throw new TableNotDisabledException(getTableName());
299       }
300 
301       if (modifiedHTableDescriptor.getRegionReplication() != unmodifiedHTableDescriptor
302           .getRegionReplication()) {
303         throw new IOException("REGION_REPLICATION change is not supported for enabled tables");
304       }
305     }
306 
307     // Find out whether all column families in unmodifiedHTableDescriptor also exists in
308     // the modifiedHTableDescriptor. This is to determine whether we are safe to rollback.
309     final Set<byte[]> oldFamilies = unmodifiedHTableDescriptor.getFamiliesKeys();
310     final Set<byte[]> newFamilies = modifiedHTableDescriptor.getFamiliesKeys();
311     for (byte[] familyName : oldFamilies) {
312       if (!newFamilies.contains(familyName)) {
313         this.deleteColumnFamilyInModify = true;
314         break;
315       }
316     }
317   }
318 
319   /**
320    * Action before modifying table.
321    * @param env MasterProcedureEnv
322    * @param state the procedure state
323    * @throws IOException
324    * @throws InterruptedException
325    */
326   private void preModify(final MasterProcedureEnv env, final ModifyTableState state)
327       throws IOException, InterruptedException {
328     runCoprocessorAction(env, state);
329   }
330 
331   /**
332    * Update descriptor
333    * @param env MasterProcedureEnv
334    * @throws IOException
335    **/
336   private void updateTableDescriptor(final MasterProcedureEnv env) throws IOException {
337     env.getMasterServices().getTableDescriptors().add(modifiedHTableDescriptor);
338   }
339 
340   /**
341    * Undo the descriptor change (for rollback)
342    * @param env MasterProcedureEnv
343    * @throws IOException
344    **/
345   private void restoreTableDescriptor(final MasterProcedureEnv env) throws IOException {
346     env.getMasterServices().getTableDescriptors().add(unmodifiedHTableDescriptor);
347 
348     // delete any new column families from the modifiedHTableDescriptor.
349     deleteFromFs(env, modifiedHTableDescriptor, unmodifiedHTableDescriptor);
350 
351     // Make sure regions are opened after table descriptor is updated.
352     reOpenAllRegionsIfTableIsOnline(env);
353   }
354 
355   /**
356    * Removes from hdfs the families that are not longer present in the new table descriptor.
357    * @param env MasterProcedureEnv
358    * @throws IOException
359    */
360   private void deleteFromFs(final MasterProcedureEnv env,
361       final HTableDescriptor oldHTableDescriptor, final HTableDescriptor newHTableDescriptor)
362       throws IOException {
363     final Set<byte[]> oldFamilies = oldHTableDescriptor.getFamiliesKeys();
364     final Set<byte[]> newFamilies = newHTableDescriptor.getFamiliesKeys();
365     for (byte[] familyName : oldFamilies) {
366       if (!newFamilies.contains(familyName)) {
367         MasterDDLOperationHelper.deleteColumnFamilyFromFileSystem(
368           env,
369           getTableName(),
370           getRegionInfoList(env),
371           familyName,
372           oldHTableDescriptor.getFamily(familyName).isMobEnabled());
373       }
374     }
375   }
376 
377   /**
378    * update replica column families if necessary.
379    * @param env MasterProcedureEnv
380    * @throws IOException
381    */
382   private void updateReplicaColumnsIfNeeded(
383     final MasterProcedureEnv env,
384     final HTableDescriptor oldHTableDescriptor,
385     final HTableDescriptor newHTableDescriptor) throws IOException {
386     final int oldReplicaCount = oldHTableDescriptor.getRegionReplication();
387     final int newReplicaCount = newHTableDescriptor.getRegionReplication();
388 
389     if (newReplicaCount < oldReplicaCount) {
390       Set<byte[]> tableRows = new HashSet<byte[]>();
391       Connection connection = env.getMasterServices().getConnection();
392       Scan scan = MetaTableAccessor.getScanForTableName(getTableName());
393       scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
394 
395       try (Table metaTable = connection.getTable(TableName.META_TABLE_NAME)) {
396         ResultScanner resScanner = metaTable.getScanner(scan);
397         for (Result result : resScanner) {
398           tableRows.add(result.getRow());
399         }
400         MetaTableAccessor.removeRegionReplicasFromMeta(
401           tableRows,
402           newReplicaCount,
403           oldReplicaCount - newReplicaCount,
404           connection);
405       }
406     }
407 
408     // Setup replication for region replicas if needed
409     if (newReplicaCount > 1 && oldReplicaCount <= 1) {
410       ServerRegionReplicaUtil.setupRegionReplicaReplication(env.getMasterConfiguration());
411     }
412   }
413 
414   /**
415    * Action after modifying table.
416    * @param env MasterProcedureEnv
417    * @param state the procedure state
418    * @throws IOException
419    * @throws InterruptedException
420    */
421   private void postModify(final MasterProcedureEnv env, final ModifyTableState state)
422       throws IOException, InterruptedException {
423     runCoprocessorAction(env, state);
424   }
425 
426   /**
427    * Last action from the procedure - executed when online schema change is supported.
428    * @param env MasterProcedureEnv
429    * @throws IOException
430    */
431   private void reOpenAllRegionsIfTableIsOnline(final MasterProcedureEnv env) throws IOException {
432     // This operation only run when the table is enabled.
433     if (!env.getMasterServices().getAssignmentManager().getTableStateManager()
434         .isTableState(getTableName(), ZooKeeperProtos.Table.State.ENABLED)) {
435       return;
436     }
437 
438     if (MasterDDLOperationHelper.reOpenAllRegions(env, getTableName(), getRegionInfoList(env))) {
439       LOG.info("Completed modify table operation on table " + getTableName());
440     } else {
441       LOG.warn("Error on reopening the regions on table " + getTableName());
442     }
443   }
444 
445   /**
446    * The procedure could be restarted from a different machine. If the variable is null, we need to
447    * retrieve it.
448    * @return traceEnabled whether the trace is enabled
449    */
450   private Boolean isTraceEnabled() {
451     if (traceEnabled == null) {
452       traceEnabled = LOG.isTraceEnabled();
453     }
454     return traceEnabled;
455   }
456 
457   /**
458    * Coprocessor Action.
459    * @param env MasterProcedureEnv
460    * @param state the procedure state
461    * @throws IOException
462    * @throws InterruptedException
463    */
464   private void runCoprocessorAction(final MasterProcedureEnv env, final ModifyTableState state)
465       throws IOException, InterruptedException {
466     final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
467     if (cpHost != null) {
468       user.doAs(new PrivilegedExceptionAction<Void>() {
469         @Override
470         public Void run() throws Exception {
471           switch (state) {
472           case MODIFY_TABLE_PRE_OPERATION:
473             cpHost.preModifyTableHandler(getTableName(), modifiedHTableDescriptor);
474             break;
475           case MODIFY_TABLE_POST_OPERATION:
476             cpHost.postModifyTableHandler(getTableName(), modifiedHTableDescriptor);
477             break;
478           default:
479             throw new UnsupportedOperationException(this + " unhandled state=" + state);
480           }
481           return null;
482         }
483       });
484     }
485   }
486 
487   /*
488    * Check whether we are in the state that can be rollback
489    */
490   private boolean isRollbackSupported(final ModifyTableState state) {
491     if (deleteColumnFamilyInModify) {
492       switch (state) {
493       case MODIFY_TABLE_DELETE_FS_LAYOUT:
494       case MODIFY_TABLE_POST_OPERATION:
495       case MODIFY_TABLE_REOPEN_ALL_REGIONS:
496         // It is not safe to rollback if we reach to these states.
497         return false;
498       default:
499         break;
500       }
501     }
502     return true;
503   }
504 
505   private List<HRegionInfo> getRegionInfoList(final MasterProcedureEnv env) throws IOException {
506     if (regionInfoList == null) {
507       regionInfoList = ProcedureSyncWait.getRegionsFromMeta(env, getTableName());
508     }
509     return regionInfoList;
510   }
511 }