View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.wal;
20  
21  import java.io.IOException;
22  import java.util.Collection;
23  import java.util.List;
24  
25  import org.apache.commons.logging.Log;
26  import org.apache.commons.logging.LogFactory;
27  import org.apache.hadoop.hbase.classification.InterfaceAudience;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.util.FSUtils;
33  import org.apache.hadoop.hbase.wal.WAL.Entry;
34  
35  import static org.apache.hadoop.hbase.wal.DefaultWALProvider.DEFAULT_PROVIDER_ID;
36  import static org.apache.hadoop.hbase.wal.DefaultWALProvider.META_WAL_PROVIDER_ID;
37  import static org.apache.hadoop.hbase.wal.DefaultWALProvider.WAL_FILE_NAME_DELIMITER;
38  
39  
40  // imports for things that haven't moved from regionserver.wal yet.
41  import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
42  import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
43  import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
44  
45  /**
46   * A WAL Provider that returns a single thread safe WAL that optionally can skip parts of our
47   * normal interactions with HDFS.
48   *
49   * This implementation picks a directory in HDFS based on the same mechanisms as the 
50   * {@link DefaultWALProvider}. Users can configure how much interaction
51   * we have with HDFS with the configuration property "hbase.wal.iotestprovider.operations".
52   * The value should be a comma separated list of allowed operations:
53   * <ul>
54   *   <li><em>append</em>   : edits will be written to the underlying filesystem
55   *   <li><em>sync</em>     : wal syncs will result in hflush calls
56   *   <li><em>fileroll</em> : roll requests will result in creating a new file on the underlying
57   *                           filesystem.
58   * </ul>
59   * Additionally, the special cases "all" and "none" are recognized.
60   * If ommited, the value defaults to "all."
61   * Behavior is undefined if "all" or "none" are paired with additional values. Behavior is also
62   * undefined if values not listed above are included.
63   *
64   * Only those operations listed will occur between the returned WAL and HDFS. All others
65   * will be no-ops.
66   *
67   * Note that in the case of allowing "append" operations but not allowing "fileroll", the returned
68   * WAL will just keep writing to the same file. This won't avoid all costs associated with file
69   * management over time, becaue the data set size may result in additional HDFS block allocations.
70   *
71   */
72  @InterfaceAudience.Private
73  public class IOTestProvider implements WALProvider {
74    private static final Log LOG = LogFactory.getLog(IOTestProvider.class);
75  
76    private static final String ALLOWED_OPERATIONS = "hbase.wal.iotestprovider.operations";
77    private enum AllowedOperations {
78      all,
79      append,
80      sync,
81      fileroll,
82      none;
83    }
84  
85    private FSHLog log = null;
86  
87    /**
88     * @param factory factory that made us, identity used for FS layout. may not be null
89     * @param conf may not be null
90     * @param listeners may be null
91     * @param providerId differentiate between providers from one facotry, used for FS layout. may be
92     *                   null
93     */
94    @Override
95    public void init(final WALFactory factory, final Configuration conf,
96        final List<WALActionsListener> listeners, String providerId) throws IOException {
97      if (null != log) {
98        throw new IllegalStateException("WALProvider.init should only be called once.");
99      }
100     if (null == providerId) {
101       providerId = DEFAULT_PROVIDER_ID;
102     }
103     final String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId;
104     log = new IOTestWAL(FSUtils.getWALFileSystem(conf), FSUtils.getWALRootDir(conf),
105         DefaultWALProvider.getWALDirectoryName(factory.factoryId),
106         HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners,
107         true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);
108   }
109 
110   @Override
111   public WAL getWAL(final byte[] identifier) throws IOException {
112    return log;
113   }
114 
115   @Override
116   public void close() throws IOException {
117     log.close();
118   }
119 
120   @Override
121   public void shutdown() throws IOException {
122     log.shutdown();
123   }
124 
125   private static class IOTestWAL extends FSHLog {
126 
127     private final boolean doFileRolls;
128 
129     // Used to differntiate between roll calls before and after we finish construction.
130     private final boolean initialized;
131 
132     /**
133      * Create an edit log at the given <code>dir</code> location.
134      *
135      * You should never have to load an existing log. If there is a log at
136      * startup, it should have already been processed and deleted by the time the
137      * WAL object is started up.
138      *
139      * @param fs filesystem handle
140      * @param rootDir path to where logs and oldlogs
141      * @param logDir dir where wals are stored
142      * @param archiveDir dir where wals are archived
143      * @param conf configuration to use
144      * @param listeners Listeners on WAL events. Listeners passed here will
145      * be registered before we do anything else; e.g. the
146      * Constructor {@link #rollWriter()}.
147      * @param failIfWALExists If true IOException will be thrown if files related to this wal
148      *        already exist.
149      * @param prefix should always be hostname and port in distributed env and
150      *        it will be URL encoded before being used.
151      *        If prefix is null, "wal" will be used
152      * @param suffix will be url encoded. null is treated as empty. non-empty must start with
153      *        {@link DefaultWALProvider#WAL_FILE_NAME_DELIMITER}
154      * @throws IOException
155      */
156     public IOTestWAL(final FileSystem fs, final Path rootDir, final String logDir,
157         final String archiveDir, final Configuration conf,
158         final List<WALActionsListener> listeners,
159         final boolean failIfWALExists, final String prefix, final String suffix)
160         throws IOException {
161       super(fs, rootDir, logDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix);
162       Collection<String> operations = conf.getStringCollection(ALLOWED_OPERATIONS);
163       doFileRolls = operations.isEmpty() || operations.contains(AllowedOperations.all.name()) ||
164           operations.contains(AllowedOperations.fileroll.name());
165       initialized = true;
166       LOG.info("Initialized with file rolling " + (doFileRolls ? "enabled" : "disabled"));
167     }
168 
169     private Writer noRollsWriter;
170 
171     // creatWriterInstance is where the new pipeline is set up for doing file rolls
172     // if we are skipping it, just keep returning the same writer.
173     @Override
174     protected Writer createWriterInstance(final Path path) throws IOException {
175       // we get called from the FSHLog constructor (!); always roll in this case since
176       // we don't know yet if we're supposed to generally roll and
177       // we need an initial file in the case of doing appends but no rolls.
178       if (!initialized || doFileRolls) {
179         LOG.info("creating new writer instance.");
180         final ProtobufLogWriter writer = new IOTestWriter();
181         writer.init(fs, path, conf, false);
182         if (!initialized) {
183           LOG.info("storing initial writer instance in case file rolling isn't allowed.");
184           noRollsWriter = writer;
185         }
186         return writer;
187       } else {
188         LOG.info("WAL rolling disabled, returning the first writer.");
189         // Initial assignment happens during the constructor call, so there ought not be
190         // a race for first assignment.
191         return noRollsWriter;
192       }
193     }
194   }
195 
196   /**
197    * Presumes init will be called by a single thread prior to any access of other methods.
198    */
199   private static class IOTestWriter extends ProtobufLogWriter {
200     private boolean doAppends;
201     private boolean doSyncs;
202 
203     @Override
204     public void init(FileSystem fs, Path path, Configuration conf, boolean overwritable) throws IOException {
205       Collection<String> operations = conf.getStringCollection(ALLOWED_OPERATIONS);
206       if (operations.isEmpty() || operations.contains(AllowedOperations.all.name())) {
207         doAppends = doSyncs = true;
208       } else if (operations.contains(AllowedOperations.none.name())) {
209         doAppends = doSyncs = false;
210       } else {
211         doAppends = operations.contains(AllowedOperations.append.name());
212         doSyncs = operations.contains(AllowedOperations.sync.name());
213       }
214       LOG.info("IOTestWriter initialized with appends " + (doAppends ? "enabled" : "disabled") +
215           " and syncs " + (doSyncs ? "enabled" : "disabled"));
216       super.init(fs, path, conf, overwritable);
217     }
218 
219     @Override
220     public void append(Entry entry) throws IOException {
221       if (doAppends) {
222         super.append(entry);
223       }
224     }
225 
226     @Override
227     public void sync() throws IOException {
228       if (doSyncs) {
229         super.sync();
230       }
231     }
232   }
233 }