1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.wal;
20
21 import java.io.Closeable;
22 import java.io.DataInput;
23 import java.io.DataOutput;
24 import java.io.IOException;
25 import java.util.List;
26 import java.util.regex.Pattern;
27
28 import com.google.common.annotations.VisibleForTesting;
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.hbase.classification.InterfaceAudience;
32 import org.apache.hadoop.hbase.classification.InterfaceStability;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FSDataInputStream;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.hbase.HConstants;
38 import org.apache.hadoop.hbase.ServerName;
39 import org.apache.hadoop.hbase.util.FSUtils;
40
41
42 import org.apache.hadoop.hbase.regionserver.wal.FSHLog;
43 import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter;
44 import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
45
46
47
48
49
50
51
52
53
54
55
56
57 @InterfaceAudience.Private
58 @InterfaceStability.Evolving
59 public class DefaultWALProvider implements WALProvider {
60 private static final Log LOG = LogFactory.getLog(DefaultWALProvider.class);
61
62
63 public interface Reader extends WAL.Reader {
64
65
66
67
68
69
70 void init(FileSystem fs, Path path, Configuration c, FSDataInputStream s) throws IOException;
71 }
72
73
74 public interface Writer extends WALProvider.Writer {
75 void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
76 }
77
78 protected FSHLog log = null;
79
80
81
82
83
84
85
86
87 @Override
88 public void init(final WALFactory factory, final Configuration conf,
89 final List<WALActionsListener> listeners, String providerId) throws IOException {
90 if (null != log) {
91 throw new IllegalStateException("WALProvider.init should only be called once.");
92 }
93 if (null == providerId) {
94 providerId = DEFAULT_PROVIDER_ID;
95 }
96 final String logPrefix = factory.factoryId + WAL_FILE_NAME_DELIMITER + providerId;
97 log = new FSHLog(FSUtils.getWALFileSystem(conf), FSUtils.getWALRootDir(conf),
98 getWALDirectoryName(factory.factoryId), HConstants.HREGION_OLDLOGDIR_NAME, conf, listeners,
99 true, logPrefix, META_WAL_PROVIDER_ID.equals(providerId) ? META_WAL_PROVIDER_ID : null);
100 }
101
102 @Override
103 public WAL getWAL(final byte[] identifier) throws IOException {
104 return log;
105 }
106
107 @Override
108 public void close() throws IOException {
109 log.close();
110 }
111
112 @Override
113 public void shutdown() throws IOException {
114 log.shutdown();
115 }
116
117
118 public static final String WAL_FILE_NAME_DELIMITER = ".";
119
120 @VisibleForTesting
121 public static final String META_WAL_PROVIDER_ID = ".meta";
122 static final String DEFAULT_PROVIDER_ID = "default";
123
124
125
126 public static final String SPLITTING_EXT = "-splitting";
127
128
129
130
131
132
133
134 public static long getNumLogFiles(WALFactory walFactory) {
135 long result = 0;
136 if (walFactory.provider instanceof DefaultWALProvider) {
137 result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getNumLogFiles();
138 }
139 WALProvider meta = walFactory.metaProvider.get();
140 if (meta instanceof DefaultWALProvider) {
141 result += ((FSHLog)((DefaultWALProvider)meta).log).getNumLogFiles();
142 }
143 return result;
144 }
145
146
147
148
149
150
151
152 public static long getLogFileSize(WALFactory walFactory) {
153 long result = 0;
154 if (walFactory.provider instanceof DefaultWALProvider) {
155 result += ((FSHLog)((DefaultWALProvider)walFactory.provider).log).getLogFileSize();
156 }
157 WALProvider meta = walFactory.metaProvider.get();
158 if (meta instanceof DefaultWALProvider) {
159 result += ((FSHLog)((DefaultWALProvider)meta).log).getLogFileSize();
160 }
161 return result;
162 }
163
164
165
166
167 @VisibleForTesting
168 public static int getNumRolledLogFiles(WAL wal) {
169 return ((FSHLog)wal).getNumRolledLogFiles();
170 }
171
172
173
174
175 @VisibleForTesting
176 public static Path getCurrentFileName(final WAL wal) {
177 return ((FSHLog)wal).getCurrentFileName();
178 }
179
180
181
182
183 @VisibleForTesting
184 static void requestLogRoll(final WAL wal) {
185 ((FSHLog)wal).requestLogRoll();
186 }
187
188
189
190
191
192
193
194
195 @VisibleForTesting
196 public static long extractFileNumFromWAL(final WAL wal) {
197 final Path walName = ((FSHLog)wal).getCurrentFileName();
198 return extractFileNumFromWAL(walName);
199 }
200
201 @VisibleForTesting
202 public static long extractFileNumFromWAL(final Path walName) {
203 if (walName == null) {
204 throw new IllegalArgumentException("The WAL path couldn't be null");
205 }
206 final String[] walPathStrs = walName.toString().split("\\" + WAL_FILE_NAME_DELIMITER);
207 return Long.parseLong(walPathStrs[walPathStrs.length - (isMetaFile(walName) ? 2:1)]);
208 }
209
210
211
212
213
214 private static final Pattern pattern = Pattern.compile(".*\\.\\d*("+META_WAL_PROVIDER_ID+")*");
215
216
217
218
219
220
221
222
223
224
225
226 public static boolean validateWALFilename(String filename) {
227 return pattern.matcher(filename).matches();
228 }
229
230
231
232
233
234
235
236
237
238
239
240 public static String getWALDirectoryName(final String serverName) {
241 StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
242 dirName.append("/");
243 dirName.append(serverName);
244 return dirName.toString();
245 }
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263 public static ServerName getServerNameFromWALDirectoryName(Configuration conf, String path)
264 throws IOException {
265 if (path == null
266 || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
267 return null;
268 }
269
270 if (conf == null) {
271 throw new IllegalArgumentException("parameter conf must be set");
272 }
273
274 final String walDir = FSUtils.getWALRootDir(conf).toString();
275
276 final StringBuilder startPathSB = new StringBuilder(walDir);
277 if (!walDir.endsWith("/"))
278 startPathSB.append('/');
279 startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
280 if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
281 startPathSB.append('/');
282 final String startPath = startPathSB.toString();
283
284 String fullPath;
285 try {
286 fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
287 } catch (IllegalArgumentException e) {
288 LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
289 return null;
290 }
291
292 if (!fullPath.startsWith(startPath)) {
293 return null;
294 }
295
296 final String serverNameAndFile = fullPath.substring(startPath.length());
297
298 if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
299
300 return null;
301 }
302
303 Path p = new Path(path);
304 return getServerNameFromWALDirectoryName(p);
305 }
306
307
308
309
310
311
312
313
314
315
316
317 public static ServerName getServerNameFromWALDirectoryName(Path logFile) {
318 String logDirName = logFile.getParent().getName();
319
320 if (logDirName.equals(HConstants.HREGION_LOGDIR_NAME)) {
321 logDirName = logFile.getName();
322 }
323 ServerName serverName = null;
324 if (logDirName.endsWith(SPLITTING_EXT)) {
325 logDirName = logDirName.substring(0, logDirName.length() - SPLITTING_EXT.length());
326 }
327 try {
328 serverName = ServerName.parseServerName(logDirName);
329 } catch (IllegalArgumentException ex) {
330 serverName = null;
331 LOG.warn("Cannot parse a server name from path=" + logFile + "; " + ex.getMessage());
332 }
333 if (serverName != null && serverName.getStartcode() < 0) {
334 LOG.warn("Invalid log file path=" + logFile);
335 serverName = null;
336 }
337 return serverName;
338 }
339
340 public static boolean isMetaFile(Path p) {
341 return isMetaFile(p.getName());
342 }
343
344 public static boolean isMetaFile(String p) {
345 if (p != null && p.endsWith(META_WAL_PROVIDER_ID)) {
346 return true;
347 }
348 return false;
349 }
350
351
352
353
354 public static Writer createWriter(final Configuration conf, final FileSystem fs, final Path path,
355 final boolean overwritable)
356 throws IOException {
357
358 Class<? extends Writer> logWriterClass = conf.getClass("hbase.regionserver.hlog.writer.impl",
359 ProtobufLogWriter.class, Writer.class);
360 Writer writer = null;
361 try {
362 writer = logWriterClass.newInstance();
363 writer.init(fs, path, conf, overwritable);
364 return writer;
365 } catch (Exception e) {
366 LOG.debug("Error instantiating log writer.", e);
367 if (writer != null) {
368 try{
369 writer.close();
370 } catch(IOException ee){
371 LOG.error("cannot close log writer", ee);
372 }
373 }
374 throw new IOException("cannot get log writer", e);
375 }
376 }
377
378 }