1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17 package org.apache.hadoop.hbase.quotas;
18
19 import static org.junit.Assert.assertEquals;
20 import static org.junit.Assert.assertFalse;
21 import static org.junit.Assert.assertNull;
22 import static org.junit.Assert.assertTrue;
23 import static org.junit.Assert.fail;
24
25 import java.util.ArrayList;
26 import java.util.HashMap;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.Map.Entry;
30 import java.util.concurrent.atomic.AtomicLong;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.DoNotRetryIOException;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.TableName;
42 import org.apache.hadoop.hbase.Waiter;
43 import org.apache.hadoop.hbase.client.Admin;
44 import org.apache.hadoop.hbase.client.Append;
45 import org.apache.hadoop.hbase.client.Connection;
46 import org.apache.hadoop.hbase.client.Delete;
47 import org.apache.hadoop.hbase.client.Increment;
48 import org.apache.hadoop.hbase.client.Mutation;
49 import org.apache.hadoop.hbase.client.Put;
50 import org.apache.hadoop.hbase.client.RegionServerCallable;
51 import org.apache.hadoop.hbase.client.Result;
52 import org.apache.hadoop.hbase.client.ResultScanner;
53 import org.apache.hadoop.hbase.client.RpcRetryingCallerFactory;
54 import org.apache.hadoop.hbase.client.Scan;
55 import org.apache.hadoop.hbase.client.Table;
56 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
57 import org.apache.hadoop.hbase.mapreduce.LoadIncrementalHFiles;
58 import org.apache.hadoop.hbase.master.HMaster;
59 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60 import org.apache.hadoop.hbase.quotas.policies.DefaultViolationPolicyEnforcement;
61 import org.apache.hadoop.hbase.regionserver.HRegionServer;
62 import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad;
63 import org.apache.hadoop.hbase.security.AccessDeniedException;
64 import org.apache.hadoop.hbase.testclassification.LargeTests;
65 import org.apache.hadoop.hbase.util.Bytes;
66 import org.apache.hadoop.hbase.util.Pair;
67 import org.apache.hadoop.util.StringUtils;
68 import org.junit.AfterClass;
69 import org.junit.Before;
70 import org.junit.BeforeClass;
71 import org.junit.Rule;
72 import org.junit.Test;
73 import org.junit.experimental.categories.Category;
74 import org.junit.rules.TestName;
75
76
77
78
79 @Category(LargeTests.class)
80 public class TestSpaceQuotas {
81 private static final Log LOG = LogFactory.getLog(TestSpaceQuotas.class);
82 private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
83 private static final AtomicLong COUNTER = new AtomicLong(0);
84 private static final int NUM_RETRIES = 10;
85
86 @Rule
87 public TestName testName = new TestName();
88 private SpaceQuotaHelperForTests helper;
89
90 @BeforeClass
91 public static void setUp() throws Exception {
92 Configuration conf = TEST_UTIL.getConfiguration();
93
94 conf.setInt("hbase.client.retries.number", 5);
95 conf.set(
96 CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
97 "org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
98
99 conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_DELAY_KEY, 1000);
100 conf.setInt(FileSystemUtilizationChore.FS_UTILIZATION_CHORE_PERIOD_KEY, 1000);
101 conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_DELAY_KEY, 1000);
102 conf.setInt(QuotaObserverChore.QUOTA_OBSERVER_CHORE_PERIOD_KEY, 1000);
103 conf.setInt(SpaceQuotaRefresherChore.POLICY_REFRESHER_CHORE_DELAY_KEY, 1000);
104 conf.setInt(SpaceQuotaRefresherChore.POLICY_REFRESHER_CHORE_PERIOD_KEY, 1000);
105 conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
106 TEST_UTIL.startMiniCluster(1);
107 }
108
109 @AfterClass
110 public static void tearDown() throws Exception {
111 TEST_UTIL.shutdownMiniCluster();
112 }
113
114 @Before
115 public void removeAllQuotas() throws Exception {
116 final Connection conn = TEST_UTIL.getConnection();
117 if (helper == null) {
118 helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
119 }
120
121 if (!conn.getAdmin().tableExists(QuotaUtil.QUOTA_TABLE_NAME)) {
122 helper.waitForQuotaTable(conn);
123 } else {
124
125 helper.removeAllQuotas(conn);
126 assertEquals(0, helper.listNumDefinedQuotas(conn));
127 }
128 }
129
130 @Test
131 public void testNoInsertsWithPut() throws Exception {
132 Put p = new Put(Bytes.toBytes("to_reject"));
133 p.addColumn(
134 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
135 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, p);
136 }
137
138 @Test
139 public void testNoInsertsWithAppend() throws Exception {
140 Append a = new Append(Bytes.toBytes("to_reject"));
141 a.add(
142 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
143 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, a);
144 }
145
146 @Test
147 public void testNoInsertsWithIncrement() throws Exception {
148 Increment i = new Increment(Bytes.toBytes("to_reject"));
149 i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
150 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_INSERTS, i);
151 }
152
153 @Test
154 public void testDeletesAfterNoInserts() throws Exception {
155 final TableName tn = writeUntilViolation(SpaceViolationPolicy.NO_INSERTS);
156
157
158 Delete d = new Delete(Bytes.toBytes("should_not_be_rejected"));
159 for (int i = 0; i < NUM_RETRIES; i++) {
160 try (Table t = TEST_UTIL.getConnection().getTable(tn)) {
161 t.delete(d);
162 }
163 }
164 }
165
166 @Test
167 public void testNoWritesWithPut() throws Exception {
168 Put p = new Put(Bytes.toBytes("to_reject"));
169 p.addColumn(
170 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
171 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
172 }
173
174 @Test
175 public void testNoWritesWithAppend() throws Exception {
176 Append a = new Append(Bytes.toBytes("to_reject"));
177 a.add(
178 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
179 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, a);
180 }
181
182 @Test
183 public void testNoWritesWithIncrement() throws Exception {
184 Increment i = new Increment(Bytes.toBytes("to_reject"));
185 i.addColumn(Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("count"), 0);
186 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, i);
187 }
188
189 @Test
190 public void testNoWritesWithDelete() throws Exception {
191 Delete d = new Delete(Bytes.toBytes("to_reject"));
192 writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, d);
193 }
194
195 @Test
196 public void testNoCompactions() throws Exception {
197 Put p = new Put(Bytes.toBytes("to_reject"));
198 p.addColumn(
199 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
200 final TableName tn = writeUntilViolationAndVerifyViolation(
201 SpaceViolationPolicy.NO_WRITES_COMPACTIONS, p);
202
203
204
205 try {
206 TEST_UTIL.getHBaseAdmin().majorCompact(tn);
207 fail("Expected that invoking the compaction should throw an Exception");
208 } catch (DoNotRetryIOException e) {
209
210 }
211
212 try {
213 TEST_UTIL.getHBaseAdmin().compact(tn);
214 fail("Expected that invoking the compaction should throw an Exception");
215 } catch (DoNotRetryIOException e) {
216
217 }
218 }
219
220 @Test
221 public void testNoEnableAfterDisablePolicy() throws Exception {
222 Put p = new Put(Bytes.toBytes("to_reject"));
223 p.addColumn(
224 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
225 final TableName tn = writeUntilViolation(SpaceViolationPolicy.DISABLE);
226 final Admin admin = TEST_UTIL.getHBaseAdmin();
227
228
229 for (int i = 0; i < NUM_RETRIES * 2; i++) {
230 if (admin.isTableEnabled(tn)) {
231 LOG.info(tn + " is still enabled, expecting it to be disabled. Will wait and re-check.");
232 Thread.sleep(2000);
233 }
234 }
235 assertFalse(tn + " is still enabled but it should be disabled", admin.isTableEnabled(tn));
236 try {
237 admin.enableTable(tn);
238 } catch (AccessDeniedException e) {
239 String exceptionContents = StringUtils.stringifyException(e);
240 final String expectedText = "violated space quota";
241 assertTrue("Expected the exception to contain " + expectedText + ", but was: "
242 + exceptionContents, exceptionContents.contains(expectedText));
243 }
244 }
245
246 @Test(timeout=120000)
247 public void testNoBulkLoadsWithNoWrites() throws Exception {
248 Put p = new Put(Bytes.toBytes("to_reject"));
249 p.addColumn(
250 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
251 TableName tableName = writeUntilViolationAndVerifyViolation(SpaceViolationPolicy.NO_WRITES, p);
252
253
254 FileSystem fs = TEST_UTIL.getTestFileSystem();
255 Path baseDir = new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files");
256 fs.mkdirs(baseDir);
257 RegionServerCallable<byte[]> callable = generateFileToLoad(tableName, 1, 50, baseDir);
258 try {
259 RpcRetryingCallerFactory.instantiate(TEST_UTIL.getConfiguration(), null).<byte[]> newCaller()
260 .callWithRetries(callable, Integer.MAX_VALUE);
261 fail("Expected the bulk load call to fail!");
262 } catch (SpaceLimitingException e) {
263
264 LOG.trace("Caught expected exception", e);
265 }
266 }
267
268 @Test(timeout=120000)
269 public void testAtomicBulkLoadUnderQuota() throws Exception {
270
271 TableName tn = helper.createTableWithRegions(10);
272
273 final long sizeLimit = 50L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
274 QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
275 TEST_UTIL.getHBaseAdmin().setQuota(settings);
276
277 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
278 RegionServerSpaceQuotaManager spaceQuotaManager = rs.getRegionServerSpaceQuotaManager();
279 Map<TableName,SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots();
280 Map<HRegionInfo,Long> regionSizes = getReportedSizesForTable(tn);
281 while (true) {
282 SpaceQuotaSnapshot snapshot = snapshots.get(tn);
283 if (snapshot != null && snapshot.getLimit() > 0) {
284 break;
285 }
286 LOG.debug("Snapshot does not yet realize quota limit: " + snapshots + ", regionsizes: " + regionSizes);
287 Thread.sleep(3000);
288 snapshots = spaceQuotaManager.copyQuotaSnapshots();
289 regionSizes = getReportedSizesForTable(tn);
290 }
291
292 SpaceQuotaSnapshot snapshot = snapshots.get(tn);
293 assertEquals(0L, snapshot.getUsage());
294 assertEquals(sizeLimit, snapshot.getLimit());
295
296
297 ActivePolicyEnforcement activePolicies = spaceQuotaManager.getActiveEnforcements();
298 SpaceViolationPolicyEnforcement enforcement = activePolicies.getPolicyEnforcement(tn);
299 assertTrue(
300 "Expected to find Noop policy, but got " + enforcement.getClass().getSimpleName(),
301 enforcement instanceof DefaultViolationPolicyEnforcement);
302
303
304 FileSystem fs = TEST_UTIL.getTestFileSystem();
305 Path baseDir = new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files");
306 fs.mkdirs(baseDir);
307 RegionServerCallable<byte[]> callable = generateFileToLoad(tn, 2, 500, baseDir);
308 FileStatus[] files = fs.listStatus(baseDir);
309 for (FileStatus file : files) {
310 assertTrue(
311 "Expected the file, " + file.getPath() + ", length to be larger than 25KB, but was "
312 + file.getLen(),
313 file.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE);
314 LOG.debug(file.getPath() + " -> " + file.getLen() +"B");
315 }
316
317 try {
318 RpcRetryingCallerFactory.instantiate(TEST_UTIL.getConfiguration(), null).<byte[]> newCaller()
319 .callWithRetries(callable, Integer.MAX_VALUE);
320 fail("Expected the bulk load call to fail!");
321 } catch (SpaceLimitingException e) {
322
323 LOG.trace("Caught expected exception", e);
324 }
325
326
327 Table table = TEST_UTIL.getConnection().getTable(tn);
328 ResultScanner scanner = table.getScanner(new Scan());
329 try {
330 assertNull("Expected no results", scanner.next());
331 } finally{
332 scanner.close();
333 }
334 }
335
336 @Test(timeout=120000)
337 public void testTableQuotaOverridesNamespaceQuota() throws Exception {
338 final SpaceViolationPolicy policy = SpaceViolationPolicy.NO_INSERTS;
339 final TableName tn = helper.createTableWithRegions(10);
340
341
342 final long tableLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
343 final long namespaceLimit = 1024L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
344 TEST_UTIL.getHBaseAdmin().setQuota(
345 QuotaSettingsFactory.limitTableSpace(tn, tableLimit, policy));
346 TEST_UTIL.getHBaseAdmin().setQuota(QuotaSettingsFactory.limitNamespaceSpace(
347 tn.getNamespaceAsString(), namespaceLimit, policy));
348
349
350 helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
351
352
353 Thread.sleep(5000);
354
355
356 Put p = new Put(Bytes.toBytes("to_reject"));
357 p.addColumn(
358 Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"), Bytes.toBytes("reject"));
359 verifyViolation(policy, tn, p);
360 }
361
362 @Test
363 public void testSecureBulkLoads() throws Exception {
364 final TableName tn = helper.createTableWithRegions(10);
365
366
367 final long sizeLimit = 1L * SpaceQuotaHelperForTests.ONE_KILOBYTE;
368 QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(
369 tn, sizeLimit, SpaceViolationPolicy.NO_INSERTS);
370 TEST_UTIL.getHBaseAdmin().setQuota(settings);
371
372
373 HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(0);
374 final RegionServerSpaceQuotaManager spaceQuotaManager = rs.getRegionServerSpaceQuotaManager();
375 TEST_UTIL.waitFor(60000, 3000, new Waiter.Predicate<Exception>() {
376 @Override
377 public boolean evaluate() throws Exception {
378 Map<TableName,SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots();
379 SpaceQuotaSnapshot snapshot = snapshots.get(tn);
380 LOG.debug("Snapshots: " + snapshots);
381 return null != snapshot && snapshot.getLimit() > 0;
382 }
383 });
384
385 Map<TableName,SpaceQuotaSnapshot> snapshots = spaceQuotaManager.copyQuotaSnapshots();
386 SpaceQuotaSnapshot snapshot = snapshots.get(tn);
387 assertEquals(0L, snapshot.getUsage());
388 assertEquals(sizeLimit, snapshot.getLimit());
389
390
391 FileSystem fs = TEST_UTIL.getTestFileSystem();
392 Path baseDir = new Path(fs.getHomeDirectory(), testName.getMethodName() + "_files");
393 fs.mkdirs(baseDir);
394 Path hfilesDir = new Path(baseDir, SpaceQuotaHelperForTests.F1);
395 fs.mkdirs(hfilesDir);
396 List<Pair<byte[], String>> filesToLoad = createFiles(tn, 1, 500, hfilesDir);
397
398 for (Pair<byte[], String> pair : filesToLoad) {
399 String file = pair.getSecond();
400 FileStatus[] statuses = fs.listStatus(new Path(file));
401 assertEquals(1, statuses.length);
402 FileStatus status = statuses[0];
403 assertTrue(
404 "Expected the file, " + file + ", length to be larger than 25KB, but was "
405 + status.getLen(),
406 status.getLen() > 25 * SpaceQuotaHelperForTests.ONE_KILOBYTE);
407 LOG.debug(file + " -> " + status.getLen() +"B");
408 }
409
410
411
412 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(TEST_UTIL.getConfiguration());
413 try {
414 loader.run(new String[] {new Path(
415 fs.getHomeDirectory(), testName.getMethodName() + "_files").toString(), tn.toString()});
416 fail("Expected the bulk load to be rejected, but it was not");
417 } catch (Exception e) {
418 LOG.debug("Caught expected exception", e);
419 String stringifiedException = StringUtils.stringifyException(e);
420 assertTrue(
421 "Expected exception message to contain the SpaceLimitingException class name: "
422 + stringifiedException,
423 stringifiedException.contains(SpaceLimitingException.class.getName()));
424 }
425 }
426
427 private Map<HRegionInfo,Long> getReportedSizesForTable(TableName tn) {
428 HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
429 MasterQuotaManager quotaManager = master.getMasterQuotaManager();
430 Map<HRegionInfo,Long> filteredRegionSizes = new HashMap<>();
431 for (Entry<HRegionInfo,Long> entry : quotaManager.snapshotRegionSizes().entrySet()) {
432 if (entry.getKey().getTable().equals(tn)) {
433 filteredRegionSizes.put(entry.getKey(), entry.getValue());
434 }
435 }
436 return filteredRegionSizes;
437 }
438
439 private TableName writeUntilViolation(SpaceViolationPolicy policyToViolate) throws Exception {
440 TableName tn = helper.createTableWithRegions(10);
441
442 final long sizeLimit = 2L * SpaceQuotaHelperForTests.ONE_MEGABYTE;
443 QuotaSettings settings = QuotaSettingsFactory.limitTableSpace(tn, sizeLimit, policyToViolate);
444 TEST_UTIL.getHBaseAdmin().setQuota(settings);
445
446
447 helper.writeData(tn, 3L * SpaceQuotaHelperForTests.ONE_MEGABYTE);
448
449
450 Thread.sleep(5000);
451
452 return tn;
453 }
454
455 private TableName writeUntilViolationAndVerifyViolation(SpaceViolationPolicy policyToViolate, Mutation m) throws Exception {
456 final TableName tn = writeUntilViolation(policyToViolate);
457 verifyViolation(policyToViolate, tn, m);
458 return tn;
459 }
460
461 private void verifyViolation(SpaceViolationPolicy policyToViolate, TableName tn, Mutation m) throws Exception {
462
463 boolean sawError = false;
464 for (int i = 0; i < NUM_RETRIES && !sawError; i++) {
465 try (Table table = TEST_UTIL.getConnection().getTable(tn)) {
466 if (m instanceof Put) {
467 table.put((Put) m);
468 } else if (m instanceof Delete) {
469 table.delete((Delete) m);
470 } else if (m instanceof Append) {
471 table.append((Append) m);
472 } else if (m instanceof Increment) {
473 table.increment((Increment) m);
474 } else {
475 fail("Failed to apply " + m.getClass().getSimpleName() + " to the table. Programming error");
476 }
477 LOG.info("Did not reject the " + m.getClass().getSimpleName() + ", will sleep and retry");
478 Thread.sleep(2000);
479 } catch (Exception e) {
480 String msg = StringUtils.stringifyException(e);
481 assertTrue("Expected exception message to contain the word '" + policyToViolate.name() + "', but was " + msg,
482 msg.contains(policyToViolate.name()));
483 sawError = true;
484 }
485 }
486 if (!sawError) {
487 try (Table quotaTable = TEST_UTIL.getConnection().getTable(QuotaUtil.QUOTA_TABLE_NAME)) {
488 ResultScanner scanner = quotaTable.getScanner(new Scan());
489 Result result = null;
490 LOG.info("Dumping contents of hbase:quota table");
491 while ((result = scanner.next()) != null) {
492 LOG.info(Bytes.toString(result.getRow()) + " => " + result.toString());
493 }
494 scanner.close();
495 }
496 }
497 assertTrue("Expected to see an exception writing data to a table exceeding its quota", sawError);
498 }
499
500 private List<Pair<byte[], String>> createFiles(
501 TableName tn, int numFiles, int numRowsPerFile, Path baseDir) throws Exception {
502 FileSystem fs = TEST_UTIL.getTestFileSystem();
503 fs.mkdirs(baseDir);
504 final List<Pair<byte[], String>> famPaths = new ArrayList<Pair<byte[], String>>();
505 for (int i = 1; i <= numFiles; i++) {
506 Path hfile = new Path(baseDir, "file" + i);
507 TestHRegionServerBulkLoad.createHFile(
508 fs, hfile, Bytes.toBytes(SpaceQuotaHelperForTests.F1), Bytes.toBytes("to"),
509 Bytes.toBytes("reject"), numRowsPerFile);
510 famPaths.add(new Pair<>(Bytes.toBytes(SpaceQuotaHelperForTests.F1), hfile.toString()));
511 }
512 return famPaths;
513 }
514
515 private RegionServerCallable<byte[]> generateFileToLoad(
516 TableName tn, int numFiles, int numRowsPerFile, Path baseDir) throws Exception {
517 final Connection conn = TEST_UTIL.getConnection();
518 final List<Pair<byte[], String>> famPaths = createFiles(tn, numFiles, numRowsPerFile, baseDir);
519
520 return new RegionServerCallable<byte[]>(conn, tn, Bytes.toBytes("row")) {
521 @Override
522 public byte[] call(int callTimeout) throws Exception {
523 byte[] regionName = getLocation().getRegionInfo().getRegionName();
524 boolean success = ProtobufUtil.bulkLoadHFile(getStub(), famPaths, regionName, true);
525 return success ? regionName : null;
526 }
527 };
528 }
529 }