View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase;
19  
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  
25  import java.util.concurrent.TimeUnit;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.CountingChore;
30  import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.DoNothingChore;
31  import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.FailInitialChore;
32  import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.SampleStopper;
33  import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.SleepingChore;
34  import org.apache.hadoop.hbase.TestChoreService.ScheduledChoreSamples.SlowChore;
35  import org.apache.hadoop.hbase.testclassification.SmallTests;
36  import org.junit.Test;
37  import org.junit.experimental.categories.Category;
38  
39  @Category(SmallTests.class)
40  public class TestChoreService {
41    private final Log LOG = LogFactory.getLog(this.getClass());
42  
43    /**
44     * A few ScheduledChore samples that are useful for testing with ChoreService
45     */
46    public static class ScheduledChoreSamples {
47      /**
48       * Straight forward stopper implementation that is used by default when one is not provided
49       */
50      public static class SampleStopper implements Stoppable {
51        private boolean stopped = false;
52  
53        @Override
54        public void stop(String why) {
55          stopped = true;
56        }
57  
58        @Override
59        public boolean isStopped() {
60          return stopped;
61        }
62      }
63  
64      /**
65       * Sleeps for longer than the scheduled period. This chore always misses its scheduled periodic
66       * executions
67       */
68      public static class SlowChore extends ScheduledChore {
69        public SlowChore(String name, int period) {
70          this(name, new SampleStopper(), period);
71        }
72  
73        public SlowChore(String name, Stoppable stopper, int period) {
74          super(name, stopper, period);
75        }
76  
77        @Override
78        protected boolean initialChore() {
79          try {
80            Thread.sleep(getPeriod() * 2);
81          } catch (InterruptedException e) {
82            e.printStackTrace();
83          }
84          return true;
85        }
86  
87        @Override
88        protected void chore() {
89          try {
90            Thread.sleep(getPeriod() * 2);
91          } catch (InterruptedException e) {
92            //e.printStackTrace();
93          }
94        }
95      }
96  
97      /**
98       * Lightweight ScheduledChore used primarily to fill the scheduling queue in tests
99       */
100     public static class DoNothingChore extends ScheduledChore {
101       public DoNothingChore(String name, int period) {
102         super(name, new SampleStopper(), period);
103       }
104 
105       public DoNothingChore(String name, Stoppable stopper, int period) {
106         super(name, stopper, period);
107       }
108 
109       @Override
110       protected void chore() {
111         // DO NOTHING
112       }
113 
114     }
115 
116     public static class SleepingChore extends ScheduledChore {
117       private int sleepTime;
118 
119       public SleepingChore(String name, int chorePeriod, int sleepTime) {
120         this(name, new SampleStopper(), chorePeriod, sleepTime);
121       }
122 
123       public SleepingChore(String name, Stoppable stopper, int period, int sleepTime) {
124         super(name, stopper, period);
125         this.sleepTime = sleepTime;
126       }
127 
128       @Override
129       protected boolean initialChore() {
130         try {
131           Thread.sleep(sleepTime);
132         } catch (InterruptedException e) {
133           e.printStackTrace();
134         }
135         return true;
136       }
137 
138       @Override
139       protected void chore() {
140         try {
141           Thread.sleep(sleepTime);
142         } catch (Exception e) {
143           System.err.println(e.getStackTrace());
144         }
145       }
146     }
147 
148     public static class CountingChore extends ScheduledChore {
149       private int countOfChoreCalls;
150       private boolean outputOnTicks = false;
151 
152       public CountingChore(String name, int period) {
153         this(name, new SampleStopper(), period);
154       }
155 
156       public CountingChore(String name, Stoppable stopper, int period) {
157         this(name, stopper, period, false);
158       }
159 
160       public CountingChore(String name, Stoppable stopper, int period,
161           final boolean outputOnTicks) {
162         super(name, stopper, period);
163         this.countOfChoreCalls = 0;
164         this.outputOnTicks = outputOnTicks;
165       }
166 
167       @Override
168       protected boolean initialChore() {
169         countOfChoreCalls++;
170         if (outputOnTicks) outputTickCount();
171         return true;
172       }
173 
174       @Override
175       protected void chore() {
176         countOfChoreCalls++;
177         if (outputOnTicks) outputTickCount();
178       }
179 
180       private void outputTickCount() {
181         System.out.println("Chore: " + getName() + ". Count of chore calls: " + countOfChoreCalls);
182       }
183 
184       public int getCountOfChoreCalls() {
185         return countOfChoreCalls;
186       }
187 
188       public boolean isOutputtingOnTicks() {
189         return outputOnTicks;
190       }
191 
192       public void setOutputOnTicks(boolean o) {
193         outputOnTicks = o;
194       }
195     }
196 
197     /**
198      * A Chore that will try to execute the initial chore a few times before succeeding. Once the
199      * initial chore is complete the chore cancels itself
200      */
201     public static class FailInitialChore extends ScheduledChore {
202       private int numberOfFailures;
203       private int failureThreshold;
204 
205       /**
206        * @param failThreshold Number of times the Chore fails when trying to execute initialChore
207        *          before succeeding.
208        */
209       public FailInitialChore(String name, int period, int failThreshold) {
210         this(name, new SampleStopper(), period, failThreshold);
211       }
212 
213       public FailInitialChore(String name, Stoppable stopper, int period, int failThreshold) {
214         super(name, stopper, period);
215         numberOfFailures = 0;
216         failureThreshold = failThreshold;
217       }
218 
219       @Override
220       protected boolean initialChore() {
221         if (numberOfFailures < failureThreshold) {
222           numberOfFailures++;
223           return false;
224         } else {
225           return true;
226         }
227       }
228 
229       @Override
230       protected void chore() {
231         assertTrue(numberOfFailures == failureThreshold);
232         cancel(false);
233       }
234 
235     }
236   }
237 
238   @Test (timeout=20000)
239   public void testInitialChorePrecedence() throws InterruptedException {
240     ChoreService service = ChoreService.getInstance("testInitialChorePrecedence");
241 
242     final int period = 100;
243     final int failureThreshold = 5;
244 
245     try {
246       ScheduledChore chore = new FailInitialChore("chore", period, failureThreshold);
247       service.scheduleChore(chore);
248 
249       int loopCount = 0;
250       boolean brokeOutOfLoop = false;
251 
252      while (!chore.isInitialChoreComplete() && chore.isScheduled()) {
253        Thread.sleep(failureThreshold * period);
254        loopCount++;
255        if (loopCount > 3) {
256          brokeOutOfLoop = true;
257          break;
258        }
259     }
260 
261     assertFalse(brokeOutOfLoop);
262     } finally {
263       shutdownService(service);
264     }
265   }
266 
267   @Test (timeout=20000)
268   public void testCancelChore() throws InterruptedException {
269     final int period = 100;
270     ScheduledChore chore1 = new DoNothingChore("chore1", period);
271     ChoreService service = ChoreService.getInstance("testCancelChore");
272     try {
273       service.scheduleChore(chore1);
274       assertTrue(chore1.isScheduled());
275 
276       chore1.cancel(true);
277       assertFalse(chore1.isScheduled());
278       assertTrue(service.getNumberOfScheduledChores() == 0);
279     } finally {
280       shutdownService(service);
281     }
282   }
283 
284   @Test (timeout=20000)
285   public void testScheduledChoreConstruction() {
286     final String NAME = "chore";
287     final int PERIOD = 100;
288     final long VALID_DELAY = 0;
289     final long INVALID_DELAY = -100;
290     final TimeUnit UNIT = TimeUnit.NANOSECONDS;
291 
292     ScheduledChore chore1 =
293         new ScheduledChore(NAME, new SampleStopper(), PERIOD, VALID_DELAY, UNIT) {
294       @Override
295       protected void chore() {
296         // DO NOTHING
297       }
298     };
299 
300     assertEquals("Name construction failed", chore1.getName(), NAME);
301     assertEquals("Period construction failed", chore1.getPeriod(), PERIOD);
302     assertEquals("Initial Delay construction failed", chore1.getInitialDelay(), VALID_DELAY);
303     assertEquals("TimeUnit construction failed", chore1.getTimeUnit(), UNIT);
304 
305     ScheduledChore invalidDelayChore =
306         new ScheduledChore(NAME, new SampleStopper(), PERIOD, INVALID_DELAY, UNIT) {
307       @Override
308       protected void chore() {
309         // DO NOTHING
310       }
311     };
312 
313     assertEquals("Initial Delay should be set to 0 when invalid", 0,
314       invalidDelayChore.getInitialDelay());
315   }
316 
317   @Test (timeout=20000)
318   public void testChoreServiceConstruction() throws InterruptedException {
319     final int corePoolSize = 10;
320     final int defaultCorePoolSize = ChoreService.MIN_CORE_POOL_SIZE;
321 
322     ChoreService customInit = new ChoreService("testChoreServiceConstruction_custom", corePoolSize);
323     try {
324       assertEquals(corePoolSize, customInit.getCorePoolSize());
325     } finally {
326       shutdownService(customInit);
327     }
328 
329     ChoreService defaultInit = new ChoreService("testChoreServiceConstruction_default");
330     try {
331       assertEquals(defaultCorePoolSize, defaultInit.getCorePoolSize());
332     } finally {
333       shutdownService(defaultInit);
334     }
335 
336     ChoreService invalidInit = new ChoreService("testChoreServiceConstruction_invalid", -10);
337     try {
338       assertEquals(defaultCorePoolSize, invalidInit.getCorePoolSize());
339     } finally {
340     shutdownService(invalidInit);
341     }
342   }
343 
344   @Test (timeout=20000)
345   public void testFrequencyOfChores() throws InterruptedException {
346     final int period = 100;
347     // Small delta that acts as time buffer (allowing chores to complete if running slowly)
348     final int delta = 5;
349     ChoreService service = ChoreService.getInstance("testFrequencyOfChores");
350     CountingChore chore = new CountingChore("countingChore", period);
351     try {
352       service.scheduleChore(chore);
353 
354       Thread.sleep(10 * period + delta);
355       assertTrue(chore.getCountOfChoreCalls() == 11);
356 
357       Thread.sleep(10 * period);
358       assertTrue(chore.getCountOfChoreCalls() == 21);
359     } finally {
360       shutdownService(service);
361     }
362   }
363 
364   public void shutdownService(ChoreService service) throws InterruptedException {
365     service.shutdown();
366     while (!service.isTerminated()) {
367       Thread.sleep(100);
368     }
369   }
370 
371   @Test (timeout=20000)
372   public void testForceTrigger() throws InterruptedException {
373     final int period = 100;
374     final int delta = 5;
375     ChoreService service = ChoreService.getInstance("testForceTrigger");
376     final CountingChore chore = new CountingChore("countingChore", period);
377     try {
378       service.scheduleChore(chore);
379       Thread.sleep(10 * period + delta);
380 
381       assertTrue(chore.getCountOfChoreCalls() == 11);
382 
383       // Force five runs of the chore to occur, sleeping between triggers to ensure the
384       // chore has time to run
385       chore.triggerNow();
386       Thread.sleep(delta);
387       chore.triggerNow();
388       Thread.sleep(delta);
389       chore.triggerNow();
390       Thread.sleep(delta);
391       chore.triggerNow();
392       Thread.sleep(delta);
393       chore.triggerNow();
394       Thread.sleep(delta);
395 
396       assertTrue("" + chore.getCountOfChoreCalls(), chore.getCountOfChoreCalls() == 16);
397 
398       Thread.sleep(10 * period + delta);
399 
400       // Be loosey-goosey. It used to be '26' but it was a big flakey relying on timing.
401       assertTrue("" + chore.getCountOfChoreCalls(), chore.getCountOfChoreCalls() > 16);
402     } finally {
403       shutdownService(service);
404     }
405   }
406 
407   @Test (timeout=20000)
408   public void testCorePoolIncrease() throws InterruptedException {
409     final int initialCorePoolSize = 3;
410     ChoreService service = new ChoreService("testCorePoolIncrease", initialCorePoolSize);
411 
412     try {
413       assertEquals("Should have a core pool of size: " + initialCorePoolSize, initialCorePoolSize,
414         service.getCorePoolSize());
415 
416       final int slowChorePeriod = 100;
417       SlowChore slowChore1 = new SlowChore("slowChore1", slowChorePeriod);
418       SlowChore slowChore2 = new SlowChore("slowChore2", slowChorePeriod);
419       SlowChore slowChore3 = new SlowChore("slowChore3", slowChorePeriod);
420 
421       service.scheduleChore(slowChore1);
422       service.scheduleChore(slowChore2);
423       service.scheduleChore(slowChore3);
424 
425       Thread.sleep(slowChorePeriod * 10);
426       assertEquals("Should not create more pools than scheduled chores", 3,
427         service.getCorePoolSize());
428 
429       SlowChore slowChore4 = new SlowChore("slowChore4", slowChorePeriod);
430       service.scheduleChore(slowChore4);
431 
432       Thread.sleep(slowChorePeriod * 10);
433       assertEquals("Chores are missing their start time. Should expand core pool size", 4,
434         service.getCorePoolSize());
435 
436       SlowChore slowChore5 = new SlowChore("slowChore5", slowChorePeriod);
437       service.scheduleChore(slowChore5);
438 
439       Thread.sleep(slowChorePeriod * 10);
440       assertEquals("Chores are missing their start time. Should expand core pool size", 5,
441         service.getCorePoolSize());
442     } finally {
443       shutdownService(service);
444     }
445   }
446 
447   @Test(timeout = 30000)
448   public void testCorePoolDecrease() throws InterruptedException {
449     final int initialCorePoolSize = 3;
450     ChoreService service = new ChoreService("testCorePoolDecrease", initialCorePoolSize);
451     final int chorePeriod = 100;
452     try {
453       // Slow chores always miss their start time and thus the core pool size should be at least as
454       // large as the number of running slow chores
455       SlowChore slowChore1 = new SlowChore("slowChore1", chorePeriod);
456       SlowChore slowChore2 = new SlowChore("slowChore2", chorePeriod);
457       SlowChore slowChore3 = new SlowChore("slowChore3", chorePeriod);
458 
459       service.scheduleChore(slowChore1);
460       service.scheduleChore(slowChore2);
461       service.scheduleChore(slowChore3);
462 
463       Thread.sleep(chorePeriod * 10);
464       assertEquals("Should not create more pools than scheduled chores",
465         service.getNumberOfScheduledChores(), service.getCorePoolSize());
466 
467       SlowChore slowChore4 = new SlowChore("slowChore4", chorePeriod);
468       service.scheduleChore(slowChore4);
469       Thread.sleep(chorePeriod * 10);
470       assertEquals("Chores are missing their start time. Should expand core pool size",
471         service.getNumberOfScheduledChores(), service.getCorePoolSize());
472 
473       SlowChore slowChore5 = new SlowChore("slowChore5", chorePeriod);
474       service.scheduleChore(slowChore5);
475       Thread.sleep(chorePeriod * 10);
476       assertEquals("Chores are missing their start time. Should expand core pool size",
477         service.getNumberOfScheduledChores(), service.getCorePoolSize());
478       assertEquals(service.getNumberOfChoresMissingStartTime(), 5);
479 
480       // Now we begin to cancel the chores that caused an increase in the core thread pool of the
481       // ChoreService. These cancellations should cause a decrease in the core thread pool.
482       slowChore5.cancel();
483       Thread.sleep(chorePeriod * 10);
484       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
485         service.getCorePoolSize());
486       assertEquals(service.getNumberOfChoresMissingStartTime(), 4);
487 
488       slowChore4.cancel();
489       Thread.sleep(chorePeriod * 10);
490       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
491         service.getCorePoolSize());
492       assertEquals(service.getNumberOfChoresMissingStartTime(), 3);
493 
494       slowChore3.cancel();
495       Thread.sleep(chorePeriod * 10);
496       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
497         service.getCorePoolSize());
498       assertEquals(service.getNumberOfChoresMissingStartTime(), 2);
499 
500       slowChore2.cancel();
501       Thread.sleep(chorePeriod * 10);
502       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
503         service.getCorePoolSize());
504       assertEquals(service.getNumberOfChoresMissingStartTime(), 1);
505 
506       slowChore1.cancel();
507       Thread.sleep(chorePeriod * 10);
508       assertEquals(Math.max(ChoreService.MIN_CORE_POOL_SIZE, service.getNumberOfScheduledChores()),
509         service.getCorePoolSize());
510       assertEquals(service.getNumberOfChoresMissingStartTime(), 0);
511     } finally {
512       shutdownService(service);
513     }
514   }
515 
516   @Test (timeout=20000)
517   public void testNumberOfRunningChores() throws InterruptedException {
518     ChoreService service = new ChoreService("testNumberOfRunningChores");
519 
520     final int period = 100;
521     final int sleepTime = 5;
522 
523     try {
524       DoNothingChore dn1 = new DoNothingChore("dn1", period);
525       DoNothingChore dn2 = new DoNothingChore("dn2", period);
526       DoNothingChore dn3 = new DoNothingChore("dn3", period);
527       DoNothingChore dn4 = new DoNothingChore("dn4", period);
528       DoNothingChore dn5 = new DoNothingChore("dn5", period);
529 
530       service.scheduleChore(dn1);
531       service.scheduleChore(dn2);
532       service.scheduleChore(dn3);
533       service.scheduleChore(dn4);
534       service.scheduleChore(dn5);
535 
536       Thread.sleep(sleepTime);
537       assertEquals("Scheduled chore mismatch", 5, service.getNumberOfScheduledChores());
538 
539       dn1.cancel();
540       Thread.sleep(sleepTime);
541       assertEquals("Scheduled chore mismatch", 4, service.getNumberOfScheduledChores());
542 
543       dn2.cancel();
544       dn3.cancel();
545       dn4.cancel();
546       Thread.sleep(sleepTime);
547       assertEquals("Scheduled chore mismatch", 1, service.getNumberOfScheduledChores());
548 
549       dn5.cancel();
550       Thread.sleep(sleepTime);
551       assertEquals("Scheduled chore mismatch", 0, service.getNumberOfScheduledChores());
552     } finally {
553       shutdownService(service);
554     }
555   }
556 
557   @Test (timeout=20000)
558   public void testNumberOfChoresMissingStartTime() throws InterruptedException {
559     ChoreService service = new ChoreService("testNumberOfChoresMissingStartTime");
560 
561     final int period = 100;
562     final int sleepTime = 5 * period;
563 
564     try {
565       // Slow chores sleep for a length of time LONGER than their period. Thus, SlowChores
566       // ALWAYS miss their start time since their execution takes longer than their period
567       SlowChore sc1 = new SlowChore("sc1", period);
568       SlowChore sc2 = new SlowChore("sc2", period);
569       SlowChore sc3 = new SlowChore("sc3", period);
570       SlowChore sc4 = new SlowChore("sc4", period);
571       SlowChore sc5 = new SlowChore("sc5", period);
572 
573       service.scheduleChore(sc1);
574       service.scheduleChore(sc2);
575       service.scheduleChore(sc3);
576       service.scheduleChore(sc4);
577       service.scheduleChore(sc5);
578 
579       Thread.sleep(sleepTime);
580       assertEquals(5, service.getNumberOfChoresMissingStartTime());
581 
582       sc1.cancel();
583       Thread.sleep(sleepTime);
584       assertEquals(4, service.getNumberOfChoresMissingStartTime());
585 
586       sc2.cancel();
587       sc3.cancel();
588       sc4.cancel();
589       Thread.sleep(sleepTime);
590       assertEquals(1, service.getNumberOfChoresMissingStartTime());
591 
592       sc5.cancel();
593       Thread.sleep(sleepTime);
594       assertEquals(0, service.getNumberOfChoresMissingStartTime());
595     } finally {
596       shutdownService(service);
597     }
598   }
599 
600   /**
601    * ChoreServices should never have a core pool size that exceeds the number of chores that have
602    * been scheduled with the service. For example, if 4 ScheduledChores are scheduled with a
603    * ChoreService, the number of threads in the ChoreService's core pool should never exceed 4
604    */
605   @Test (timeout=20000)
606   public void testMaximumChoreServiceThreads() throws InterruptedException {
607     ChoreService service = new ChoreService("testMaximumChoreServiceThreads");
608 
609     final int period = 100;
610     final int sleepTime = 5 * period;
611 
612     try {
613       // Slow chores sleep for a length of time LONGER than their period. Thus, SlowChores
614       // ALWAYS miss their start time since their execution takes longer than their period.
615       // Chores that miss their start time will trigger the onChoreMissedStartTime callback
616       // in the ChoreService. This callback will try to increase the number of core pool
617       // threads.
618       SlowChore sc1 = new SlowChore("sc1", period);
619       SlowChore sc2 = new SlowChore("sc2", period);
620       SlowChore sc3 = new SlowChore("sc3", period);
621       SlowChore sc4 = new SlowChore("sc4", period);
622       SlowChore sc5 = new SlowChore("sc5", period);
623 
624       service.scheduleChore(sc1);
625       service.scheduleChore(sc2);
626       service.scheduleChore(sc3);
627       service.scheduleChore(sc4);
628       service.scheduleChore(sc5);
629 
630       Thread.sleep(sleepTime);
631       assertTrue(service.getCorePoolSize() <= service.getNumberOfScheduledChores());
632 
633       SlowChore sc6 = new SlowChore("sc6", period);
634       SlowChore sc7 = new SlowChore("sc7", period);
635       SlowChore sc8 = new SlowChore("sc8", period);
636       SlowChore sc9 = new SlowChore("sc9", period);
637       SlowChore sc10 = new SlowChore("sc10", period);
638 
639       service.scheduleChore(sc6);
640       service.scheduleChore(sc7);
641       service.scheduleChore(sc8);
642       service.scheduleChore(sc9);
643       service.scheduleChore(sc10);
644 
645       Thread.sleep(sleepTime);
646       assertTrue(service.getCorePoolSize() <= service.getNumberOfScheduledChores());
647     } finally {
648       shutdownService(service);
649     }
650   }
651 
652   @Test (timeout=20000)
653   public void testChangingChoreServices() throws InterruptedException {
654     final int period = 100;
655     final int sleepTime = 10;
656     ChoreService service1 = new ChoreService("testChangingChoreServices_1");
657     ChoreService service2 = new ChoreService("testChangingChoreServices_2");
658     ScheduledChore chore = new DoNothingChore("sample", period);
659 
660     try {
661       assertFalse(chore.isScheduled());
662       assertFalse(service1.isChoreScheduled(chore));
663       assertFalse(service2.isChoreScheduled(chore));
664       assertTrue(chore.getChoreServicer() == null);
665 
666       service1.scheduleChore(chore);
667       Thread.sleep(sleepTime);
668       assertTrue(chore.isScheduled());
669       assertTrue(service1.isChoreScheduled(chore));
670       assertFalse(service2.isChoreScheduled(chore));
671       assertFalse(chore.getChoreServicer() == null);
672 
673       service2.scheduleChore(chore);
674       Thread.sleep(sleepTime);
675       assertTrue(chore.isScheduled());
676       assertFalse(service1.isChoreScheduled(chore));
677       assertTrue(service2.isChoreScheduled(chore));
678       assertFalse(chore.getChoreServicer() == null);
679 
680       chore.cancel();
681       assertFalse(chore.isScheduled());
682       assertFalse(service1.isChoreScheduled(chore));
683       assertFalse(service2.isChoreScheduled(chore));
684       assertTrue(chore.getChoreServicer() == null);
685     } finally {
686       shutdownService(service1);
687       shutdownService(service2);
688     }
689   }
690 
691   @Test (timeout=20000)
692   public void testTriggerNowFailsWhenNotScheduled() throws InterruptedException {
693     final int period = 100;
694     // Small sleep time buffer to allow CountingChore to complete
695     final int sleep = 5;
696     ChoreService service = new ChoreService("testTriggerNowFailsWhenNotScheduled");
697     CountingChore chore = new CountingChore("dn", period);
698 
699     try {
700       assertFalse(chore.triggerNow());
701       assertTrue(chore.getCountOfChoreCalls() == 0);
702 
703       service.scheduleChore(chore);
704       Thread.sleep(sleep);
705       assertEquals(1, chore.getCountOfChoreCalls());
706       Thread.sleep(period);
707       assertEquals(2, chore.getCountOfChoreCalls());
708       assertTrue(chore.triggerNow());
709       Thread.sleep(sleep);
710       assertTrue(chore.triggerNow());
711       Thread.sleep(sleep);
712       assertTrue(chore.triggerNow());
713       Thread.sleep(sleep);
714       assertEquals(5, chore.getCountOfChoreCalls());
715     } finally {
716       shutdownService(service);
717     }
718   }
719 
720   @Test (timeout=20000)
721   public void testStopperForScheduledChores() throws InterruptedException {
722     ChoreService service = ChoreService.getInstance("testStopperForScheduledChores");
723     Stoppable stopperForGroup1 = new SampleStopper();
724     Stoppable stopperForGroup2 = new SampleStopper();
725     final int period = 100;
726     final int delta = 10;
727 
728     try {
729       ScheduledChore chore1_group1 = new DoNothingChore("c1g1", stopperForGroup1, period);
730       ScheduledChore chore2_group1 = new DoNothingChore("c2g1", stopperForGroup1, period);
731       ScheduledChore chore3_group1 = new DoNothingChore("c3g1", stopperForGroup1, period);
732 
733       ScheduledChore chore1_group2 = new DoNothingChore("c1g2", stopperForGroup2, period);
734       ScheduledChore chore2_group2 = new DoNothingChore("c2g2", stopperForGroup2, period);
735       ScheduledChore chore3_group2 = new DoNothingChore("c3g2", stopperForGroup2, period);
736 
737       service.scheduleChore(chore1_group1);
738       service.scheduleChore(chore2_group1);
739       service.scheduleChore(chore3_group1);
740       service.scheduleChore(chore1_group2);
741       service.scheduleChore(chore2_group2);
742       service.scheduleChore(chore3_group2);
743 
744       Thread.sleep(delta);
745       Thread.sleep(10 * period);
746       assertTrue(chore1_group1.isScheduled());
747       assertTrue(chore2_group1.isScheduled());
748       assertTrue(chore3_group1.isScheduled());
749       assertTrue(chore1_group2.isScheduled());
750       assertTrue(chore2_group2.isScheduled());
751       assertTrue(chore3_group2.isScheduled());
752 
753       stopperForGroup1.stop("test stopping group 1");
754       Thread.sleep(period);
755       assertFalse(chore1_group1.isScheduled());
756       assertFalse(chore2_group1.isScheduled());
757       assertFalse(chore3_group1.isScheduled());
758       assertTrue(chore1_group2.isScheduled());
759       assertTrue(chore2_group2.isScheduled());
760       assertTrue(chore3_group2.isScheduled());
761 
762       stopperForGroup2.stop("test stopping group 2");
763       Thread.sleep(period);
764       assertFalse(chore1_group1.isScheduled());
765       assertFalse(chore2_group1.isScheduled());
766       assertFalse(chore3_group1.isScheduled());
767       assertFalse(chore1_group2.isScheduled());
768       assertFalse(chore2_group2.isScheduled());
769       assertFalse(chore3_group2.isScheduled());
770     } finally {
771       shutdownService(service);
772     }
773   }
774 
775   @Test (timeout=20000)
776   public void testShutdownCancelsScheduledChores() throws InterruptedException {
777     final int period = 100;
778     ChoreService service = new ChoreService("testShutdownCancelsScheduledChores");
779     ScheduledChore successChore1 = new DoNothingChore("sc1", period);
780     ScheduledChore successChore2 = new DoNothingChore("sc2", period);
781     ScheduledChore successChore3 = new DoNothingChore("sc3", period);
782 
783     try {
784       assertTrue(service.scheduleChore(successChore1));
785       assertTrue(successChore1.isScheduled());
786       assertTrue(service.scheduleChore(successChore2));
787       assertTrue(successChore2.isScheduled());
788       assertTrue(service.scheduleChore(successChore3));
789       assertTrue(successChore3.isScheduled());
790     } finally {
791       shutdownService(service);
792     }
793 
794     assertFalse(successChore1.isScheduled());
795     assertFalse(successChore2.isScheduled());
796     assertFalse(successChore3.isScheduled());
797   }
798 
799   @Test (timeout=20000)
800   public void testShutdownWorksWhileChoresAreExecuting() throws InterruptedException {
801     final int period = 100;
802     final int sleep = 5 * period;
803     ChoreService service = new ChoreService("testShutdownWorksWhileChoresAreExecuting");
804     ScheduledChore slowChore1 = new SleepingChore("sc1", period, sleep);
805     ScheduledChore slowChore2 = new SleepingChore("sc2", period, sleep);
806     ScheduledChore slowChore3 = new SleepingChore("sc3", period, sleep);
807     try {
808       assertTrue(service.scheduleChore(slowChore1));
809       assertTrue(service.scheduleChore(slowChore2));
810       assertTrue(service.scheduleChore(slowChore3));
811 
812       Thread.sleep(sleep / 2);
813       shutdownService(service);
814 
815       assertFalse(slowChore1.isScheduled());
816       assertFalse(slowChore2.isScheduled());
817       assertFalse(slowChore3.isScheduled());
818       assertTrue(service.isShutdown());
819 
820       Thread.sleep(5);
821       assertTrue(service.isTerminated());
822     } finally {
823       shutdownService(service);
824     }
825   }
826 
827   @Test (timeout=20000)
828   public void testShutdownRejectsNewSchedules() throws InterruptedException {
829     final int period = 100;
830     ChoreService service = new ChoreService("testShutdownRejectsNewSchedules");
831     ScheduledChore successChore1 = new DoNothingChore("sc1", period);
832     ScheduledChore successChore2 = new DoNothingChore("sc2", period);
833     ScheduledChore successChore3 = new DoNothingChore("sc3", period);
834     ScheduledChore failChore1 = new DoNothingChore("fc1", period);
835     ScheduledChore failChore2 = new DoNothingChore("fc2", period);
836     ScheduledChore failChore3 = new DoNothingChore("fc3", period);
837 
838     try {
839       assertTrue(service.scheduleChore(successChore1));
840       assertTrue(successChore1.isScheduled());
841       assertTrue(service.scheduleChore(successChore2));
842       assertTrue(successChore2.isScheduled());
843       assertTrue(service.scheduleChore(successChore3));
844       assertTrue(successChore3.isScheduled());
845     } finally {
846       shutdownService(service);
847     }
848 
849     assertFalse(service.scheduleChore(failChore1));
850     assertFalse(failChore1.isScheduled());
851     assertFalse(service.scheduleChore(failChore2));
852     assertFalse(failChore2.isScheduled());
853     assertFalse(service.scheduleChore(failChore3));
854     assertFalse(failChore3.isScheduled());
855   }
856 }