001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.juneau.petstore.console.ops;
018
019import static org.apache.juneau.commons.utils.Shorts.*;
020
021import java.io.*;
022import java.time.*;
023import java.util.*;
024import java.util.concurrent.*;
025import java.util.concurrent.atomic.*;
026import java.util.function.*;
027
028import org.apache.juneau.commons.inject.*;
029import org.apache.juneau.http.*;
030import org.apache.juneau.http.response.*;
031import org.apache.juneau.marshall.collections.*;
032import org.apache.juneau.petstore.console.*;
033import org.apache.juneau.petstore.console.data.*;
034import org.apache.juneau.petstore.console.pages.*;
035import org.apache.juneau.petstore.dto.*;
036import org.apache.juneau.rest.server.*;
037import org.apache.juneau.rest.server.view.*;
038import org.apache.juneau.rest.server.view.freemarker.*;
039import org.apache.juneau.console.views.*;
040import org.apache.juneau.terminal.region.*;
041import org.apache.juneau.console.widgets.*;
042
043/**
044 * Jobs: the console's async-job reference.  A restock submits once per idempotency key, answers
045 * 202 with an {@link AsyncJobRef}, and the views runtime follows its {@code streamUrl} until the job settles.
046 *
047 * <p>
048 * The registry de-duplicates on the idempotency key itself ({@link AsyncJobRegistry#tryCreate(IdempotencyKey)}), so a
049 * resubmit with the same key returns the same job and this resource keeps no key map of its own.
050 *
051 * <p>
052 * The page's {@code job-log} card posts the same restock in its flat form and follows the job's stream.  A restock
053 * with {@code slowMs} waits that long before it touches the store (cancel it then and nothing changes), and every
054 * restock holds for {@code holdMs} (a second by default) after it has added its pets (cancel it then and the result
055 * says the change was made).
056 * The registry's hard timeout is 30 s, or whatever a {@link HardTimeout} bean on the host says; a {@link GroomPeriod}
057 * bean sets the groom step and a {@link SpeciesHold} bean the species-row hold.
058 *
059 * <p>
060 * The seeded groom run is created on the first groom request rather than at construction, so a {@link GroomPeriod} bean
061 * (injected after construction) can set its pace; every groom endpoint goes through it, so the page sees the same data.
062 *
063 * <p>
064 * The page also hosts the terminal demo: a "groom pet" job ({@link GroomRuns}) whose output is served by
065 * {@link TerminalMixin} and shown two ways &mdash; a FreeMarker terminal card over the in-memory output, and a second
066 * card over the same run's file-backed transcript.
067 *
068 * <h2 class='section'>Example:</h2>
069 * <p class='bjava'>
070 *    <jc>// POST /console/ops/jobs/restock {DOG:2, slowMs:3000, holdMs:1000, idempotencyKey:'7f3c...'}</jc>
071 *    <jk>var</jk> <jv>body</jv> = Restock.<jsm>of</jsm>(<jv>json</jv>);
072 *    <jk>var</jk> <jv>job</jv> = asyncJobRegistry().tryCreate(IdempotencyKey.<jsm>of</jsm>(<jv>body</jv>.idempotencyKey(), <js>"restock"</js>, <js>"restock"</js>)).orElse(<jk>null</jk>);
073 *    <jk>if</jk> (<jv>job</jv> == <jk>null</jk>)
074 *       <jk>return</jk> ActionResult.<jsm>refusal</jsm>(<js>"app:job-limit"</js>);  <jc>// answered 429</jc>
075 *    <jv>pool</jv>.execute(() -&gt; run(<jv>job</jv>, <jv>body</jv>, () -&gt; store().restock(<jv>body</jv>.perSpecies(), <js>"job:restock"</js>)));
076 *    <jk>return</jk> AsyncJobRef.<jsm>of</jsm>(<jv>job</jv>);  <jc>// answered 202</jc>
077 * </p>
078 */
079@Rest(path="/jobs", title="Jobs")
080@SuppressWarnings({
081   "java:S110", // Inheritance depth comes from the BasicRestServlet hierarchy, not this demo page.
082   "java:S1192", // Duplicated literals are wire keys of the job rows; constants would obscure the schema.
083   "java:S2654", // The demo intentionally runs restocks on its own pool and serializes the idempotency check.
084   "resource" // The registry and pool are owned by this resource and closed in closeJobs(); Eclipse JDT @Owning warning is by design.
085})
086public class JobsRest extends PetstoreConsolePage implements AsyncJobsMixin, TerminalMixin {
087
088   private static final long serialVersionUID = 1L;
089
090   /** System property: the most restocks that may run at once (default 2; the registry's own cap is 8). */
091   public static final String LIMIT_PROPERTY = "petstore.jobs.limit";
092
093   /**
094    * System property: how long a species row's restock holds after its effect, in milliseconds (default {@value #DEFAULT_HOLD_MS}, at most {@value #MAX_HOLD_MS}).
095    *
096    * <p>
097    * Read on every request, so a test can widen the window in which a cancel lands as {@code cancelled-after-effect}.
098    * A sample-only knob: the row form never accepts a hold.
099    */
100   public static final String SPECIES_HOLD_PROPERTY = "petstore.jobs.speciesHoldMs";
101
102   private static final String ACTION = "restock";
103
104   /** Where this resource is mounted; the terminal URLs are built from it. */
105   public static final String MOUNT = "/console/ops/jobs";
106
107   private static final String GROOM_JOB = "groom";
108
109   /** The species row's restock action; its keys come from the ledger, bound to the species and the caller. */
110   static final String RESTOCK_SPECIES = "restock-species";
111
112   /** The groom run row's "groom again" action. */
113   static final String REGROOM = "regroom";
114
115   /** Row keys: issued by the {@code /key} endpoints, checked and claimed by the row submits. */
116   private final transient JobKeyLedger jobKeys = JobKeyLedger.create();
117
118   /** When each species was last restocked from its row, for the row's detail panel. */
119   private final transient Map<Species,Instant> lastRestock = new ConcurrentHashMap<>();
120
121   /** The groom-pet demo runs, created on first use; one verbose run is seeded so the page always has a log. */
122   private final transient AtomicReference<GroomRuns> groomRef = new AtomicReference<>();
123
124   /** The default hard timeout of a restock job. */
125   public static final Duration DEFAULT_HARD_TIMEOUT = Duration.ofSeconds(30);
126
127   /** The longest {@code slowMs} a restock accepts. */
128   public static final int MAX_SLOW_MS = 10_000;
129
130   /** The longest {@code holdMs} a restock accepts. */
131   public static final int MAX_HOLD_MS = 10_000;
132
133   /** How long a restock holds after its effect by default, so a late cancel lands as {@code cancelled-after-effect}. */
134   public static final int DEFAULT_HOLD_MS = 1000;
135
136   private static final long SLOW_STEP_MS = 100;
137
138   /**
139    * A per-host override of the jobs hard timeout.  Declare it as a {@code @Bean} on the host that mounts the
140    * console; it is per instance, so one test host's short timeout does not leak into another.
141    *
142    * <h2 class='section'>Example:</h2>
143    * <p class='bjava'>
144    *    <ja>@Bean</ja> <jk>public</jk> JobsRest.HardTimeout jobsHardTimeout() {
145    *       <jk>return new</jk> JobsRest.HardTimeout(Duration.<jsm>ofSeconds</jsm>(5));
146    *    }
147    * </p>
148    *
149    * @param value The hard timeout of every restock job.  Must be positive.
150    */
151   public record HardTimeout(Duration value) {}
152
153   /**
154    * A per-host clock for the row-key ledger, so a test can age a key past its TTL.  Declare it as a {@code @Bean} on
155    * the host, as {@link HardTimeout}.
156    *
157    * @param value The clock.
158    */
159   public record KeyClock(Clock value) {}
160
161   @SuppressWarnings({
162      "java:S2226" // the field is set by the framework's @Bean injection after construction, so it cannot be final.
163   })
164   @Bean
165   private transient HardTimeout hardTimeout;
166
167   /**
168    * A per-host groom step period.  Declare it as a {@code @Bean} on the host, as {@link HardTimeout}.
169    *
170    * @param value Time between steps, at least 1 ms.
171    */
172   public record GroomPeriod(Duration value) {}
173
174   /**
175    * A per-host species-row hold: how long a species row's restock holds after its effect.  Declare it as a
176    * {@code @Bean} on the host, as {@link HardTimeout}.
177    *
178    * @param value The post-effect hold, 0 to {@value #MAX_HOLD_MS} ms.
179    */
180   public record SpeciesHold(Duration value) {}
181
182   @SuppressWarnings({
183      "java:S2226" // the field is set by the framework's @Bean injection after construction, so it cannot be final.
184   })
185   @Bean
186   private transient KeyClock keyClock;
187
188   @SuppressWarnings({
189      "java:S2226" // the field is set by the framework's @Bean injection after construction, so it cannot be final.
190   })
191   @Bean
192   private transient GroomPeriod groomPeriod;
193
194   @SuppressWarnings({
195      "java:S2226" // the field is set by the framework's @Bean injection after construction, so it cannot be final.
196   })
197   @Bean
198   private transient SpeciesHold speciesHold;
199
200   private final transient AtomicReference<AsyncJobRegistry> jobs = new AtomicReference<>();
201   private final transient ExecutorService pool = Executors.newCachedThreadPool(r -> {
202      var t = new Thread(r, "petstore-restock");
203      t.setDaemon(true);
204      return t;
205   });
206   /** Submitted jobs, newest first, for the Jobs table. */
207   private final transient Deque<JsonMap> submitted = new ConcurrentLinkedDeque<>();
208
209   /** How many submitted jobs the table remembers. */
210   static final int MAX_SUBMITTED = 100;
211
212   /**
213    * The restock request.
214    *
215    * <h2 class='section'>Example:</h2>
216    * <p class='bjava'>
217    *    <jc>// Wire form: {perSpecies:{DOG:2, CAT:1}, idempotencyKey:'7f3c...'}</jc>
218    *    <jk>var</jk> <jv>r</jv> = <jk>new</jk> Restock(Map.<jsm>of</jsm>(Species.<jsf>DOG</jsf>, 2), <js>"7f3c"</js>, 0, 1000);
219    * </p>
220    *
221    * @param perSpecies How many pets of each species to add (1-50 each).
222    * @param idempotencyKey The client-minted key; resubmits with the same key return the same job.
223    * @param slowMs How long to wait before restocking, in whole milliseconds (0-{@value JobsRest#MAX_SLOW_MS}), so a demo can
224    *    cancel before the effect.
225    * @param holdMs How long to hold after restocking, in whole milliseconds (0-{@value JobsRest#MAX_HOLD_MS}, default
226    *    {@value JobsRest#DEFAULT_HOLD_MS}), so a demo can cancel after the effect.
227    */
228   public record Restock(Map<Species,Integer> perSpecies, String idempotencyKey, int slowMs, int holdMs) {
229
230      /**
231       * Reads a restock from any of its wire shapes: the API form {@code {perSpecies:{DOG:2}, idempotencyKey}}, the
232       * dialog form the views runtime posts, {@code {action, targetId, idempotencyKey, fields:{DOG:"2"}}}, or the
233       * job-log card's flat form {@code {DOG:2, slowMs:0, idempotencyKey}}.  In the dialog and flat forms a blank,
234       * <jk>null</jk> or {@code 0} count means "none of that species".  Any form may carry {@code slowMs} and
235       * {@code holdMs}; each must be a whole number of milliseconds, so a decimal such as {@code 1.5} is rejected
236       * rather than truncated.
237       *
238       * @param body The request body.
239       * @return The restock.
240       * @throws BadRequest If a species is unknown, a count is not a whole number, {@code slowMs} is not a whole
241       *    number 0-{@value JobsRest#MAX_SLOW_MS}, or {@code holdMs} is not a whole number 0-{@value JobsRest#MAX_HOLD_MS}.
242       */
243      public static Restock of(JsonMap body) {
244         var per = new LinkedHashMap<Species,Integer>();
245         try {
246            var api = body.getMap("perSpecies");
247            var form = body.getMap("fields");
248            if (api != null) {
249               api.forEach((k, v) -> per.put(Species.valueOf(k), v == null ? null : Integer.valueOf(String.valueOf(v))));
250            } else if (form != null) {
251               form.forEach((k, v) -> putCount(per, Species.valueOf(k), v));
252            } else {
253               for (var s : Species.values())
254                  putCount(per, s, body.get(s.name()));
255            }
256         } catch (RuntimeException e) { // NumberFormat/IllegalArgument for bad counts or species; a conversion error when perSpecies/fields is not a map.
257            throw new BadRequest("Restock counts must be whole numbers for known species");
258         }
259         return new Restock(per, body.getString("idempotencyKey"), millis(body, "slowMs", 0, MAX_SLOW_MS), millis(body, "holdMs", DEFAULT_HOLD_MS, MAX_HOLD_MS));
260      }
261
262      private static void putCount(Map<Species,Integer> per, Species s, Object v) {
263         var x = v == null ? "" : String.valueOf(v).trim();
264         if (! x.isEmpty() && ! "0".equals(x))
265            per.put(s, Integer.valueOf(x));
266      }
267
268      private static int millis(JsonMap body, String name, int dflt, int max) {
269         var v = body.get(name);
270         if (v == null || String.valueOf(v).isBlank())
271            return dflt;
272         try {
273            var n = Integer.parseInt(String.valueOf(v).trim());
274            if (n >= 0 && n <= max)
275               return n;
276         } catch (NumberFormatException e) {
277            // Not a whole number (a decimal included): falls through to the 400 below.
278         }
279         throw new BadRequest("Invalid %s '%s'; must be a whole number 0-%s", name, v, max);
280      }
281   }
282
283   /** @return The groom runs, created (with their seeded run) on first use. */
284   private GroomRuns groomRuns() {
285      var g = groomRef.get();
286      if (g == null) {
287         var n = GroomRuns.seeded(groomPeriod != null ? groomPeriod.value().toMillis() : Long.getLong(GroomRuns.PERIOD_PROPERTY, GroomRuns.DEFAULT_PERIOD_MS));
288         if (groomRef.compareAndSet(null, n)) {
289            g = n;
290         } else {
291            n.close();
292            g = groomRef.get();
293         }
294      }
295      return g;
296   }
297
298   /** @return The row-key ledger, on the host's {@link KeyClock} if it declares one. */
299   private JobKeyLedger jobKeys() {
300      var c = keyClock;
301      return c == null ? jobKeys : jobKeys.clock(c.value());
302   }
303
304   @Override /* AsyncJobsMixin */
305   public AsyncJobRegistry asyncJobRegistry() {
306      var j = jobs.get();
307      if (j == null) {
308         synchronized (this) {
309            j = jobs.get();
310            if (j == null) {
311               j = new AsyncJobRegistry(hardTimeout == null ? DEFAULT_HARD_TIMEOUT : hardTimeout.value());
312               jobs.set(j);
313            }
314         }
315      }
316      return j;
317   }
318
319   /**
320    * Stops the restock pool, the registry's timeout scheduler and the groom runs when the resource is destroyed.
321    */
322   @RestDestroy
323   public void closeJobs() {
324      pool.shutdownNow();
325      var j = jobs.get();
326      if (j != null)
327         j.close();
328      var g = groomRef.get();
329      if (g != null)
330         g.close();
331   }
332
333   /**
334    * Renders the Jobs page.
335    *
336    * @param req The request; {@code ?builder} renders the builder twin.
337    * @param run The groom run to show, or <jk>null</jk> for the newest.
338    * @return The page view.
339    * @throws NotFound If there is no groom run to show.
340    */
341   @RestGet(path="/")
342   public View page(RestRequest req, @Query("run") String run) {
343      var r = groomRuns().get(run).orElseGet(groomRuns()::latest);
344      if (r == null)
345         throw new NotFound("No groom run to show");
346      var main = terminal("groom-output", r.id(), "Groom log");
347      var file = terminal("groom-file", r.id() + GroomRuns.FILE_SUFFIX, "Transcript file");
348      if (builder(req))
349         return twin(req, OpsPageBuilders.jobs(main, file));
350      return FreemarkerView.of("jobs.ftlh")
351         .attr("groomBytesUrl", main.bytesUrl)
352         .attr("groomFileBytesUrl", file.bytesUrl);
353   }
354
355   /** @return The restock intro as a chrome-free fragment. */
356   @RestGet(path="/fragments/restock-help")
357   public View restockHelp() {
358      return FreemarkerView.of("fragments/restock-help.ftlh");
359   }
360
361   private static TerminalDef terminal(String id, String terminalId, String title) {
362      var def = TerminalDef.forMixin(id, MOUNT, terminalId).title(title);
363      def.inputId = null; // Output only: the groom terminals take no keystrokes.
364      return def.validate();
365   }
366
367   /**
368    * The restock dialog's form source: a read-only GET returning the modal with a freshly minted self-targeted key.
369    *
370    * <p>
371    * The dialog is the plain restock, so it asks only for counts and always runs with no wait and the default
372    * {@value #DEFAULT_HOLD_MS} ms hold.  The timing knobs ({@code slowMs}, {@code holdMs}) belong to the job-log card,
373    * which exists to demonstrate cancelling before and after the effect.
374    *
375    * @return The modal.
376    */
377   @RestGet(path="/restock-form")
378   public ModalDef restockForm() {
379      var form = FormDef.create();
380      for (var s : Species.values())
381         form.field(FormDef.Input.of(s.name(), s.name().charAt(0) + s.name().substring(1).toLowerCase() + " (0-50)", "text").value("0"));
382      return ModalDef.create("Restock the store").form(form)
383         .idempotencyKey(IdempotencyKey.mintSelfTargeted(ACTION).value()).selfTargeted(true);
384   }
385
386   /**
387    * @return Submitted jobs, newest first, with their current state.
388    */
389   @RestGet(path="/rows")
390   public List<JsonMap> rows() {
391      return submitted.stream().map(x -> {
392         var row = new JsonMap(state(x.getString("jobId"))).append(x);
393         row.remove("idempotencyKey");
394         return row;
395      }).toList();
396   }
397
398   /**
399    * Submits a restock.
400    *
401    * @param req The request.
402    * @param res The response (202).
403    * @param json The restock body (see {@link Restock#of(JsonMap)}).
404    * @return The job ref (202), or a 429 {@code app:job-limit} refusal whose message names the running count, so a
405    *    JSON client such as the job-log card can show it.
406    * @throws BadRequest On a missing key, an out-of-range count, or a bad {@code slowMs} or {@code holdMs} (see
407    *    {@link Restock#of(JsonMap)}).
408    */
409   @RestPost(path="/restock")
410   public Object restock(RestRequest req, RestResponse res, @Content JsonMap json) {
411      var actor = ConsoleWrites.actor(req);
412      var body = Restock.of(json);
413      var k = body.idempotencyKey();
414      if (ib(k))
415         throw new BadRequest("Missing 'idempotencyKey'");
416      var key = IdempotencyKey.of(k, ACTION, k.equals(json.getString("targetId")) ? k : ACTION);
417      if (body.perSpecies().isEmpty())
418         throw new BadRequest("Nothing to restock");
419      body.perSpecies().forEach((s, n) -> {
420         if (n == null || n < 1 || n > 50)
421            throw new BadRequest("Count for '%s' must be between 1 and 50", s);
422      });
423      var limit = Integer.getInteger(LIMIT_PROPERTY, 2);
424      var registry = asyncJobRegistry();
425      AsyncJob job;
426      synchronized (submitted) {
427         // Resolve the idempotency key first: a replay returns its existing job and never counts against the limit.
428         var prior = submitted.stream().filter(x -> k.equals(x.getString("idempotencyKey"))).findFirst();
429         job = prior.flatMap(x -> registry.get(x.getString("jobId"))).orElse(null);
430         if (job == null) {
431            prior.ifPresent(submitted::remove); // The job aged out of the registry; this is a fresh submission.
432            var created = registry.runningCount() >= limit ? null : registry.tryCreate(key).orElse(null);
433            if (created == null)
434               return limitReached(res, registry.runningCount());
435            job = created;
436            submitted.addFirst(JsonMap.of("jobId", created.id(), "submitted", created.createdAt().toString(), "request", body.perSpecies(), "idempotencyKey", k));
437            while (submitted.size() > MAX_SUBMITTED)
438               submitted.removeLast();
439            pool.execute(() -> run(created, body, () -> store().restock(body.perSpecies(), "job:restock:" + actor)));
440         }
441      }
442      res.setStatus(202);
443      return AsyncJobRef.of(job);
444   }
445
446   private static ActionResult limitReached(RestResponse res, int running) {
447      res.setStatus(429);
448      return ActionResult.refusal("app:job-limit").message(f("Job limit reached; '%s' running", running));
449   }
450
451   /**
452    * Runs one restock: the optional slow wait (cancellable, nothing changed yet), the restock itself, then the
453    * after-effect hold.  Every step stops as soon as the job has settled (a cancel or the hard timeout).
454    *
455    * @param job The job.
456    * @param body The restock.
457    * @param effect The restock itself; returns the added pets.
458    */
459   static void run(AsyncJob job, Restock body, Supplier<List<Pet>> effect) {
460      run(job, body, effect, () -> null);
461   }
462
463   /**
464    * Same as {@link #run(AsyncJob, Restock, Supplier)}, but the success result carries a value, read after the
465    * effect: a row job returns its row's new state, which the card's {@code output} writes back into the row.
466    *
467    * @param job The job.
468    * @param body The restock.
469    * @param effect The restock itself; returns the added pets.
470    * @param value The success result's value.
471    */
472   static void run(AsyncJob job, Restock body, Supplier<List<Pet>> effect, Supplier<Object> value) {
473      try {
474         if (body.slowMs() > 0) {
475            job.progress(f("Waiting %s ms before restocking", body.slowMs()));
476            if (! hold(job, body.slowMs()))
477               return;
478         }
479         // One lock acquisition, shared with cancel and the timeout: a job settled after the wait is left as it was.
480         if (! job.markEffectStarted())
481            return;
482         var total = body.perSpecies().values().stream().mapToInt(Integer::intValue).sum();
483         job.progress(f("Restocking %s pets", total));
484         var pets = effect.get();
485         pets.forEach(p -> job.progress(f("Added %s", p.getName())));
486         if (hold(job, body.holdMs()))
487            job.complete(ActionResult.success(value.get()).message(f("Restocked %s pets", pets.size())));
488      } catch (InterruptedException e) {
489         Thread.currentThread().interrupt();
490         job.complete(ActionResult.unknown().message("Interrupted"));
491      } catch (RuntimeException e) {
492         job.complete(ActionResult.failure().message(e.getMessage()));
493      }
494   }
495
496   /** Sleeps in short steps; returns <jk>false</jk> as soon as the job has settled. */
497   private static boolean hold(AsyncJob job, long ms) throws InterruptedException {
498      for (var left = ms; left > 0; left -= SLOW_STEP_MS) {
499         if (job.isTerminal())
500            return false;
501         Thread.sleep(Math.min(SLOW_STEP_MS, left));
502      }
503      return ! job.isTerminal();
504   }
505
506   /**
507    * @param id The job id.
508    * @return {@code {state: RUNNING|DONE, result}}.
509    * @throws NotFound If the job is unknown or has aged out.
510    */
511   @RestGet(path="/{id}")
512   public JsonMap job(@Path("id") String id) {
513      if (asyncJobRegistry().get(id).isEmpty())
514         throw new NotFound("Unknown job '%s'", id);
515      return state(id);
516   }
517
518   private JsonMap state(String id) {
519      var j = asyncJobRegistry().get(id);
520      if (j.isEmpty())
521         return JsonMap.of("state", "EXPIRED");
522      return j.get().isTerminal()
523         ? JsonMap.of("state", "DONE", "result", j.get().result())
524         : JsonMap.of("state", "RUNNING");
525   }
526
527   @Override /* TerminalMixin */
528   public Optional<TerminalSource> terminalSource(String id, RestRequest req) {
529      return groomRuns().source(id); // The run id is an unguessable 128-bit secret; knowing it is the access rule.
530   }
531
532   /**
533    * The Groom dialog's form source.
534    *
535    * @return The modal.
536    */
537   @RestGet(path="/groom-form")
538   public ModalDef groomForm() {
539      var form = FormDef.create();
540      form.field(FormDef.Input.of("pet", "Pet name", "text").value("Rex"));
541      form.field(FormDef.Input.of("verbose", "Verbose (8000 extra lines)", "checkbox"));
542      return ModalDef.create("Groom a pet").form(form)
543         .idempotencyKey(IdempotencyKey.mintSelfTargeted(GROOM_JOB).value()).selfTargeted(true);
544   }
545
546   /**
547    * Starts a groom run.
548    *
549    * <p>
550    * Accepts the API form {@code {pet, verbose}} or the dialog form {@code {action, targetId, idempotencyKey,
551    * fields:{pet, verbose}}}; a resubmit with the same {@code idempotencyKey} returns the same run.
552    *
553    * @param json The body.
554    * @return The new run's row.
555    * @throws IOException If the run's temp file cannot be created.
556    */
557   @RestPost(path="/groom")
558   public ActionResult groom(@Content JsonMap json) throws IOException {
559      var fields = json.getMap("fields");
560      var src = fields != null ? fields : json;
561      var run = groomRuns().start(src.getString("pet"), isOn(src.get("verbose")), json.getString("idempotencyKey"));
562      return ActionResult.success(run.row()).message(f("Grooming %s", run.pet()));
563   }
564
565   private static boolean isOn(Object v) {
566      var s = v == null ? "" : String.valueOf(v);
567      return "true".equals(s) || "on".equals(s);
568   }
569
570   /**
571    * @return The groom runs, newest first.
572    */
573   @RestGet(path="/groom-rows")
574   public List<JsonMap> groomRows() {
575      return groomRuns().list().stream().map(GroomRuns.Run::row).toList();
576   }
577
578   //-----------------------------------------------------------------------------------------------------------------
579   // Row jobs: a key bound to one row, issued by the server and checked on submit.
580   //-----------------------------------------------------------------------------------------------------------------
581
582   /**
583    * @return One row per species: {@code {id, species, available}}.
584    */
585   @RestGet(path="/species")
586   public List<JsonMap> speciesRows() {
587      return Arrays.stream(Species.values()).map(this::speciesRow).toList();
588   }
589
590   /**
591    * A species row's detail panel.
592    *
593    * @param id The species name, for example {@code DOG}.
594    * @return {@code {contractVersion, fields:{available, lastRestock}}}.
595    * @throws NotFound If the species is unknown.
596    */
597   @RestGet(path="/species/{id}")
598   public JsonMap speciesDetail(@Path("id") String id) {
599      var s = species(id);
600      var last = lastRestock.get(s);
601      var fields = new JsonMap().append("available", available(s)).append("lastRestock", last == null ? null : last.toString());
602      return RowDetail.response(fields);
603   }
604
605   /**
606    * Issues a restock key bound to one species and the caller.
607    *
608    * <p>
609    * A POST, so it sits behind the console's CSRF boundary like every other console write; it changes nothing but the
610    * key ledger.
611    *
612    * @param req The request.
613    * @param id The species name.
614    * @return {@code {contractVersion, action, targetId, idempotencyKey}}.
615    * @throws NotFound If the species is unknown.
616    */
617   @RestPost(path="/species/{id}/key")
618   public Map<String,Object> speciesKey(RestRequest req, @Path("id") String id) {
619      var s = species(id);
620      return JobKeyLedger.response(jobKeys().issue(RESTOCK_SPECIES, s.name(), ConsoleWrites.actor(req)));
621   }
622
623   /**
624    * Restocks one species from its row's detail panel.
625    *
626    * <p>
627    * The species is the path's {@code {id}}, which must equal the body's {@code targetId}, which the key must be bound
628    * to.  The body's {@code fields} supply only {@code count}; any other field (the panel's hidden {@code species}
629    * included) is ignored.  A replay of a claimed key returns the same job, or a {@link JobKeyLedger#SPENT_CODE}
630    * refusal once the registry has aged the job out.
631    *
632    * @param req The request.
633    * @param res The response (202, or 409 on a refusal).
634    * @param id The species name.
635    * @param json The action envelope {@code {action, targetId, idempotencyKey, fields:{count}}}.
636    * @return The job ref (202), a 409 refusal, or a 429 {@code app:job-limit} refusal.
637    * @throws NotFound If the species is unknown.
638    * @throws BadRequest If the envelope is malformed or {@code count} is not a whole number 1-50.
639    */
640   @RestPost(path="/species/{id}/restock")
641   public Object restockSpecies(RestRequest req, RestResponse res, @Path("id") String id, @Content JsonMap json) {
642      var s = species(id);
643      var submit = RowJobSubmit.parse(json);
644      if (! s.name().equals(submit.targetId()))
645         return refused(res, RowJobSubmit.bindingRefusal());
646      var actor = ConsoleWrites.actor(req);
647      var ledger = jobKeys();
648      var registry = asyncJobRegistry();
649      AsyncJob job;
650      // The ledger's own monitor: the verify that reads the claim and the claim itself happen under one lock, so of
651      // two concurrent submits of one key the second sees the first's job and replays it instead of starting it again.
652      synchronized (ledger) {
653         var grant = submit.verify(ledger, RESTOCK_SPECIES, actor).orElse(null);
654         if (grant == null)
655            return refused(res, RowJobSubmit.bindingRefusal());
656         var claimed = grant.jobId().orElse(null);
657         if (claimed != null) {
658            var r = replay(claimed, registry);
659            res.setStatus(r instanceof ActionResult ? RowJobSubmit.REFUSAL_STATUS : 202);
660            return r;
661         }
662         // After the replay check: a replay answers its job even if this resubmit's fields have changed.
663         var body = new Restock(Map.of(s, count(submit.fields())), grant.key().value(), 0,
664            Math.max(0, Math.min(speciesHold != null ? (int) speciesHold.value().toMillis() : Integer.getInteger(SPECIES_HOLD_PROPERTY, DEFAULT_HOLD_MS), MAX_HOLD_MS)));
665         var created = registry.runningCount() >= Integer.getInteger(LIMIT_PROPERTY, 2) ? null : registry.tryCreate(grant.key()).orElse(null);
666         if (created == null)
667            return limitReached(res, registry.runningCount());
668         ledger.claim(grant.key(), created.id());
669         job = created;
670         remember(created, body);
671         pool.execute(() -> run(created, body, () -> restocked(s, body, actor), () -> speciesRow(s)));
672      }
673      res.setStatus(202);
674      return AsyncJobRef.of(job);
675   }
676
677   /**
678    * A claimed key's replay: its job's ref, or a {@link JobKeyLedger#SPENT_CODE} refusal once the registry has reaped
679    * the job.  Never a second job: the key was spent on the first.
680    *
681    * @param jobId The job the key was claimed for.
682    * @param registry The registry.
683    * @return The {@link AsyncJobRef}, or the refusal.
684    */
685   static Object replay(String jobId, AsyncJobRegistry registry) {
686      return registry.get(jobId).<Object>map(AsyncJobRef::of)
687         .orElseGet(() -> ActionResult.refusal(JobKeyLedger.SPENT_CODE).message("This key's job has aged out; fetch a new key"));
688   }
689
690   private List<Pet> restocked(Species s, Restock body, String actor) {
691      var pets = store().restock(body.perSpecies(), "job:restock-species:" + actor);
692      lastRestock.put(s, PetstoreSeed.now());
693      return pets;
694   }
695
696   /**
697    * A groom run's detail panel.
698    *
699    * @param id The run id.
700    * @return {@code {contractVersion, fields:<the run's row>}}.
701    * @throws NotFound If the run is unknown.
702    */
703   @RestGet(path="/groom-runs/{id}")
704   public JsonMap groomRunDetail(@Path("id") String id) {
705      return RowDetail.response(groomRun(id).row());
706   }
707
708   /**
709    * Issues a regroom key bound to one groom run and the caller.
710    *
711    * @param req The request.
712    * @param id The run id.
713    * @return {@code {contractVersion, action, targetId, idempotencyKey}}.
714    * @throws NotFound If the run is unknown.
715    */
716   @RestPost(path="/groom-runs/{id}/key")
717   public Map<String,Object> groomRunKey(RestRequest req, @Path("id") String id) {
718      var r = groomRun(id);
719      return JobKeyLedger.response(jobKeys().issue(REGROOM, r.id(), ConsoleWrites.actor(req)));
720   }
721
722   /**
723    * Grooms a run's pet again.
724    *
725    * <p>
726    * Ledger keys only, bound to the path's run: the pet comes from the server's run record, never the request.
727    * {@code /groom} stays the dialog's self-targeted endpoint; one endpoint taking both kinds of key would be only as
728    * strong as the self-targeted one.  A replay returns the same new run while it is kept, then a
729    * {@link JobKeyLedger#SPENT_CODE} refusal once the run list has evicted it; never a second run.
730    *
731    * @param req The request.
732    * @param res The response (409 on a refusal).
733    * @param id The run id.
734    * @param json The action envelope {@code {action, targetId, idempotencyKey}}.
735    * @return The new run's row, or a 409 refusal.
736    * @throws NotFound If the key is unclaimed and the run is unknown.
737    * @throws BadRequest If the envelope is malformed.
738    * @throws IOException If the new run's temp file cannot be created.
739    */
740   @RestPost(path="/groom-runs/{id}/regroom")
741   public ActionResult regroom(RestRequest req, RestResponse res, @Path("id") String id, @Content JsonMap json) throws IOException {
742      var submit = RowJobSubmit.parse(json);
743      if (! id.equals(submit.targetId()))
744         return refused(res, RowJobSubmit.bindingRefusal());
745      var ledger = jobKeys();
746      // As restockSpecies: the claim is read and recorded under the ledger's lock.  The replay check comes before the
747      // run lookup, because GroomRuns evicts the oldest runs first and a replay must still answer once its target is gone.
748      synchronized (ledger) {
749         var grant = submit.verify(ledger, REGROOM, ConsoleWrites.actor(req)).orElse(null);
750         if (grant == null)
751            return refused(res, RowJobSubmit.bindingRefusal());
752         var claimed = grant.jobId().orElse(null);
753         if (claimed != null)
754            return groomRuns().get(claimed).map(x -> ActionResult.success(x.row()).message(f("Grooming %s again", x.pet())))
755               .orElseGet(() -> refused(res, ActionResult.refusal(JobKeyLedger.SPENT_CODE).message("This key's run has aged out; fetch a new key")));
756         var again = groomRuns().start(groomRun(id).pet(), false, grant.key().value());
757         ledger.claim(grant.key(), again.id());
758         return ActionResult.success(again.row()).message(f("Grooming %s again", again.pet()));
759      }
760   }
761
762   private static ActionResult refused(RestResponse res, ActionResult refusal) {
763      res.setStatus(RowJobSubmit.REFUSAL_STATUS);
764      return refusal;
765   }
766
767   private void remember(AsyncJob job, Restock body) {
768      synchronized (submitted) {
769         submitted.addFirst(JsonMap.of("jobId", job.id(), "submitted", job.createdAt().toString(), "request", body.perSpecies(), "idempotencyKey", body.idempotencyKey()));
770         while (submitted.size() > MAX_SUBMITTED)
771            submitted.removeLast();
772      }
773   }
774
775   private JsonMap speciesRow(Species s) {
776      return JsonMap.of("id", s.name(), "species", s.name(), "available", available(s));
777   }
778
779   private long available(Species s) {
780      return store().getPets().stream().filter(p -> p.getSpecies() == s && p.getStatus() == PetStatus.AVAILABLE).count();
781   }
782
783   private static Species species(String id) {
784      for (var s : Species.values())
785         if (s.name().equals(id))
786            return s;
787      throw new NotFound("Unknown species '%s'", id);
788   }
789
790   private GroomRuns.Run groomRun(String id) {
791      return groomRuns().get(id).orElseThrow(() -> new NotFound("Unknown groom run '%s'", id));
792   }
793
794   private static int count(JsonMap fields) {
795      var v = fields == null ? null : fields.get("count");
796      try {
797         var n = v == null ? 0 : Integer.parseInt(String.valueOf(v).trim());
798         if (n >= 1 && n <= 50)
799            return n;
800      } catch (NumberFormatException e) {
801         // Not a whole number (a decimal included): falls through to the 400 below.
802      }
803      throw new BadRequest("Field 'count' must be a whole number between 1 and 50");
804   }
805}