001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.juneau.terminal.region;
018
019import static java.nio.charset.StandardCharsets.*;
020import static java.nio.file.StandardOpenOption.*;
021import static org.apache.juneau.commons.utils.Shorts.*;
022import static org.apache.juneau.terminal.report.TerminalViewChecks.*;
023
024import java.io.*;
025import java.nio.*;
026import java.nio.channels.*;
027import java.nio.file.*;
028import java.nio.file.attribute.*;
029import java.util.*;
030import java.util.concurrent.*;
031import java.util.concurrent.atomic.*;
032import java.util.concurrent.locks.*;
033import java.util.logging.*;
034
035import org.apache.juneau.marshall.json.JsonSerializer;
036import org.apache.juneau.marshall.marshaller.Json;
037import org.apache.juneau.terminal.report.*;
038import org.apache.juneau.console.views.*;
039
040/**
041 * Runs one tool under a pseudo-terminal through {@code juneau_terminal.py --pty} and exposes its output as a
042 * {@link FileTerminalSource} and its steps as a {@link FileTerminalViewSource}.
043 *
044 * <p>
045 * The command line is
046 * {@code python3 <script> --console none --pty --size <cols>x<rows> --full-log <log> --events <events> <tool> -- <cmd...>},
047 * with the log ({@value #LOG_NAME}), its {@code .size} sidecar and the events file ({@value #EVENTS_NAME}) in the
048 * given run directory.  The script is found through the system property {@value #SCRIPT_PROPERTY}, then the
049 * environment variable {@value #SCRIPT_ENV}.
050 *
051 * <p>
052 * The run directory is single-use.  The tool's working directory is the {@code work} subdirectory of it, so the log is not in the tool's working directory; the directory is limited to its owner where the file system allows. The runner's own stdout and stderr
053 * go to {@code runner.out} there, in append mode and uncapped; that file is diagnostic only.
054 *
055 * <p>
056 * Both sources share one done flag, set once the process has exited.  If the runner exits without writing a
057 * {@code done} event, a warning {@code note} and {@code done {status:"fail"}} are appended first.  Only the last
058 * {@value #TAIL_BYTES} bytes of the events file are scanned for that {@code done} event, because the protocol puts it
059 * last.
060 *
061 * <p>
062 * The exit hook runs on one daemon thread per process, named {@code juneau-terminal-exit-<pid>}.  Its executor is
063 * shut down once the hook has run, and {@link #close()} waits for that.
064 *
065 * <p>
066 * Started with input on, the runner also gets {@code --input stdin} (right after the events pair) and the process is a
067 * {@link TerminalSink}: {@link #write(byte[])} queues the bytes in a {@value #INPUT_LIMIT}-byte pending buffer and never
068 * blocks, and one daemon thread per process, named {@code juneau-terminal-input-<pid>}, copies them to the runner's
069 * stdin, flushing after each chunk.  A closed or broken pipe turns {@link #acceptsInput()} false and discards the
070 * buffer.  With input off the command line and the streams are exactly as above, no writer thread starts, and
071 * {@link #acceptsInput()} is always false.
072 *
073 * <p>
074 * Started with a {@link TerminalPrompts}, the runner gets {@code --prompts stdio} and its stdout and stdin become the
075 * prompt channel: one daemon thread per process, named {@code terminal-prompts-<dir name>}, reads the stdout lines (at
076 * most {@value #PROMPT_LINE_LIMIT} bytes each; a longer one is dropped) and hands them to
077 * {@link TerminalPrompts#onLine(String)}, then calls {@link TerminalPrompts#onEnd()} once.  The runner's stderr goes to
078 * {@code runner.out}.  {@link #answer(String)} writes one line back.  Neither a prompt line nor an answer is ever
079 * logged.  Input and prompts are mutually exclusive.
080 *
081 * <p>
082 * POSIX only: there is no Windows pseudo-terminal support.
083 *
084 * <h2 class='section'>Example:</h2>
085 * <p class='bjava'>
086 *    TerminalProcess <jv>p</jv> = TerminalProcess.<jsm>start</jsm>(<js>"maven"</js>, List.<jsm>of</jsm>(<js>"mvn"</js>, <js>"test"</js>), 120, 40, <jv>runDir</jv>);
087 *    <jv>terminals</jv>.put(<jv>id</jv>, <jv>p</jv>.terminal());
088 *    <jv>terminalViews</jv>.put(<jv>id</jv>, <jv>p</jv>.events());
089 * </p>
090 */
091@SuppressWarnings({
092   "java:S3776" // Start-up validation and shutdown are linear sequences of independent steps.
093})
094public final class TerminalProcess implements AutoCloseable, TerminalSink {
095
096   /** The system property naming {@code juneau_terminal.py}. */
097   public static final String SCRIPT_PROPERTY = "juneau.terminal.script";
098
099   /** The environment variable naming {@code juneau_terminal.py}, read when the property is not set. */
100   public static final String SCRIPT_ENV = "JUNEAU_TERMINAL_SCRIPT";
101
102   /** The byte log's file name in the run directory. */
103   public static final String LOG_NAME = "terminal.log";
104
105   /** The events file's name in the run directory. */
106   public static final String EVENTS_NAME = "terminal.events.jsonl";
107
108   /** The tool's working directory, a subdirectory of the run directory. */
109   static final String WORK_DIR = "work";
110
111   /** How much of the end of the events file is scanned for a {@code done} event. */
112   static final int TAIL_BYTES = 64 * 1024;
113
114   /** How long {@link #close()} waits for the exit hook. */
115   static final int CLOSE_WAIT_SECONDS = 5;
116
117   /** The most input bytes waiting for the writer thread: queued bytes only, not the chunk being written. */
118   static final int INPUT_LIMIT = 64 * 1024;
119
120   /** The longest prompt line read from the runner, in bytes. */
121   static final int PROMPT_LINE_LIMIT = 1 << 20;
122
123   /** How long {@link #close()} lets a prompt-channel runner end by itself after its stdin closes, in milliseconds. */
124   static final int PROMPTS_CLOSE_GRACE_MS = 3000;
125
126   /** How long {@link #close()} waits for the prompt reader thread, in milliseconds. */
127   static final int PROMPTS_JOIN_MILLIS = 2000;
128
129   /** How often the nudge thread checks the log size, in milliseconds. */
130   static final int NUDGE_POLL_MILLIS = 100;
131
132   /** How long {@link #close()} waits for the nudge thread, in milliseconds. */
133   static final int NUDGE_JOIN_MILLIS = 1000;
134
135   /** How long {@link #close()} waits for the input writer thread, in milliseconds. */
136   static final int WRITER_JOIN_MILLIS = 1000;
137
138   private static final Set<String> TOOLS = Set.of("maven", "pytest", "playwright", "generic");
139   private static final Logger LOG = Logger.getLogger(TerminalProcess.class.getName());
140
141   /** What the launcher reads from its environment; replaced in tests. */
142   record Host(String osName, String scriptProperty, String scriptEnv, String path) {
143      static Host system() {
144         return new Host(System.getProperty("os.name", ""), System.getProperty(SCRIPT_PROPERTY), System.getenv(SCRIPT_ENV), System.getenv("PATH"));
145      }
146   }
147
148   private final Process process;
149   private final Path logFile;
150   private final Path events;
151   private final Path runDir;
152   private final AtomicBoolean done = new AtomicBoolean();
153   private final FileTerminalSource terminal;
154   private final FileTerminalViewSource terminalView;
155   private final ExecutorService executor;
156   private final CompletableFuture<Integer> exit;
157   private final AtomicReference<Thread> exitThread = new AtomicReference<>();
158
159   // Input mode.  pending, pendingBytes, writing, inputFailed and closed are guarded by inLock.
160   private final boolean input;
161   private final Object inLock = new Object();
162   private final ArrayDeque<byte[]> pending = new ArrayDeque<>();
163   private int pendingBytes;  // Queued bytes only; a chunk leaves the count when the writer takes it, before it is written.
164   private boolean writing;
165   private boolean inputFailed;
166   private boolean closed;
167   private final Thread writer;
168
169   // Prompt channel.  answerLock serializes answers; answersClosed is set once the channel can take no more.
170   private final TerminalPrompts prompts;
171   private final ReentrantLock answerLock = new ReentrantLock();
172   private volatile boolean answersClosed;
173   private final Thread promptThread;
174
175   // Nudges.  nudgeThread, nudgeSet and nudgeClosed are guarded by nudgeLock.
176   private final Object nudgeLock = new Object();
177   private Thread nudgeThread;
178   private boolean nudgeSet;
179   private boolean nudgeClosed;
180   private volatile boolean nudgeStop;
181
182   /** Appends the missing done event; replaced in tests. */
183   @FunctionalInterface
184   interface DoneWriter {
185      void ensureDone(Path events, int exitCode) throws IOException;
186   }
187
188   private TerminalProcess(Process process, Path log, Path events, DoneWriter doneWriter, boolean input, TerminalPrompts prompts, Path runDir) {
189      this.runDir = runDir;
190      this.process = process;
191      this.input = input;
192      this.prompts = prompts;
193      this.logFile = log;
194      this.events = events;
195      terminal = FileTerminalSource.create(log).terminal(done::get).build();
196      terminalView = FileTerminalViewSource.create(events).terminal(done::get).build();
197      executor = Executors.newSingleThreadExecutor(r -> {
198         var t = new Thread(r, "juneau-terminal-exit-" + process.pid());
199         t.setDaemon(true);
200         return t;
201      });
202      exit = process.onExit().thenApplyAsync(p -> {
203         exitThread.set(Thread.currentThread());
204         try {
205            doneWriter.ensureDone(events, p.exitValue());
206         } catch (Exception e) {
207            LOG.log(Level.WARNING, e, () -> "Could not append the done event to '" + events + "'.");
208         } finally {
209            done.set(true);
210            executor.shutdown();
211         }
212         return p.exitValue();
213      }, executor);
214      if (input) {
215         // Every field the writer thread and the onExit callback read is assigned above; keep that order.
216         writer = new Thread(this::pump, "juneau-terminal-input-" + process.pid());
217         writer.setDaemon(true);
218         process.onExit().thenRun(() -> {
219            synchronized (inLock) {
220               inLock.notifyAll();
221            }
222         });
223         writer.start();
224      } else {
225         writer = null;
226      }
227      if (prompts != null) {
228         promptThread = new Thread(this::readPrompts, "terminal-prompts-" + id());
229         promptThread.setDaemon(true);
230         promptThread.start();
231      } else {
232         promptThread = null;
233      }
234   }
235
236   /** The prompt reader thread: hands each stdout line to the prompts, then reports the end exactly once. */
237   private void readPrompts() {
238      try (var in = new BufferedInputStream(process.getInputStream())) {
239         var line = new ByteArrayOutputStream();
240         var over = false;
241         for (var b = in.read(); b >= 0; b = in.read()) {
242            if (b == '\n') {
243               if (over)
244                  LOG.log(Level.WARNING, "prompt line over limit dropped");
245               else
246                  prompts.onLine(line.toString(UTF_8));
247               line.reset();
248               over = false;
249            } else if (! over) {
250               if (line.size() >= PROMPT_LINE_LIMIT) {
251                  over = true;
252                  line.reset();
253               } else {
254                  line.write(b);
255               }
256            }
257         }
258      } catch (IOException e) {
259         // A closed pipe ends the channel; the exception text could hold a prompt, so it is not logged.
260         LOG.log(Level.FINE, () -> "TerminalProcess prompt channel closed for '" + events + "'.");
261      } finally {
262         answersClosed = true;
263         prompts.onEnd();
264      }
265   }
266
267   /** The writer thread: copies pending chunks to stdin, never holding {@link #inLock} while writing. */
268   @SuppressWarnings({
269      "java:S3626" // The early return is what ends the endless writer loop after a failed write.
270   })
271   private void pump() {
272      var out = process.getOutputStream();
273      try {
274         while (true) {
275            byte[] chunk;
276            synchronized (inLock) {
277               while (pending.isEmpty() && ! closed && process.isAlive())
278                  inLock.wait();
279               if (closed || pending.isEmpty())
280                  return;
281               chunk = pending.poll();
282               pendingBytes -= chunk.length;
283               writing = true;
284            }
285            if (! writeChunk(out, chunk))
286               return;
287         }
288      } catch (InterruptedException e) {
289         Thread.currentThread().interrupt();
290      } finally {
291         closeQuietly(out);
292      }
293   }
294
295   // Writes one chunk to stdin; false once the pipe has failed and the pending input has been dropped.
296   private boolean writeChunk(OutputStream out, byte[] chunk) {
297      try {
298         out.write(chunk);
299         out.flush();
300         return true;
301      } catch (IOException e) {
302         synchronized (inLock) {
303            inputFailed = true;
304            pending.clear();
305            pendingBytes = 0;
306         }
307         LOG.log(Level.FINE, e, () -> "TerminalProcess input pipe failed for '" + events + "'.");
308         return false;
309      } finally {
310         synchronized (inLock) {
311            writing = false;
312         }
313      }
314   }
315
316   private static void closeQuietly(OutputStream out) {
317      try {
318         out.close();
319      } catch (IOException e) {
320         // A broken pipe has nothing left to close.
321      }
322   }
323
324   /**
325    * Starts a tool under a pseudo-terminal.
326    *
327    * @param tool The juneau_terminal tool parser: {@code maven}, {@code pytest}, {@code playwright} or {@code generic}.
328    * @param cmd The command and its arguments; at least one element.
329    *    It is a token list: each element reaches the tool as one argument, through {@link ProcessBuilder}.  It is never
330    *    joined into a string or given to a shell, so shell metacharacters in an element are literal.
331    * @param cols The terminal width, 1-9999.
332    * @param rows The terminal height, 1-9999.
333    * @param root The configured root directory for runs; created if missing.  The process creates and owns a new run
334    *    directory under it (mode 700, named by a generated id); callers never choose that path.  The tool works in a
335    *    {@code work} subdirectory of the run directory, and the runner's own stdout and stderr go to
336    *    {@code runner.out} in the run directory, in append mode and uncapped; that file is diagnostic only.
337    *    See {@link #runDir()} and {@link #id()}.
338    * @return The running process.
339    * @throws IOException If the directory could not be prepared or the process could not start.
340    * @throws IllegalArgumentException If an argument is out of range.
341    * @throws IllegalStateException If the script is not configured or {@code python3} is not on the {@code PATH}.
342    * @throws UnsupportedOperationException On Windows.
343    */
344   public static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root) throws IOException {
345      return start(tool, cmd, cols, rows, root, false);
346   }
347
348   /**
349    * Starts a tool under a pseudo-terminal, optionally with input.
350    *
351    * <p>
352    * Same as {@link #start(String, List, int, int, Path)}, except that with {@code input} on the runner gets
353    * {@code --input stdin} and the returned process accepts {@link #write(byte[])}.
354    *
355    * @param tool The juneau_terminal tool parser.
356    * @param cmd The command and its arguments; at least one element.
357    * @param cols The terminal width, 1-9999.
358    * @param rows The terminal height, 1-9999.
359    * @param root The run root; see {@link #start(String, List, int, int, Path)}.
360    * @param input Whether the job takes input through {@link #write(byte[])}.
361    * @return The running process.
362    * @throws IOException If the directory could not be prepared or the process could not start.
363    * @throws IllegalArgumentException If an argument is out of range.
364    * @throws IllegalStateException If the script is not configured or {@code python3} is not on the {@code PATH}.
365    * @throws UnsupportedOperationException On Windows.
366    */
367   public static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, boolean input) throws IOException {
368      return start(tool, cmd, cols, rows, root, Host.system(), input);
369   }
370
371   /**
372    * Starts a tool under a pseudo-terminal with a prompt channel.
373    *
374    * <p>
375    * Same as {@link #start(String, List, int, int, Path)}, except that the runner gets {@code --prompts stdio}, the
376    * script's prompt lines go to {@code prompts} and {@link #answer(String)} sends the replies.
377    *
378    * @param tool The juneau_terminal tool parser.
379    * @param cmd The command and its arguments; at least one element.
380    * @param cols The terminal width, 1-9999.
381    * @param rows The terminal height, 1-9999.
382    * @param root The run root; see {@link #start(String, List, int, int, Path)}.
383    * @param prompts Receives the prompt lines; not <jk>null</jk>.
384    * @return The running process.
385    * @throws IOException If the directory could not be prepared or the process could not start.
386    * @throws IllegalArgumentException If an argument is out of range.
387    * @throws IllegalStateException If the script is not configured or {@code python3} is not on the {@code PATH}.
388    * @throws UnsupportedOperationException On Windows.
389    */
390   public static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, TerminalPrompts prompts) throws IOException {
391      Objects.requireNonNull(prompts, "prompts");
392      return start(tool, cmd, cols, rows, root, Host.system(), TerminalProcess::ensureDone, false, prompts);
393   }
394
395   /**
396    * Starts a tool under a pseudo-terminal with extra environment variables.
397    *
398    * <p>
399    * Same as {@link #start(String, List, int, int, Path)}, except that {@code env} is added to the child's
400    * environment (over what it inherits).
401    * Use it for values that must not appear on the command line, such as secrets.
402    *
403    * @param tool The juneau_terminal tool parser.
404    * @param cmd The command and its arguments; at least one element.
405    * @param cols The terminal width, 1-9999.
406    * @param rows The terminal height, 1-9999.
407    * @param root The run root; see {@link #start(String, List, int, int, Path)}.
408    * @param env Variables to add to the child's environment; not <jk>null</jk>, may be empty.
409    * @return The running process.
410    * @throws IOException If the directory could not be prepared or the process could not start.
411    * @throws IllegalArgumentException If an argument is out of range.
412    * @throws IllegalStateException If the script is not configured or {@code python3} is not on the {@code PATH}.
413    * @throws UnsupportedOperationException On Windows.
414    */
415   public static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, Map<String,String> env) throws IOException {
416      Objects.requireNonNull(env, "env");
417      return start(tool, cmd, cols, rows, root, Host.system(), TerminalProcess::ensureDone, false, null, env);
418   }
419
420   static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, Host host, Map<String,String> env) throws IOException {
421      Objects.requireNonNull(env, "env");
422      return start(tool, cmd, cols, rows, root, host, TerminalProcess::ensureDone, false, null, env);
423   }
424
425   static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, Host host) throws IOException {
426      return start(tool, cmd, cols, rows, root, host, false);
427   }
428
429   static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, Host host, boolean input) throws IOException {
430      return start(tool, cmd, cols, rows, root, host, TerminalProcess::ensureDone, input);
431   }
432
433   static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, Host host, DoneWriter doneWriter) throws IOException {
434      return start(tool, cmd, cols, rows, root, host, doneWriter, false);
435   }
436
437   @SuppressWarnings({
438      "java:S107" // internal entry point; the parameters are the process configuration
439   })
440   static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, Host host, DoneWriter doneWriter, boolean input) throws IOException {
441      return start(tool, cmd, cols, rows, root, host, doneWriter, input, null);
442   }
443
444   @SuppressWarnings({
445      "java:S107" // internal entry point; the parameters are the process configuration
446   })
447   static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, Host host, DoneWriter doneWriter, boolean input, TerminalPrompts prompts) throws IOException {
448      return start(tool, cmd, cols, rows, root, host, doneWriter, input, prompts, Map.of());
449   }
450
451   @SuppressWarnings({
452      "java:S107" // internal entry point; the parameters are the process configuration
453   })
454   static TerminalProcess start(String tool, List<String> cmd, int cols, int rows, Path root, Host host, DoneWriter doneWriter, boolean input, TerminalPrompts prompts, Map<String,String> env) throws IOException {
455      if (input && prompts != null)
456         throw new IllegalArgumentException("input and prompts are mutually exclusive");
457      if (host.osName().toLowerCase(Locale.ROOT).startsWith("windows"))
458         throw new UnsupportedOperationException("TerminalProcess needs a POSIX pseudo-terminal; Windows is not supported.");
459      if (! TOOLS.contains(tool))
460         throw iaex("TerminalProcess tool must be one of %s; got '%s'.", new TreeSet<>(TOOLS), tool);
461      if (ie(cmd))
462         throw iaex("TerminalProcess cmd must have at least one element.");
463      if (cols < 1 || cols > 9999 || rows < 1 || rows > 9999)
464         throw iaex("TerminalProcess size must be 1-9999 x 1-9999; got '%sx%s'.", cols, rows);
465      var script = host.scriptProperty() != null && ! host.scriptProperty().isBlank() ? host.scriptProperty() : host.scriptEnv();
466      if (ib(script))
467         throw new IllegalStateException("TerminalProcess cannot find juneau_terminal.py: set the system property " + SCRIPT_PROPERTY + " or the environment variable " + SCRIPT_ENV + ".");
468      var python = findOnPath("python3", host.path());
469      if (python == null)
470         throw new IllegalStateException("TerminalProcess cannot find python3 on the PATH.");
471      var dir = newRunDir(root);
472      // The tool works in its own subdirectory, so the log, events and size files sit outside its working directory.
473      var work = dir.resolve(WORK_DIR);
474      Files.createDirectories(work);
475      restrictToOwner(work);
476      var log = dir.resolve(LOG_NAME);
477      var events = dir.resolve(EVENTS_NAME);
478      // Written here as well as by juneau_terminal, so a request that arrives before the child starts sees the real size.
479      Files.writeString(log.resolveSibling(LOG_NAME + ".size"), "{\"cols\":" + cols + ",\"rows\":" + rows + "}");
480      var argv = new ArrayList<String>();
481      argv.addAll(List.of(python.toString(), script, "--console", "none", "--pty", "--size", cols + "x" + rows,
482         "--full-log", log.toString(), "--events", events.toString()));
483      if (input)
484         argv.addAll(List.of("--input", "stdin"));
485      if (prompts != null)
486         argv.addAll(List.of("--prompts", "stdio"));
487      argv.addAll(List.of(tool, "--"));
488      argv.addAll(cmd);
489      var runnerOut = ProcessBuilder.Redirect.appendTo(dir.resolve("runner.out").toFile());
490      var pb = new ProcessBuilder(argv).directory(work.toFile());
491      pb.environment().putAll(env);
492      if (prompts == null)
493         pb.redirectErrorStream(true).redirectOutput(runnerOut);
494      else
495         pb.redirectError(runnerOut);  // stdout stays a pipe: it is the prompt channel.
496      return new TerminalProcess(pb.start(), log, events, doneWriter, input, prompts, dir);
497   }
498
499   /** Creates a new, empty run directory under the root, readable only by its owner; the name is generated. */
500   private static Path newRunDir(Path root) throws IOException {
501      Files.createDirectories(root);
502      try {
503         return Files.createTempDirectory(root, "run-", PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------")));
504      } catch (UnsupportedOperationException e) {
505         LOG.log(Level.FINE, e, () -> "No POSIX permissions for '" + root + "'.");
506         return Files.createTempDirectory(root, "run-");
507      }
508   }
509
510   /** Limits a run directory to its owner (mode 700) where the file system supports POSIX permissions. */
511   private static void restrictToOwner(Path d) throws IOException {
512      try {
513         Files.setPosixFilePermissions(d, PosixFilePermissions.fromString("rwx------"));
514      } catch (UnsupportedOperationException e) {
515         LOG.log(Level.FINE, e, () -> "No POSIX permissions for '" + d + "'.");
516      }
517   }
518
519   /** Returns the first executable {@code name} in the {@code PATH} directories, or <jk>null</jk>. */
520   static Path findOnPath(String name, String path) {
521      if (path == null)
522         return null;
523      for (var d : path.split(File.pathSeparator)) {
524         if (d.isEmpty())
525            continue;
526         var p = Paths.get(d, name);
527         if (Files.isRegularFile(p) && Files.isExecutable(p))
528            return p;
529      }
530      return null;
531   }
532
533   /** Appends a warning note and a failing done event when the last {@value #TAIL_BYTES} bytes of the events file hold no done event. */
534   static void ensureDone(Path events, int exitCode) throws IOException {
535      var text = "";
536      if (Files.exists(events)) {
537         try (var ch = FileChannel.open(events, StandardOpenOption.READ)) {
538            var from = Math.max(0, ch.size() - TAIL_BYTES);
539            var buf = ByteBuffer.allocate((int)(ch.size() - from));
540            while (buf.hasRemaining() && ch.read(buf, from + buf.position()) >= 0) {
541               // The condition does the reading; the loop has no body.
542            }
543            text = new String(buf.array(), 0, buf.position(), UTF_8);
544         }
545      }
546      // A fragment at the start of the tail never parses, so it is never taken for a done event.
547      for (var line : text.split("\n")) {
548         if (isDone(line))
549            return;
550      }
551      var sb = new StringBuilder();
552      if (! text.isEmpty() && ! text.endsWith("\n"))
553         sb.append('\n');
554      sb.append(JsonSerializer.DEFAULT.toString(TerminalEvent.note(TerminalEvent.Level.WARN, "runner exited without done, exit " + exitCode).toContractMap())).append('\n');
555      sb.append(JsonSerializer.DEFAULT.toString(TerminalEvent.done(TerminalEvent.DoneStatus.FAIL).toContractMap())).append('\n');
556      Files.writeString(events, sb, UTF_8, CREATE, APPEND);
557   }
558
559   private static boolean isDone(String line) {
560      if (line.isBlank())
561         return false;
562      try {
563         return TerminalEvent.fromMap(Json.to(line, Map.class)).kind() == TerminalEvent.Kind.DONE;
564      } catch (Exception e) {
565         return false;
566      }
567   }
568
569   /**
570    * The raw byte source over the log.
571    *
572    * @return The source.
573    */
574   public FileTerminalSource terminal() {
575      return terminal;
576   }
577
578   /**
579    * The step events source over the events file.
580    *
581    * @return The source.
582    */
583   public FileTerminalViewSource events() {
584      return terminalView;
585   }
586
587   /**
588    * The run directory the process created under its root.
589    *
590    * @return The run directory.
591    */
592   public Path runDir() {
593      return runDir;
594   }
595
596   /**
597    * The generated run id, the name of the run directory.
598    *
599    * @return The id; it satisfies the terminal id grammar.
600    */
601   public String id() {
602      return String.valueOf(runDir.getFileName());
603   }
604
605   /**
606    * The byte log.
607    *
608    * @return The log path.
609    */
610   public Path log() {
611      return logFile;
612   }
613
614   /**
615    * The events file.
616    *
617    * @return The events path.
618    */
619   public Path eventsFile() {
620      return events;
621   }
622
623   /**
624    * Whether the process has exited and the events file ends with a {@code done} event.
625    *
626    * @return <jk>true</jk> once finished.
627    */
628   public boolean isDone() {
629      return done.get();
630   }
631
632   /**
633    * Waits for the process to exit and for the done event to be in place.
634    *
635    * @param timeout The longest wait.
636    * @param unit The unit of {@code timeout}.
637    * @return The runner's exit code.
638    * @throws InterruptedException If interrupted.
639    * @throws TimeoutException If the process is still running.
640    */
641   public int waitFor(long timeout, TimeUnit unit) throws InterruptedException, TimeoutException {
642      try {
643         return exit.get(timeout, unit);
644      } catch (ExecutionException e) {
645         throw new IllegalStateException(e.getCause());
646      }
647   }
648
649   /**
650    * Whether input is accepted right now.
651    *
652    * @return <jk>true</jk> only when input is on, the process is alive, no write has failed and the process has not been
653    *    closed.
654    */
655   @Override /* TerminalSink */
656   public boolean acceptsInput() {
657      synchronized (inLock) {
658         // HTT: close() waits for the process to exit, so isAlive() is already false afterwards.  The closed term
659         // guards the window inside close(); it is not observable without a test seam.
660         return input && ! closed && ! inputFailed && process.isAlive();
661      }
662   }
663
664   /**
665    * Queues the bytes for the runner's stdin; never blocks.
666    *
667    * @param bytes The bytes to send; copied.
668    * @return <jk>false</jk> when input is not accepted, or when the bytes do not fit in the {@value #INPUT_LIMIT}-byte
669    *    pending buffer.  An empty array returns {@link #acceptsInput()}.
670    */
671   @Override /* TerminalSink */
672   public boolean write(byte[] bytes) {
673      synchronized (inLock) {
674         if (! acceptsInput())
675            return false;
676         if (bytes.length == 0)
677            return true;
678         if (pendingBytes + bytes.length > INPUT_LIMIT)
679            return false;
680         pending.add(bytes.clone());
681         pendingBytes += bytes.length;
682         inLock.notifyAll();
683         return true;
684      }
685   }
686
687   /**
688    * Whether an answer can be sent right now.
689    *
690    * @return <jk>true</jk> only when a prompt channel is on, the process is alive and the channel has not ended or failed.
691    */
692   public boolean acceptsAnswers() {
693      return prompts != null && ! answersClosed && process.isAlive();
694   }
695
696   /**
697    * Writes one answer line to the script's prompt channel; never blocks behind another answer and never logs the line.
698    *
699    * @param jsonLine The answer as one JSON line, without a newline.
700    * @return <jk>false</jk> when there is no channel, it has ended, the line holds a newline, another answer is being
701    *    written, or the write failed.
702    */
703   public boolean answer(String jsonLine) {
704      if (jsonLine.indexOf('\n') >= 0 || ! acceptsAnswers() || ! answerLock.tryLock())
705         return false;
706      try {
707         var out = process.getOutputStream();
708         out.write((jsonLine + "\n").getBytes(UTF_8));
709         out.flush();
710         return true;
711      } catch (IOException e) {
712         answersClosed = true;
713         return false;
714      } finally {
715         answerLock.unlock();
716      }
717   }
718
719   /**
720    * Polls the log's size every 100 ms and reports growth, then a final done, to a publisher.
721    *
722    * <p>
723    * The runner writes the log in another process, so there is no in-process append hook to use.  The publisher runs on
724    * a daemon thread named {@code juneau-terminal-nudge-<pid>}; a {@link RuntimeException} it throws is logged at
725    * {@code FINE} and otherwise ignored.  The thread ends after the done report, or when {@link #close()} is called.
726    *
727    * @param terminalId The terminal id passed to the publisher; must be a valid run id.
728    * @param publisher The publisher.
729    * @return This object.
730    * @throws NullPointerException If an argument is <jk>null</jk>.
731    * @throws IllegalArgumentException If the id is not a valid run id.
732    * @throws IllegalStateException If a publisher is already set, or this process has been closed.
733    */
734   public TerminalProcess nudges(String terminalId, TerminalNudgePublisher publisher) {
735      Objects.requireNonNull(terminalId, "terminalId");
736      Objects.requireNonNull(publisher, "publisher");
737      if (! isRunId(terminalId))
738         throw iaex("TerminalProcess nudge id is not a valid run id: '%s'.", terminalId);
739      synchronized (nudgeLock) {
740         if (nudgeClosed)
741            throw new IllegalStateException("TerminalProcess is closed.");
742         if (nudgeSet)
743            throw new IllegalStateException("TerminalProcess nudges are already set.");
744         nudgeSet = true;
745         nudgeThread = new Thread(() -> watch(terminalId, publisher), "juneau-terminal-nudge-" + process.pid());
746         nudgeThread.setDaemon(true);
747         nudgeThread.start();
748      }
749      return this;
750   }
751
752   /** The nudge thread: reports log growth until the process is done or the thread is stopped. */
753   private void watch(String id, TerminalNudgePublisher publisher) {
754      var last = -1L;
755      while (! nudgeStop) {
756         var fin = isDone();
757         var size = sizeOrZero(logFile);
758         if (size > last || fin) {
759            try {
760               publisher.nudge(id, size, fin);
761            } catch (RuntimeException e) {
762               LOG.log(Level.FINE, e, () -> "Terminal nudge publisher failed for '" + id + "'.");
763            }
764            last = size;
765         }
766         if (fin)
767            return;
768         try {
769            Thread.sleep(NUDGE_POLL_MILLIS);
770         } catch (InterruptedException e) {
771            Thread.currentThread().interrupt();
772            return;
773         }
774      }
775   }
776
777   private static long sizeOrZero(Path p) {
778      try {
779         return Files.size(p);
780      } catch (IOException e) {
781         return 0;
782      }
783   }
784
785   /** The exit hook's executor; for tests. */
786   ExecutorService executor() {
787      return executor;
788   }
789
790   /** The thread the exit hook ran on, or <jk>null</jk> before it has run; for tests. */
791   Thread exitThread() {
792      return exitThread.get();
793   }
794
795   /**
796    * Asks the runner to stop and waits for it, killing it if it does not.
797    *
798    * <p>
799    * Sends SIGTERM to the runner, which passes it to the tool so that the tool can clean up and record that it was
800    * cancelled.  If the process is still running after {@code grace}, this falls back to {@link #close()}, which kills it.
801    * It does nothing when the process has already exited, and it is safe to call more than once.
802    * Call {@link #close()} afterwards as usual to release the process's resources.
803    *
804    * @param grace How long to wait for the process to exit after SIGTERM; not <jk>null</jk>, zero or negative kills at once.
805    */
806   public void terminate(java.time.Duration grace) {
807      Objects.requireNonNull(grace, "grace");
808      if (! process.isAlive())
809         return;
810      process.destroy();
811      try {
812         if (process.waitFor(Math.max(0, grace.toMillis()), TimeUnit.MILLISECONDS))
813            return;
814      } catch (InterruptedException e) {
815         Thread.currentThread().interrupt();
816      }
817      close();
818   }
819
820   /**
821    * Kills the runner and its tool if they are still running, then waits up to {@value #CLOSE_WAIT_SECONDS} seconds for
822    * the exit hook.
823    *
824    * <p>
825    * The hook writes the done event and shuts its executor down, so once this returns normally both have happened.  The
826    * executor is not shut down here: a hook submitted after that would be refused, and the done flag would never be set.
827    *
828    * <p>
829    * With input on, it first stops input: pending bytes are dropped, stdin is closed and the writer thread is told to
830    * stop.  If the writer thread is blocked writing to a full pipe, it closes stdin itself once the kill breaks the pipe.
831    * Afterwards this waits up to {@value #WRITER_JOIN_MILLIS} ms for the writer thread.  The nudge thread, if any, is
832    * stopped first of all and joined for up to {@value #NUDGE_JOIN_MILLIS} ms.  With input off, it only kills.
833    */
834   @Override /* AutoCloseable */
835   public void close() {
836      Thread nudger;
837      synchronized (nudgeLock) {
838         nudgeClosed = true;
839         nudgeStop = true;
840         nudger = nudgeThread;
841      }
842      if (nudger != null)
843         nudger.interrupt();
844      if (prompts != null) {
845         // Closing stdin tells the runner the host is gone; it ends the job and writes done itself.
846         answersClosed = true;
847         closeQuietly(process.getOutputStream());
848         try {
849            process.waitFor(PROMPTS_CLOSE_GRACE_MS, TimeUnit.MILLISECONDS);
850         } catch (InterruptedException e) {
851            Thread.currentThread().interrupt();
852         }
853      }
854      if (input) {
855         synchronized (inLock) {
856            closed = true;
857            pending.clear();
858            pendingBytes = 0;
859            inLock.notifyAll();
860            // Closing flushes the stream, which waits for a write in progress; the writer flushes after every chunk, so
861            // with no write in progress there is nothing to flush and this cannot block.
862            if (! writing)
863               closeQuietly(process.getOutputStream());
864         }
865      }
866      process.descendants().forEach(ProcessHandle::destroyForcibly);
867      process.destroyForcibly();
868      try {
869         exit.get(CLOSE_WAIT_SECONDS, TimeUnit.SECONDS);
870      } catch (InterruptedException e) {
871         Thread.currentThread().interrupt();
872      } catch (ExecutionException | TimeoutException e) {
873         LOG.log(Level.WARNING, e, () -> "TerminalProcess exit hook did not finish for '" + events + "'.");
874      }
875      if (nudger != null) {
876         try {
877            nudger.join(NUDGE_JOIN_MILLIS);
878         } catch (InterruptedException e) {
879            Thread.currentThread().interrupt();
880         }
881         if (nudger.isAlive())
882            LOG.log(Level.WARNING, () -> "TerminalProcess nudge thread did not stop for '" + events + "'.");
883      }
884      if (promptThread != null) {
885         try {
886            promptThread.join(PROMPTS_JOIN_MILLIS);
887         } catch (InterruptedException e) {
888            Thread.currentThread().interrupt();
889         }
890         if (promptThread.isAlive())
891            LOG.log(Level.WARNING, () -> "TerminalProcess prompt thread did not stop for '" + events + "'.");
892      }
893      if (writer != null) {
894         try {
895            writer.join(WRITER_JOIN_MILLIS);
896         } catch (InterruptedException e) {
897            Thread.currentThread().interrupt();
898         }
899         if (writer.isAlive())
900            LOG.log(Level.WARNING, () -> "TerminalProcess input writer thread did not stop for '" + events + "'.");
901      }
902   }
903}