001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.juneau.rest.client;
018
019import static org.apache.juneau.commons.utils.Shorts.*;
020
021import java.io.*;
022import java.net.*;
023import java.net.http.*;
024import java.net.http.HttpClient.*;
025import java.net.http.HttpRequest.*;
026import java.net.http.HttpResponse.*;
027import java.time.*;
028import java.util.*;
029
030import org.apache.juneau.http.*;
031import org.apache.juneau.http.remote.*;
032
033/**
034 * {@link HttpTransport} implementation backed by the JDK's built-in {@link HttpClient}.
035 *
036 * <p>
037 * This is the <b>default</b> transport used by {@link RestClient}.  It is built into the
038 * {@code juneau-rest-client} artifact, requires no external dependencies, and is always available on Java 11+.
039 *
040 * <p>
041 * It is auto-discovered via {@link java.util.ServiceLoader} and can also be instantiated explicitly:
042 *
043 * <p class='bjava'>
044 *    <jv>transport</jv> = JavaHttpTransport.<jsm>builder</jsm>()
045 *       .httpClient(HttpClient.newHttpClient())
046 *       .build();
047 *
048 *    <jv>client</jv> = RestClient.<jsm>builder</jsm>()
049 *       .transport(<jv>transport</jv>)
050 *       .build();
051 * </p>
052 *
053 * <p>
054 * When a pooled keep-alive connection is torn down by the server before any response bytes are received (an
055 * immediate {@code EOF} surfaced by the JDK client as an {@code IOException} whose message is
056 * {@code "HTTP/1.1 header parser received no bytes"}), this transport replays the request exactly once on a fresh
057 * connection &mdash; but only when {@link TransportRequest#isSafeToReplay()} proves it is safe (an idempotent method
058 * with an absent or repeatable body).  The JDK client already retries body-less {@code GET} requests itself, but not
059 * other idempotent methods (e.g. {@code PUT}/{@code DELETE}); this backstop closes that gap while never replaying a
060 * {@code POST} or any other non-idempotent request.
061 *
062 * <p>
063 * For SSRF-guard-active ({@code @Remote}) requests (see {@link TransportRequest#isSsrfGuardActive()}), this
064 * transport pin-on-connects: it resolves the request host, selects an address that passes
065 * {@link RemoteUrlPolicy}, and rewrites the request URI to that literal address
066 * (preserving the original host via an explicit {@code Host} header) so the socket connects only to the
067 * validated address &mdash; and re-runs the same check on every {@code Location} hop via
068 * {@link PolicyEnforcedRedirects}. The JDK {@link HttpClient} has no connect-time/DNS SPI, so an IP-literal
069 * rewrite cannot preserve TLS SNI/hostname verification for the original hostname; per the locked design, policy
070 * -covered <b>HTTPS</b> requests are therefore refused (fail closed) on this transport &mdash; use a transport
071 * that supports connect-time pinning (e.g. Apache HttpClient 4.5/5), or set {@code allowPrivateUrls(true)} if the
072 * target is an intentional local-dev/intranet endpoint. A policy-covered request is also refused if the
073 * underlying {@link HttpClient} was built with automatic redirect-following enabled ({@link Redirect#NORMAL} or
074 * {@link Redirect#ALWAYS}), since this transport cannot re-validate hops that client follows on its own; build it
075 * with the default {@link Redirect#NEVER} (the JDK default) to let this transport's own redirect loop apply.
076 *
077 * <h2 class='section'>See Also:</h2><ul>
078 *    <li class='link'><a class="doclink" href="https://juneau.apache.org/docs/topics/NextGenRestClient">Next-Generation REST Client</a>
079 * </ul>
080 *
081 * @since 9.2.1
082 */
083@SuppressWarnings({
084   "resource" // Not owned here; lifecycle is managed by the surrounding context
085})
086public final class JavaHttpTransport implements HttpTransport {
087
088   /** Default connect timeout of the {@link HttpClient} this transport creates when none is supplied. */
089   public static final Duration DEFAULT_CONNECT_TIMEOUT = Duration.ofSeconds(30);
090
091   /** Default per-request timeout (until the response headers arrive) applied when the request carries none. */
092   public static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(60);
093
094   /** Default cap, in bytes, on the response body this transport will deliver. */
095   public static final long DEFAULT_MAX_RESPONSE_BYTES = 128L * 1024 * 1024;
096
097   private static final String RESTRICTED_HEADERS_PROPERTY = "jdk.httpclient.allowRestrictedHeaders";
098
099   private final HttpClient httpClient;
100
101   private final long maxResponseBytes;
102
103   JavaHttpTransport(JavaHttpTransportBuilder builder) {
104      this.httpClient = builder.httpClient != null ? builder.httpClient : HttpClient.newBuilder().connectTimeout(DEFAULT_CONNECT_TIMEOUT).build();
105      this.maxResponseBytes = builder.maxResponseBytes;
106      if (builder.allowHostHeaderOverride)
107         allowHostHeader();
108   }
109
110   // Raw System property access is intentional: the JDK reads this value from System properties only, so Settings overrides/sources must not be consulted.
111   private static void allowHostHeader() {
112      var existing = System.getProperty(RESTRICTED_HEADERS_PROPERTY);
113      if (ib(existing))
114         System.setProperty(RESTRICTED_HEADERS_PROPERTY, "host");
115      else if (! existing.toLowerCase(Locale.ROOT).contains("host"))
116         System.setProperty(RESTRICTED_HEADERS_PROPERTY, existing + ",host");
117   }
118
119   /**
120    * Returns a new builder for this transport.
121    *
122    * @return A new builder. Never <jk>null</jk>.
123    */
124   public static JavaHttpTransportBuilder builder() {
125      return new JavaHttpTransportBuilder();
126   }
127
128   /**
129    * Returns a new instance backed by a default {@link HttpClient}.
130    *
131    * @return A new instance. Never <jk>null</jk>.
132    */
133   public static JavaHttpTransport create() {
134      return builder().build();
135   }
136
137   @Override /* HttpTransport */
138   public TransportResponse execute(TransportRequest request) throws TransportException {
139      if (request.isSsrfGuardActive())
140         return executeWithPolicy(request);
141      requireNoUnguardedRedirects(request);
142      try {
143         return sendOnce(request);
144      } catch (StaleConnectionException e) {
145         // A pooled keep-alive connection was torn down by the server before any response was received.
146         // Replay once on a fresh connection, but only when it is provably safe (idempotent + repeatable body).
147         if (! request.isSafeToReplay())
148            throw e.asTransportException();
149         try {
150            return sendOnce(request);
151         } catch (StaleConnectionException e2) {
152            throw e2.asTransportException();
153         }
154      }
155   }
156
157   @Override /* HttpTransport */
158   public boolean supportsUrlPolicy() {
159      return true;
160   }
161
162   // A caller-supplied HttpClient that follows redirects itself would replay the request, credential headers included, against
163   // a redirect target of another origin; the JDK client offers no hook to strip them.  Refuse (fail closed) rather than send.
164   private void requireNoUnguardedRedirects(TransportRequest request) throws TransportException {
165      if (httpClient.followRedirects() == Redirect.NEVER)
166         return;
167      for (var h : request.getHeaders())
168         if (RedactedHeaders.isSensitive(h.name(), RedirectSecurity.stripOnCrossOrigin()))
169            throw new TransportException("Refusing to send a request carrying the credential header '" + h.name()
170               + "' through a JavaHttpTransport whose HttpClient auto-follows redirects (" + httpClient.followRedirects()
171               + "); the JDK client would forward it to a redirect target of another origin.  Build the HttpClient with "
172               + "the default Redirect.NEVER, or pass the credential another way.");
173   }
174
175   // Runs the Juneau-controlled redirect loop, pinning each hop to a policy-allowed address.
176   private TransportResponse executeWithPolicy(TransportRequest request) throws TransportException {
177      if (httpClient.followRedirects() != Redirect.NEVER)
178         throw new TransportException("Refusing to send a policy-covered @Remote request through a "
179            + "JavaHttpTransport whose HttpClient auto-follows redirects (" + httpClient.followRedirects()
180            + "); this transport cannot re-validate hops that the client follows on its own. Build the "
181            + "HttpClient with the default Redirect.NEVER, or set allowPrivateUrls(true) if this is an "
182            + "intentional local-dev/intranet target.");
183      return PolicyEnforcedRedirects.execute(request, this::sendOncePinned);
184   }
185
186   // Performs one pin-on-connect hop: resolves and pins the request host, then sends without following redirects.
187   private TransportResponse sendOncePinned(TransportRequest request) throws TransportException {
188      var uri = request.getUri();
189      var scheme = uri.getScheme();
190      if (neqic("http", scheme))
191         throw new TransportException("JavaHttpTransport cannot preserve TLS SNI/hostname verification while "
192            + "pinning the resolved address (the JDK HttpClient has no connect-time/DNS SPI); refusing "
193            + "(fail closed) a policy-covered HTTPS request: " + uri
194            + ".  Use a transport with connect-time pinning support (e.g. Apache HttpClient 4.5/5), or set "
195            + "allowPrivateUrls(true) if this is an intentional local-dev/intranet target.");
196      InetAddress pinned;
197      try {
198         pinned = RemoteUrlPolicy.selectAllowedAddress(requireHost(uri), false, RemoteUrlPolicy.AddressResolver.DEFAULT, request.getAllowedAddressRanges());
199      } catch (UnknownHostException e) {
200         throw new TransportException("Could not resolve @Remote request host: " + uri.getHost(), e);
201      } catch (IllegalArgumentException e) {
202         // A refused pin (every resolved address is denied) is reported as the documented TransportException.
203         throw new TransportException("HTTP transport error: " + e.getMessage(), e);
204      }
205      URI pinnedUri;
206      try {
207         pinnedUri = new URI(pinnedUriString(scheme, pinned, uri));
208      } catch (URISyntaxException e) {
209         throw new TransportException("Could not build a pinned request URI for: " + uri, e);
210      }
211      var hostHeader = uri.getPort() >= 0 ? uri.getHost() + ":" + uri.getPort() : uri.getHost();
212      try {
213         return sendOnce(request, pinnedUri, hostHeader);
214      } catch (StaleConnectionException e) {
215         throw e.asTransportException();
216      }
217   }
218
219   // java.net.URI parses some host names (an underscore name such as my_service) with a null host, which
220   // InetAddress would resolve as the loopback address; refuse such a URI by name instead.
221   private static String requireHost(URI uri) {
222      var host = uri.getHost();
223      if (host == null)
224         throw iaex("Request URI has no host that java.net.URI can parse, so it cannot be pinned: %s", uri);
225      return host;
226   }
227
228   // Performs a single HTTP exchange.  Throws StaleConnectionException (a retryable signal) when the failure is a
229   // pre-response stale-connection failure; throws TransportException for every other failure.
230   private TransportResponse sendOnce(TransportRequest request) throws TransportException, StaleConnectionException {
231      return sendOnce(request, request.getUri(), null);
232   }
233
234   // Performs a single HTTP exchange against overrideUri (the pinned address) instead of request.getUri(),
235   // adding an explicit Host header (the original hostname[:port]) when hostHeader is non-null so virtual-hosting
236   // still works after the IP-literal rewrite.
237   private TransportResponse sendOnce(TransportRequest request, URI overrideUri, String hostHeader) throws TransportException, StaleConnectionException {
238      HttpRequest jdkRequest;
239      try {
240         jdkRequest = buildJdkRequest(request, overrideUri, hostHeader);
241      } catch (IllegalArgumentException e) {
242         // The JDK refuses a URI it cannot use (for example an underscore host name, which java.net.URI parses
243         // with a null host) or an invalid header.
244         if (hostHeader == null)
245            throw new TransportException("Could not build the HTTP request for " + request.getUri() + ": " + e.getMessage(), e);
246         // With hostHeader set, the JDK rejected the "Host" header override -- i.e. the JVM
247         // was not started with -Djdk.httpclient.allowRestrictedHeaders=host. Fail closed rather than silently pinning without preserving the original Host.
248         throw new TransportException("Could not preserve the original Host header while pin-on-connecting a "
249            + "policy-covered HTTP request (the JDK HttpClient restricts the Host header unless the JVM "
250            + "is started with the flag below); refusing (fail closed): " + request.getUri()
251            + ".  Call allowHostHeaderOverride() on the transport builder (a JVM-wide setting) or set the JVM flag -Djdk.httpclient.allowRestrictedHeaders=host, use a transport with native "
252            + "connect-time pinning support (e.g. Apache HttpClient 4.5/5), or set allowPrivateUrls(true) if "
253            + "this is an intentional local-dev/intranet target.", e);
254      }
255      HttpResponse<InputStream> jdkResponse;
256      try {
257         jdkResponse = httpClient.send(jdkRequest, BodyHandlers.ofInputStream());
258      } catch (IOException e) {
259         // httpClient.send() only returns once the response headers have been read, so any IOException here
260         // occurred before any response bytes were received.
261         if (TransportException.isStaleConnectionFailure(e))
262            throw new StaleConnectionException(e);
263         throw new TransportException("HTTP transport error: " + e.getMessage(), e);
264      } catch (InterruptedException e) {
265         Thread.currentThread().interrupt();
266         throw new TransportException("HTTP request interrupted", e);
267      }
268      return buildTransportResponse(jdkResponse, maxResponseBytes);
269   }
270
271   // -----------------------------------------------------------------------------------------------------------------
272   // Internal helpers
273   // -----------------------------------------------------------------------------------------------------------------
274
275   // Builds a URI string identical to "uri" except with its host replaced by the pinned literal address, by
276   // concatenating uri's already-percent-encoded raw components -- NOT via the multi-arg java.net.URI
277   // constructor, which treats its arguments as decoded components and would double-encode any "%" already
278   // present in the raw path/query/fragment (e.g. "%20" would become "%2520").
279   private static String pinnedUriString(String scheme, InetAddress pinned, URI uri) {
280      var hostLiteral = pinned instanceof Inet6Address ? "[" + pinned.getHostAddress() + "]" : pinned.getHostAddress();
281      var sb = new StringBuilder(scheme).append("://").append(hostLiteral);
282      if (uri.getPort() >= 0)
283         sb.append(':').append(uri.getPort());
284      if (uri.getRawPath() != null)
285         sb.append(uri.getRawPath());
286      if (uri.getRawQuery() != null)
287         sb.append('?').append(uri.getRawQuery());
288      if (uri.getRawFragment() != null)
289         sb.append('#').append(uri.getRawFragment());
290      return sb.toString();
291   }
292
293   private static HttpRequest buildJdkRequest(TransportRequest request, URI uri, String hostHeader) {
294      var builder = HttpRequest.newBuilder().uri(uri).method(request.getMethod(), buildBodyPublisher(request.getBody()));
295      if (hostHeader != null)
296         builder.header("Host", hostHeader);
297      for (var h : request.getHeaders())
298         builder.header(h.name(), h.value());
299      var timeout = request.getTimeout();
300      builder.timeout(timeout != null ? timeout : DEFAULT_REQUEST_TIMEOUT);
301      return builder.build();
302   }
303
304   @SuppressWarnings({
305      "java:S2095" // PipedInputStream closed by HttpClient when body publishing finishes; PipedOutputStream closed in writer thread try-with-resources
306   })
307   private static BodyPublisher buildBodyPublisher(TransportBody body) {
308      if (body == null)
309         return BodyPublishers.noBody();
310      // Use a pipe so body.writeTo() streams directly to the JDK client without full in-memory buffering.
311      // The writer runs on a daemon thread; the JDK client reads from the PipedInputStream on its own threads.
312      // The supplier runs when the client publishes the request body (not when the HttpRequest is built).
313      var publisher = BodyPublishers.ofInputStream(() -> {
314         var in = new PipedInputStream();
315         try {
316            var out = new PipedOutputStream(in);
317            var writer = new Thread(() -> {
318               try (out) {
319                  body.writeTo(out);
320               } catch (IOException e) {
321                  // Closing the pipe on error causes the reader to see an IOException,
322                  // which propagates to the JDK client as a send failure.
323               }
324            }, "juneau-ng-body-writer");
325            writer.setDaemon(true);
326            writer.start();
327            return in;
328         } catch (IOException e) {
329            try {
330               in.close();
331            } catch (IOException e2) {
332               e.addSuppressed(e2);
333            }
334            throw new UncheckedIOException(e);
335         }
336      });
337      // A repeatable body with a known length is sent fixed-length (Content-Length) rather than chunked;
338      // unknown-length (or non-repeatable streaming) bodies keep chunked transfer encoding.
339      var length = body.getContentLength();
340      if (body.isRepeatable() && length >= 0)
341         return length == 0 ? BodyPublishers.noBody() : BodyPublishers.fromPublisher(publisher, length);
342      return publisher;
343   }
344
345   private static TransportResponse buildTransportResponse(HttpResponse<InputStream> jdkResponse, long maxBytes) {
346      var raw = jdkResponse.body();
347      var body = maxBytes > 0 ? new LimitedInputStream(raw, maxBytes) : raw;
348      var builder = TransportResponse.builder()
349         .statusCode(jdkResponse.statusCode())
350         .body(body)
351         .closeCallback(raw);
352      jdkResponse.headers().map().forEach((name, values) ->
353         values.forEach(value -> builder.header(name, value)));
354      return builder.build();
355   }
356
357   // Fails a read once more than the configured number of body bytes has been delivered.
358   private static final class LimitedInputStream extends FilterInputStream {
359      private final long limit;
360      private long count;
361
362      LimitedInputStream(InputStream in, long limit) {
363         super(in);
364         this.limit = limit;
365      }
366
367      @Override /* InputStream */
368      public int read() throws IOException {
369         var b = super.read();
370         if (b >= 0)
371            add(1);
372         return b;
373      }
374
375      @Override /* InputStream */
376      public int read(byte[] b, int off, int len) throws IOException {
377         var n = super.read(b, off, len);
378         if (n > 0)
379            add(n);
380         return n;
381      }
382
383      @Override /* InputStream */
384      public long skip(long n) throws IOException {
385         var skipped = super.skip(n);
386         if (skipped > 0)
387            add(skipped);
388         return skipped;
389      }
390
391      private void add(long n) throws IOException {
392         count += n;
393         if (count > limit)
394            throw new IOException("Response body exceeds the configured maximum of " + limit + " bytes");
395      }
396   }
397
398   // Internal signal that a pre-response stale-connection failure occurred and may be retried once.  Carries the
399   // original cause so the caller can build the user-visible TransportException if the retry is not attempted or
400   // also fails.
401   private static final class StaleConnectionException extends Exception {
402      private static final long serialVersionUID = 1L;
403
404      StaleConnectionException(Throwable cause) {
405         super(cause);
406      }
407
408      TransportException asTransportException() {
409         return new TransportException("HTTP transport error: " + getCause().getMessage(), getCause());
410      }
411   }
412}