001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache License, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the License for the specific language governing permissions and 015 * limitations under the License. 016 */ 017package org.apache.juneau.rest.client; 018 019import static org.apache.juneau.commons.utils.Shorts.*; 020 021import java.io.*; 022import java.net.*; 023import java.net.http.*; 024import java.net.http.HttpClient.*; 025import java.net.http.HttpRequest.*; 026import java.net.http.HttpResponse.*; 027import java.time.*; 028import java.util.*; 029 030import org.apache.juneau.http.*; 031import org.apache.juneau.http.remote.*; 032 033/** 034 * {@link HttpTransport} implementation backed by the JDK's built-in {@link HttpClient}. 035 * 036 * <p> 037 * This is the <b>default</b> transport used by {@link RestClient}. It is built into the 038 * {@code juneau-rest-client} artifact, requires no external dependencies, and is always available on Java 11+. 039 * 040 * <p> 041 * It is auto-discovered via {@link java.util.ServiceLoader} and can also be instantiated explicitly: 042 * 043 * <p class='bjava'> 044 * <jv>transport</jv> = JavaHttpTransport.<jsm>builder</jsm>() 045 * .httpClient(HttpClient.newHttpClient()) 046 * .build(); 047 * 048 * <jv>client</jv> = RestClient.<jsm>builder</jsm>() 049 * .transport(<jv>transport</jv>) 050 * .build(); 051 * </p> 052 * 053 * <p> 054 * When a pooled keep-alive connection is torn down by the server before any response bytes are received (an 055 * immediate {@code EOF} surfaced by the JDK client as an {@code IOException} whose message is 056 * {@code "HTTP/1.1 header parser received no bytes"}), this transport replays the request exactly once on a fresh 057 * connection — but only when {@link TransportRequest#isSafeToReplay()} proves it is safe (an idempotent method 058 * with an absent or repeatable body). The JDK client already retries body-less {@code GET} requests itself, but not 059 * other idempotent methods (e.g. {@code PUT}/{@code DELETE}); this backstop closes that gap while never replaying a 060 * {@code POST} or any other non-idempotent request. 061 * 062 * <p> 063 * For SSRF-guard-active ({@code @Remote}) requests (see {@link TransportRequest#isSsrfGuardActive()}), this 064 * transport pin-on-connects: it resolves the request host, selects an address that passes 065 * {@link RemoteUrlPolicy}, and rewrites the request URI to that literal address 066 * (preserving the original host via an explicit {@code Host} header) so the socket connects only to the 067 * validated address — and re-runs the same check on every {@code Location} hop via 068 * {@link PolicyEnforcedRedirects}. The JDK {@link HttpClient} has no connect-time/DNS SPI, so an IP-literal 069 * rewrite cannot preserve TLS SNI/hostname verification for the original hostname; per the locked design, policy 070 * -covered <b>HTTPS</b> requests are therefore refused (fail closed) on this transport — use a transport 071 * that supports connect-time pinning (e.g. Apache HttpClient 4.5/5), or set {@code allowPrivateUrls(true)} if the 072 * target is an intentional local-dev/intranet endpoint. A policy-covered request is also refused if the 073 * underlying {@link HttpClient} was built with automatic redirect-following enabled ({@link Redirect#NORMAL} or 074 * {@link Redirect#ALWAYS}), since this transport cannot re-validate hops that client follows on its own; build it 075 * with the default {@link Redirect#NEVER} (the JDK default) to let this transport's own redirect loop apply. 076 * 077 * <h2 class='section'>See Also:</h2><ul> 078 * <li class='link'><a class="doclink" href="https://juneau.apache.org/docs/topics/NextGenRestClient">Next-Generation REST Client</a> 079 * </ul> 080 * 081 * @since 9.2.1 082 */ 083@SuppressWarnings({ 084 "resource" // Not owned here; lifecycle is managed by the surrounding context 085}) 086public final class JavaHttpTransport implements HttpTransport { 087 088 /** Default connect timeout of the {@link HttpClient} this transport creates when none is supplied. */ 089 public static final Duration DEFAULT_CONNECT_TIMEOUT = Duration.ofSeconds(30); 090 091 /** Default per-request timeout (until the response headers arrive) applied when the request carries none. */ 092 public static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(60); 093 094 /** Default cap, in bytes, on the response body this transport will deliver. */ 095 public static final long DEFAULT_MAX_RESPONSE_BYTES = 128L * 1024 * 1024; 096 097 private static final String RESTRICTED_HEADERS_PROPERTY = "jdk.httpclient.allowRestrictedHeaders"; 098 099 private final HttpClient httpClient; 100 101 private final long maxResponseBytes; 102 103 JavaHttpTransport(JavaHttpTransportBuilder builder) { 104 this.httpClient = builder.httpClient != null ? builder.httpClient : HttpClient.newBuilder().connectTimeout(DEFAULT_CONNECT_TIMEOUT).build(); 105 this.maxResponseBytes = builder.maxResponseBytes; 106 if (builder.allowHostHeaderOverride) 107 allowHostHeader(); 108 } 109 110 // Raw System property access is intentional: the JDK reads this value from System properties only, so Settings overrides/sources must not be consulted. 111 private static void allowHostHeader() { 112 var existing = System.getProperty(RESTRICTED_HEADERS_PROPERTY); 113 if (ib(existing)) 114 System.setProperty(RESTRICTED_HEADERS_PROPERTY, "host"); 115 else if (! existing.toLowerCase(Locale.ROOT).contains("host")) 116 System.setProperty(RESTRICTED_HEADERS_PROPERTY, existing + ",host"); 117 } 118 119 /** 120 * Returns a new builder for this transport. 121 * 122 * @return A new builder. Never <jk>null</jk>. 123 */ 124 public static JavaHttpTransportBuilder builder() { 125 return new JavaHttpTransportBuilder(); 126 } 127 128 /** 129 * Returns a new instance backed by a default {@link HttpClient}. 130 * 131 * @return A new instance. Never <jk>null</jk>. 132 */ 133 public static JavaHttpTransport create() { 134 return builder().build(); 135 } 136 137 @Override /* HttpTransport */ 138 public TransportResponse execute(TransportRequest request) throws TransportException { 139 if (request.isSsrfGuardActive()) 140 return executeWithPolicy(request); 141 requireNoUnguardedRedirects(request); 142 try { 143 return sendOnce(request); 144 } catch (StaleConnectionException e) { 145 // A pooled keep-alive connection was torn down by the server before any response was received. 146 // Replay once on a fresh connection, but only when it is provably safe (idempotent + repeatable body). 147 if (! request.isSafeToReplay()) 148 throw e.asTransportException(); 149 try { 150 return sendOnce(request); 151 } catch (StaleConnectionException e2) { 152 throw e2.asTransportException(); 153 } 154 } 155 } 156 157 @Override /* HttpTransport */ 158 public boolean supportsUrlPolicy() { 159 return true; 160 } 161 162 // A caller-supplied HttpClient that follows redirects itself would replay the request, credential headers included, against 163 // a redirect target of another origin; the JDK client offers no hook to strip them. Refuse (fail closed) rather than send. 164 private void requireNoUnguardedRedirects(TransportRequest request) throws TransportException { 165 if (httpClient.followRedirects() == Redirect.NEVER) 166 return; 167 for (var h : request.getHeaders()) 168 if (RedactedHeaders.isSensitive(h.name(), RedirectSecurity.stripOnCrossOrigin())) 169 throw new TransportException("Refusing to send a request carrying the credential header '" + h.name() 170 + "' through a JavaHttpTransport whose HttpClient auto-follows redirects (" + httpClient.followRedirects() 171 + "); the JDK client would forward it to a redirect target of another origin. Build the HttpClient with " 172 + "the default Redirect.NEVER, or pass the credential another way."); 173 } 174 175 // Runs the Juneau-controlled redirect loop, pinning each hop to a policy-allowed address. 176 private TransportResponse executeWithPolicy(TransportRequest request) throws TransportException { 177 if (httpClient.followRedirects() != Redirect.NEVER) 178 throw new TransportException("Refusing to send a policy-covered @Remote request through a " 179 + "JavaHttpTransport whose HttpClient auto-follows redirects (" + httpClient.followRedirects() 180 + "); this transport cannot re-validate hops that the client follows on its own. Build the " 181 + "HttpClient with the default Redirect.NEVER, or set allowPrivateUrls(true) if this is an " 182 + "intentional local-dev/intranet target."); 183 return PolicyEnforcedRedirects.execute(request, this::sendOncePinned); 184 } 185 186 // Performs one pin-on-connect hop: resolves and pins the request host, then sends without following redirects. 187 private TransportResponse sendOncePinned(TransportRequest request) throws TransportException { 188 var uri = request.getUri(); 189 var scheme = uri.getScheme(); 190 if (neqic("http", scheme)) 191 throw new TransportException("JavaHttpTransport cannot preserve TLS SNI/hostname verification while " 192 + "pinning the resolved address (the JDK HttpClient has no connect-time/DNS SPI); refusing " 193 + "(fail closed) a policy-covered HTTPS request: " + uri 194 + ". Use a transport with connect-time pinning support (e.g. Apache HttpClient 4.5/5), or set " 195 + "allowPrivateUrls(true) if this is an intentional local-dev/intranet target."); 196 InetAddress pinned; 197 try { 198 pinned = RemoteUrlPolicy.selectAllowedAddress(requireHost(uri), false, RemoteUrlPolicy.AddressResolver.DEFAULT, request.getAllowedAddressRanges()); 199 } catch (UnknownHostException e) { 200 throw new TransportException("Could not resolve @Remote request host: " + uri.getHost(), e); 201 } catch (IllegalArgumentException e) { 202 // A refused pin (every resolved address is denied) is reported as the documented TransportException. 203 throw new TransportException("HTTP transport error: " + e.getMessage(), e); 204 } 205 URI pinnedUri; 206 try { 207 pinnedUri = new URI(pinnedUriString(scheme, pinned, uri)); 208 } catch (URISyntaxException e) { 209 throw new TransportException("Could not build a pinned request URI for: " + uri, e); 210 } 211 var hostHeader = uri.getPort() >= 0 ? uri.getHost() + ":" + uri.getPort() : uri.getHost(); 212 try { 213 return sendOnce(request, pinnedUri, hostHeader); 214 } catch (StaleConnectionException e) { 215 throw e.asTransportException(); 216 } 217 } 218 219 // java.net.URI parses some host names (an underscore name such as my_service) with a null host, which 220 // InetAddress would resolve as the loopback address; refuse such a URI by name instead. 221 private static String requireHost(URI uri) { 222 var host = uri.getHost(); 223 if (host == null) 224 throw iaex("Request URI has no host that java.net.URI can parse, so it cannot be pinned: %s", uri); 225 return host; 226 } 227 228 // Performs a single HTTP exchange. Throws StaleConnectionException (a retryable signal) when the failure is a 229 // pre-response stale-connection failure; throws TransportException for every other failure. 230 private TransportResponse sendOnce(TransportRequest request) throws TransportException, StaleConnectionException { 231 return sendOnce(request, request.getUri(), null); 232 } 233 234 // Performs a single HTTP exchange against overrideUri (the pinned address) instead of request.getUri(), 235 // adding an explicit Host header (the original hostname[:port]) when hostHeader is non-null so virtual-hosting 236 // still works after the IP-literal rewrite. 237 private TransportResponse sendOnce(TransportRequest request, URI overrideUri, String hostHeader) throws TransportException, StaleConnectionException { 238 HttpRequest jdkRequest; 239 try { 240 jdkRequest = buildJdkRequest(request, overrideUri, hostHeader); 241 } catch (IllegalArgumentException e) { 242 // The JDK refuses a URI it cannot use (for example an underscore host name, which java.net.URI parses 243 // with a null host) or an invalid header. 244 if (hostHeader == null) 245 throw new TransportException("Could not build the HTTP request for " + request.getUri() + ": " + e.getMessage(), e); 246 // With hostHeader set, the JDK rejected the "Host" header override -- i.e. the JVM 247 // was not started with -Djdk.httpclient.allowRestrictedHeaders=host. Fail closed rather than silently pinning without preserving the original Host. 248 throw new TransportException("Could not preserve the original Host header while pin-on-connecting a " 249 + "policy-covered HTTP request (the JDK HttpClient restricts the Host header unless the JVM " 250 + "is started with the flag below); refusing (fail closed): " + request.getUri() 251 + ". Call allowHostHeaderOverride() on the transport builder (a JVM-wide setting) or set the JVM flag -Djdk.httpclient.allowRestrictedHeaders=host, use a transport with native " 252 + "connect-time pinning support (e.g. Apache HttpClient 4.5/5), or set allowPrivateUrls(true) if " 253 + "this is an intentional local-dev/intranet target.", e); 254 } 255 HttpResponse<InputStream> jdkResponse; 256 try { 257 jdkResponse = httpClient.send(jdkRequest, BodyHandlers.ofInputStream()); 258 } catch (IOException e) { 259 // httpClient.send() only returns once the response headers have been read, so any IOException here 260 // occurred before any response bytes were received. 261 if (TransportException.isStaleConnectionFailure(e)) 262 throw new StaleConnectionException(e); 263 throw new TransportException("HTTP transport error: " + e.getMessage(), e); 264 } catch (InterruptedException e) { 265 Thread.currentThread().interrupt(); 266 throw new TransportException("HTTP request interrupted", e); 267 } 268 return buildTransportResponse(jdkResponse, maxResponseBytes); 269 } 270 271 // ----------------------------------------------------------------------------------------------------------------- 272 // Internal helpers 273 // ----------------------------------------------------------------------------------------------------------------- 274 275 // Builds a URI string identical to "uri" except with its host replaced by the pinned literal address, by 276 // concatenating uri's already-percent-encoded raw components -- NOT via the multi-arg java.net.URI 277 // constructor, which treats its arguments as decoded components and would double-encode any "%" already 278 // present in the raw path/query/fragment (e.g. "%20" would become "%2520"). 279 private static String pinnedUriString(String scheme, InetAddress pinned, URI uri) { 280 var hostLiteral = pinned instanceof Inet6Address ? "[" + pinned.getHostAddress() + "]" : pinned.getHostAddress(); 281 var sb = new StringBuilder(scheme).append("://").append(hostLiteral); 282 if (uri.getPort() >= 0) 283 sb.append(':').append(uri.getPort()); 284 if (uri.getRawPath() != null) 285 sb.append(uri.getRawPath()); 286 if (uri.getRawQuery() != null) 287 sb.append('?').append(uri.getRawQuery()); 288 if (uri.getRawFragment() != null) 289 sb.append('#').append(uri.getRawFragment()); 290 return sb.toString(); 291 } 292 293 private static HttpRequest buildJdkRequest(TransportRequest request, URI uri, String hostHeader) { 294 var builder = HttpRequest.newBuilder().uri(uri).method(request.getMethod(), buildBodyPublisher(request.getBody())); 295 if (hostHeader != null) 296 builder.header("Host", hostHeader); 297 for (var h : request.getHeaders()) 298 builder.header(h.name(), h.value()); 299 var timeout = request.getTimeout(); 300 builder.timeout(timeout != null ? timeout : DEFAULT_REQUEST_TIMEOUT); 301 return builder.build(); 302 } 303 304 @SuppressWarnings({ 305 "java:S2095" // PipedInputStream closed by HttpClient when body publishing finishes; PipedOutputStream closed in writer thread try-with-resources 306 }) 307 private static BodyPublisher buildBodyPublisher(TransportBody body) { 308 if (body == null) 309 return BodyPublishers.noBody(); 310 // Use a pipe so body.writeTo() streams directly to the JDK client without full in-memory buffering. 311 // The writer runs on a daemon thread; the JDK client reads from the PipedInputStream on its own threads. 312 // The supplier runs when the client publishes the request body (not when the HttpRequest is built). 313 var publisher = BodyPublishers.ofInputStream(() -> { 314 var in = new PipedInputStream(); 315 try { 316 var out = new PipedOutputStream(in); 317 var writer = new Thread(() -> { 318 try (out) { 319 body.writeTo(out); 320 } catch (IOException e) { 321 // Closing the pipe on error causes the reader to see an IOException, 322 // which propagates to the JDK client as a send failure. 323 } 324 }, "juneau-ng-body-writer"); 325 writer.setDaemon(true); 326 writer.start(); 327 return in; 328 } catch (IOException e) { 329 try { 330 in.close(); 331 } catch (IOException e2) { 332 e.addSuppressed(e2); 333 } 334 throw new UncheckedIOException(e); 335 } 336 }); 337 // A repeatable body with a known length is sent fixed-length (Content-Length) rather than chunked; 338 // unknown-length (or non-repeatable streaming) bodies keep chunked transfer encoding. 339 var length = body.getContentLength(); 340 if (body.isRepeatable() && length >= 0) 341 return length == 0 ? BodyPublishers.noBody() : BodyPublishers.fromPublisher(publisher, length); 342 return publisher; 343 } 344 345 private static TransportResponse buildTransportResponse(HttpResponse<InputStream> jdkResponse, long maxBytes) { 346 var raw = jdkResponse.body(); 347 var body = maxBytes > 0 ? new LimitedInputStream(raw, maxBytes) : raw; 348 var builder = TransportResponse.builder() 349 .statusCode(jdkResponse.statusCode()) 350 .body(body) 351 .closeCallback(raw); 352 jdkResponse.headers().map().forEach((name, values) -> 353 values.forEach(value -> builder.header(name, value))); 354 return builder.build(); 355 } 356 357 // Fails a read once more than the configured number of body bytes has been delivered. 358 private static final class LimitedInputStream extends FilterInputStream { 359 private final long limit; 360 private long count; 361 362 LimitedInputStream(InputStream in, long limit) { 363 super(in); 364 this.limit = limit; 365 } 366 367 @Override /* InputStream */ 368 public int read() throws IOException { 369 var b = super.read(); 370 if (b >= 0) 371 add(1); 372 return b; 373 } 374 375 @Override /* InputStream */ 376 public int read(byte[] b, int off, int len) throws IOException { 377 var n = super.read(b, off, len); 378 if (n > 0) 379 add(n); 380 return n; 381 } 382 383 @Override /* InputStream */ 384 public long skip(long n) throws IOException { 385 var skipped = super.skip(n); 386 if (skipped > 0) 387 add(skipped); 388 return skipped; 389 } 390 391 private void add(long n) throws IOException { 392 count += n; 393 if (count > limit) 394 throw new IOException("Response body exceeds the configured maximum of " + limit + " bytes"); 395 } 396 } 397 398 // Internal signal that a pre-response stale-connection failure occurred and may be retried once. Carries the 399 // original cause so the caller can build the user-visible TransportException if the retry is not attempted or 400 // also fails. 401 private static final class StaleConnectionException extends Exception { 402 private static final long serialVersionUID = 1L; 403 404 StaleConnectionException(Throwable cause) { 405 super(cause); 406 } 407 408 TransportException asTransportException() { 409 return new TransportException("HTTP transport error: " + getCause().getMessage(), getCause()); 410 } 411 } 412}