2012-03-27 23:13:14 +00:00
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
2019-06-02 15:48:37 +00:00
// HTTP client. See RFC 7230 through 7235.
2014-09-21 17:33:12 +00:00
//
2012-03-27 23:13:14 +00:00
// This is the high-level Client interface.
// The low-level implementation is in transport.go.
package http
import (
2022-10-27 18:55:19 +00:00
"context"
2017-04-10 11:32:00 +00:00
"crypto/tls"
2012-03-27 23:13:14 +00:00
"encoding/base64"
"errors"
"fmt"
"io"
2014-09-21 17:33:12 +00:00
"log"
2022-10-27 18:55:19 +00:00
"net/http/internal/ascii"
2012-03-27 23:13:14 +00:00
"net/url"
2022-10-27 18:55:19 +00:00
"reflect"
2017-10-07 00:16:47 +00:00
"sort"
2012-03-27 23:13:14 +00:00
"strings"
2015-08-28 15:33:40 +00:00
"sync"
"time"
2012-03-27 23:13:14 +00:00
)
2014-09-21 17:33:12 +00:00
// A Client is an HTTP client. Its zero value (DefaultClient) is a
// usable client that uses DefaultTransport.
2012-03-27 23:13:14 +00:00
//
2014-09-21 17:33:12 +00:00
// The Client's Transport typically has internal state (cached TCP
// connections), so Clients should be reused instead of created as
2012-03-27 23:13:14 +00:00
// needed. Clients are safe for concurrent use by multiple goroutines.
2014-09-21 17:33:12 +00:00
//
// A Client is higher-level than a RoundTripper (such as Transport)
// and additionally handles HTTP details such as cookies and
// redirects.
2017-10-07 00:16:47 +00:00
//
// When following redirects, the Client will forward all headers set on the
// initial Request except:
//
2018-12-28 15:30:48 +00:00
// • when forwarding sensitive headers like "Authorization",
// "WWW-Authenticate", and "Cookie" to untrusted targets.
// These headers will be ignored when following a redirect to a domain
// that is not a subdomain match or exact match of the initial domain.
// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com"
// will forward the sensitive headers, but a redirect to "bar.com" will not.
//
// • when forwarding the "Cookie" header with a non-nil cookie Jar.
// Since each redirect may mutate the state of the cookie jar,
// a redirect may possibly alter a cookie set in the initial request.
// When forwarding the "Cookie" header, any mutated cookies will be omitted,
// with the expectation that the Jar will insert those mutated cookies
// with the updated values (assuming the origin matches).
// If Jar is nil, the initial cookies are forwarded without change.
2017-10-07 00:16:47 +00:00
//
2012-03-27 23:13:14 +00:00
type Client struct {
// Transport specifies the mechanism by which individual
// HTTP requests are made.
// If nil, DefaultTransport is used.
Transport RoundTripper
// CheckRedirect specifies the policy for handling redirects.
// If CheckRedirect is not nil, the client calls it before
2014-09-21 17:33:12 +00:00
// following an HTTP redirect. The arguments req and via are
// the upcoming request and the requests made already, oldest
// first. If CheckRedirect returns an error, the Client's Get
2017-10-07 00:16:47 +00:00
// method returns both the previous Response (with its Body
// closed) and CheckRedirect's error (wrapped in a url.Error)
// instead of issuing the Request req.
// As a special case, if CheckRedirect returns ErrUseLastResponse,
// then the most recent response is returned with its body
// unclosed, along with a nil error.
2012-03-27 23:13:14 +00:00
//
// If CheckRedirect is nil, the Client uses its default policy,
// which is to stop after 10 consecutive requests.
CheckRedirect func ( req * Request , via [ ] * Request ) error
2014-09-21 17:33:12 +00:00
// Jar specifies the cookie jar.
2017-10-07 00:16:47 +00:00
//
// The Jar is used to insert relevant cookies into every
// outbound Request and is updated with the cookie values
// of every inbound Response. The Jar is consulted for every
// redirect that the Client follows.
//
// If Jar is nil, cookies are only sent if they are explicitly
// set on the Request.
2012-03-27 23:13:14 +00:00
Jar CookieJar
2015-08-28 15:33:40 +00:00
// Timeout specifies a time limit for requests made by this
// Client. The timeout includes connection time, any
// redirects, and reading the response body. The timer remains
// running after Get, Head, Post, or Do return and will
// interrupt reading of the Response.Body.
//
// A Timeout of zero means no timeout.
//
2017-04-10 11:32:00 +00:00
// The Client cancels requests to the underlying Transport
2019-06-02 15:48:37 +00:00
// as if the Request's Context ended.
2017-04-10 11:32:00 +00:00
//
// For compatibility, the Client will also use the deprecated
// CancelRequest method on Transport if found. New
2019-06-02 15:48:37 +00:00
// RoundTripper implementations should use the Request's Context
2022-10-27 18:55:19 +00:00
// for cancellation instead of implementing CancelRequest.
2015-08-28 15:33:40 +00:00
Timeout time . Duration
2012-03-27 23:13:14 +00:00
}
// DefaultClient is the default Client and is used by Get, Head, and Post.
var DefaultClient = & Client { }
// RoundTripper is an interface representing the ability to execute a
// single HTTP transaction, obtaining the Response for a given Request.
//
// A RoundTripper must be safe for concurrent use by multiple
// goroutines.
type RoundTripper interface {
// RoundTrip executes a single HTTP transaction, returning
2017-04-10 11:32:00 +00:00
// a Response for the provided Request.
//
// RoundTrip should not attempt to interpret the response. In
// particular, RoundTrip must return err == nil if it obtained
// a response, regardless of the response's HTTP status code.
// A non-nil err should be reserved for failure to obtain a
// response. Similarly, RoundTrip should not attempt to
// handle higher-level protocol details such as redirects,
2012-03-27 23:13:14 +00:00
// authentication, or cookies.
//
// RoundTrip should not modify the request, except for
2018-12-28 15:30:48 +00:00
// consuming and closing the Request's Body. RoundTrip may
// read fields of the request in a separate goroutine. Callers
2019-06-02 15:48:37 +00:00
// should not mutate or reuse the request until the Response's
// Body has been closed.
2017-04-10 11:32:00 +00:00
//
// RoundTrip must always close the body, including on errors,
// but depending on the implementation may do so in a separate
// goroutine even after RoundTrip returns. This means that
// callers wanting to reuse the body for subsequent requests
// must arrange to wait for the Close call before doing so.
//
// The Request's URL and Header fields must be initialized.
2012-03-27 23:13:14 +00:00
RoundTrip ( * Request ) ( * Response , error )
}
2015-08-28 15:33:40 +00:00
// refererForURL returns a referer without any authentication info or
// an empty string if lastReq scheme is https and newReq scheme is http.
func refererForURL ( lastReq , newReq * url . URL ) string {
// https://tools.ietf.org/html/rfc7231#section-5.5.2
// "Clients SHOULD NOT include a Referer header field in a
// (non-secure) HTTP request if the referring page was
// transferred with a secure protocol."
if lastReq . Scheme == "https" && newReq . Scheme == "http" {
return ""
}
referer := lastReq . String ( )
if lastReq . User != nil {
// This is not very efficient, but is the best we can
// do without:
// - introducing a new method on URL
// - creating a race condition
// - copying the URL struct manually, which would cause
// maintenance problems down the line
auth := lastReq . User . String ( ) + "@"
referer = strings . Replace ( referer , auth , "" , 1 )
}
return referer
}
2017-10-07 00:16:47 +00:00
// didTimeout is non-nil only if err != nil.
func ( c * Client ) send ( req * Request , deadline time . Time ) ( resp * Response , didTimeout func ( ) bool , err error ) {
2014-09-21 17:33:12 +00:00
if c . Jar != nil {
for _ , cookie := range c . Jar . Cookies ( req . URL ) {
req . AddCookie ( cookie )
}
}
2017-10-07 00:16:47 +00:00
resp , didTimeout , err = send ( req , c . transport ( ) , deadline )
2014-09-21 17:33:12 +00:00
if err != nil {
2017-10-07 00:16:47 +00:00
return nil , didTimeout , err
2014-09-21 17:33:12 +00:00
}
if c . Jar != nil {
if rc := resp . Cookies ( ) ; len ( rc ) > 0 {
c . Jar . SetCookies ( req . URL , rc )
}
}
2017-10-07 00:16:47 +00:00
return resp , nil , nil
2017-04-10 11:32:00 +00:00
}
func ( c * Client ) deadline ( ) time . Time {
if c . Timeout > 0 {
return time . Now ( ) . Add ( c . Timeout )
}
return time . Time { }
2012-03-27 23:13:14 +00:00
}
2015-08-28 15:33:40 +00:00
func ( c * Client ) transport ( ) RoundTripper {
if c . Transport != nil {
return c . Transport
}
return DefaultTransport
}
2014-09-21 17:33:12 +00:00
// send issues an HTTP request.
// Caller should close resp.Body when done reading from it.
2017-10-07 00:16:47 +00:00
func send ( ireq * Request , rt RoundTripper , deadline time . Time ) ( resp * Response , didTimeout func ( ) bool , err error ) {
2017-04-10 11:32:00 +00:00
req := ireq // req is either the original request, or a modified fork
if rt == nil {
2015-08-28 15:33:40 +00:00
req . closeBody ( )
2017-10-07 00:16:47 +00:00
return nil , alwaysFalse , errors . New ( "http: no Client.Transport or DefaultTransport" )
2012-03-27 23:13:14 +00:00
}
if req . URL == nil {
2015-08-28 15:33:40 +00:00
req . closeBody ( )
2017-10-07 00:16:47 +00:00
return nil , alwaysFalse , errors . New ( "http: nil Request.URL" )
2012-03-27 23:13:14 +00:00
}
if req . RequestURI != "" {
2015-08-28 15:33:40 +00:00
req . closeBody ( )
2022-10-27 18:55:19 +00:00
return nil , alwaysFalse , errors . New ( "http: Request.RequestURI can't be set in client requests" )
2012-03-27 23:13:14 +00:00
}
2017-04-10 11:32:00 +00:00
// forkReq forks req into a shallow clone of ireq the first
// time it's called.
forkReq := func ( ) {
if ireq == req {
req = new ( Request )
* req = * ireq // shallow clone
}
}
2012-03-27 23:13:14 +00:00
// Most the callers of send (Get, Post, et al) don't need
2017-10-07 00:16:47 +00:00
// Headers, leaving it uninitialized. We guarantee to the
2012-03-27 23:13:14 +00:00
// Transport that this has been initialized, though.
if req . Header == nil {
2017-04-10 11:32:00 +00:00
forkReq ( )
2012-03-27 23:13:14 +00:00
req . Header = make ( Header )
}
2017-04-10 11:32:00 +00:00
if u := req . URL . User ; u != nil && req . Header . Get ( "Authorization" ) == "" {
2014-09-21 17:33:12 +00:00
username := u . Username ( )
password , _ := u . Password ( )
2017-04-10 11:32:00 +00:00
forkReq ( )
2022-10-27 18:55:19 +00:00
req . Header = cloneOrMakeHeader ( ireq . Header )
2014-09-21 17:33:12 +00:00
req . Header . Set ( "Authorization" , "Basic " + basicAuth ( username , password ) )
}
2017-04-10 11:32:00 +00:00
if ! deadline . IsZero ( ) {
forkReq ( )
}
2017-10-07 00:16:47 +00:00
stopTimer , didTimeout := setRequestCancel ( req , rt , deadline )
2017-04-10 11:32:00 +00:00
2017-10-07 00:16:47 +00:00
resp , err = rt . RoundTrip ( req )
2014-09-21 17:33:12 +00:00
if err != nil {
2017-04-10 11:32:00 +00:00
stopTimer ( )
2014-09-21 17:33:12 +00:00
if resp != nil {
log . Printf ( "RoundTripper returned a response & error; ignoring response" )
}
2017-04-10 11:32:00 +00:00
if tlsErr , ok := err . ( tls . RecordHeaderError ) ; ok {
// If we get a bad TLS record header, check to see if the
// response looks like HTTP and give a more helpful error.
// See golang.org/issue/11111.
if string ( tlsErr . RecordHeader [ : ] ) == "HTTP/" {
err = errors . New ( "http: server gave HTTP response to HTTPS client" )
}
}
2017-10-07 00:16:47 +00:00
return nil , didTimeout , err
2012-03-27 23:13:14 +00:00
}
2022-10-27 18:55:19 +00:00
if resp == nil {
return nil , didTimeout , fmt . Errorf ( "http: RoundTripper implementation (%T) returned a nil *Response with a nil error" , rt )
}
if resp . Body == nil {
// The documentation on the Body field says “The http Client and Transport
// guarantee that Body is always non-nil, even on responses without a body
// or responses with a zero-length body.” Unfortunately, we didn't document
// that same constraint for arbitrary RoundTripper implementations, and
// RoundTripper implementations in the wild (mostly in tests) assume that
// they can use a nil Body to mean an empty one (similar to Request.Body).
// (See https://golang.org/issue/38095.)
//
// If the ContentLength allows the Body to be empty, fill in an empty one
// here to ensure that it is non-nil.
if resp . ContentLength > 0 && req . Method != "HEAD" {
return nil , didTimeout , fmt . Errorf ( "http: RoundTripper implementation (%T) returned a *Response with content length %d but a nil Body" , rt , resp . ContentLength )
}
resp . Body = io . NopCloser ( strings . NewReader ( "" ) )
}
2017-04-10 11:32:00 +00:00
if ! deadline . IsZero ( ) {
resp . Body = & cancelTimerBody {
2017-10-07 00:16:47 +00:00
stop : stopTimer ,
rc : resp . Body ,
reqDidTimeout : didTimeout ,
2017-04-10 11:32:00 +00:00
}
}
2017-10-07 00:16:47 +00:00
return resp , nil , nil
2014-09-21 17:33:12 +00:00
}
2022-10-27 18:55:19 +00:00
// timeBeforeContextDeadline reports whether the non-zero Time t is
// before ctx's deadline, if any. If ctx does not have a deadline, it
// always reports true (the deadline is considered infinite).
func timeBeforeContextDeadline ( t time . Time , ctx context . Context ) bool {
d , ok := ctx . Deadline ( )
if ! ok {
return true
}
return t . Before ( d )
}
// knownRoundTripperImpl reports whether rt is a RoundTripper that's
// maintained by the Go team and known to implement the latest
// optional semantics (notably contexts). The Request is used
// to check whether this particular request is using an alternate protocol,
// in which case we need to check the RoundTripper for that protocol.
func knownRoundTripperImpl ( rt RoundTripper , req * Request ) bool {
switch t := rt . ( type ) {
case * Transport :
if altRT := t . alternateRoundTripper ( req ) ; altRT != nil {
return knownRoundTripperImpl ( altRT , req )
}
return true
case * http2Transport , http2noDialH2RoundTripper :
return true
}
// There's a very minor chance of a false positive with this.
// Instead of detecting our golang.org/x/net/http2.Transport,
// it might detect a Transport type in a different http2
// package. But I know of none, and the only problem would be
// some temporarily leaked goroutines if the transport didn't
// support contexts. So this is a good enough heuristic:
if reflect . TypeOf ( rt ) . String ( ) == "*http2.Transport" {
return true
}
return false
}
// setRequestCancel sets req.Cancel and adds a deadline context to req
// if deadline is non-zero. The RoundTripper's type is used to
// determine whether the legacy CancelRequest behavior should be used.
2017-10-07 00:16:47 +00:00
//
// As background, there are three ways to cancel a request:
// First was Transport.CancelRequest. (deprecated)
2022-10-27 18:55:19 +00:00
// Second was Request.Cancel.
2017-10-07 00:16:47 +00:00
// Third was Request.Context.
2022-10-27 18:55:19 +00:00
// This function populates the second and third, and uses the first if it really needs to.
2017-10-07 00:16:47 +00:00
func setRequestCancel ( req * Request , rt RoundTripper , deadline time . Time ) ( stopTimer func ( ) , didTimeout func ( ) bool ) {
2017-04-10 11:32:00 +00:00
if deadline . IsZero ( ) {
return nop , alwaysFalse
}
2022-10-27 18:55:19 +00:00
knownTransport := knownRoundTripperImpl ( rt , req )
oldCtx := req . Context ( )
if req . Cancel == nil && knownTransport {
// If they already had a Request.Context that's
// expiring sooner, do nothing:
if ! timeBeforeContextDeadline ( deadline , oldCtx ) {
return nop , alwaysFalse
}
2017-04-10 11:32:00 +00:00
2022-10-27 18:55:19 +00:00
var cancelCtx func ( )
req . ctx , cancelCtx = context . WithDeadline ( oldCtx , deadline )
return cancelCtx , func ( ) bool { return time . Now ( ) . After ( deadline ) }
}
2017-04-10 11:32:00 +00:00
initialReqCancel := req . Cancel // the user's original Request.Cancel, if any
2022-10-27 18:55:19 +00:00
var cancelCtx func ( )
if oldCtx := req . Context ( ) ; timeBeforeContextDeadline ( deadline , oldCtx ) {
req . ctx , cancelCtx = context . WithDeadline ( oldCtx , deadline )
}
2017-04-10 11:32:00 +00:00
cancel := make ( chan struct { } )
req . Cancel = cancel
doCancel := func ( ) {
2022-10-27 18:55:19 +00:00
// The second way in the func comment above:
2017-04-10 11:32:00 +00:00
close ( cancel )
2022-10-27 18:55:19 +00:00
// The first way, used only for RoundTripper
// implementations written before Go 1.5 or Go 1.6.
type canceler interface { CancelRequest ( * Request ) }
if v , ok := rt . ( canceler ) ; ok {
2017-04-10 11:32:00 +00:00
v . CancelRequest ( req )
}
}
stopTimerCh := make ( chan struct { } )
var once sync . Once
2022-10-27 18:55:19 +00:00
stopTimer = func ( ) {
once . Do ( func ( ) {
close ( stopTimerCh )
if cancelCtx != nil {
cancelCtx ( )
}
} )
}
2017-04-10 11:32:00 +00:00
2017-10-07 00:16:47 +00:00
timer := time . NewTimer ( time . Until ( deadline ) )
var timedOut atomicBool
2017-04-10 11:32:00 +00:00
go func ( ) {
select {
case <- initialReqCancel :
doCancel ( )
2017-10-07 00:16:47 +00:00
timer . Stop ( )
2017-04-10 11:32:00 +00:00
case <- timer . C :
2017-10-07 00:16:47 +00:00
timedOut . setTrue ( )
2017-04-10 11:32:00 +00:00
doCancel ( )
case <- stopTimerCh :
timer . Stop ( )
}
} ( )
2017-10-07 00:16:47 +00:00
return stopTimer , timedOut . isSet
2017-04-10 11:32:00 +00:00
}
2019-06-02 15:48:37 +00:00
// See 2 (end of page 4) https://www.ietf.org/rfc/rfc2617.txt
2014-09-21 17:33:12 +00:00
// "To receive authorization, the client sends the userid and password,
// separated by a single colon (":") character, within a base64
// encoded string in the credentials."
// It is not meant to be urlencoded.
func basicAuth ( username , password string ) string {
auth := username + ":" + password
return base64 . StdEncoding . EncodeToString ( [ ] byte ( auth ) )
2012-03-27 23:13:14 +00:00
}
2017-04-10 11:32:00 +00:00
// Get issues a GET to the specified URL. If the response is one of
// the following redirect codes, Get follows the redirect, up to a
// maximum of 10 redirects:
2012-03-27 23:13:14 +00:00
//
// 301 (Moved Permanently)
// 302 (Found)
// 303 (See Other)
// 307 (Temporary Redirect)
2017-10-07 00:16:47 +00:00
// 308 (Permanent Redirect)
2012-03-27 23:13:14 +00:00
//
2014-09-21 17:33:12 +00:00
// An error is returned if there were too many redirects or if there
// was an HTTP protocol error. A non-2xx response doesn't cause an
2019-06-02 15:48:37 +00:00
// error. Any returned error will be of type *url.Error. The url.Error
2022-10-27 18:55:19 +00:00
// value's Timeout method will report true if the request timed out.
2014-09-21 17:33:12 +00:00
//
// When err is nil, resp always contains a non-nil resp.Body.
// Caller should close resp.Body when done reading from it.
2012-03-27 23:13:14 +00:00
//
// Get is a wrapper around DefaultClient.Get.
2017-04-10 11:32:00 +00:00
//
// To make a request with custom headers, use NewRequest and
// DefaultClient.Do.
2022-10-27 18:55:19 +00:00
//
// To make a request with a specified context.Context, use NewRequestWithContext
// and DefaultClient.Do.
2014-09-21 17:33:12 +00:00
func Get ( url string ) ( resp * Response , err error ) {
2012-03-27 23:13:14 +00:00
return DefaultClient . Get ( url )
}
2017-04-10 11:32:00 +00:00
// Get issues a GET to the specified URL. If the response is one of the
2012-03-27 23:13:14 +00:00
// following redirect codes, Get follows the redirect after calling the
2017-04-10 11:32:00 +00:00
// Client's CheckRedirect function:
2012-03-27 23:13:14 +00:00
//
// 301 (Moved Permanently)
// 302 (Found)
// 303 (See Other)
// 307 (Temporary Redirect)
2017-10-07 00:16:47 +00:00
// 308 (Permanent Redirect)
2012-03-27 23:13:14 +00:00
//
2014-09-21 17:33:12 +00:00
// An error is returned if the Client's CheckRedirect function fails
// or if there was an HTTP protocol error. A non-2xx response doesn't
2019-06-02 15:48:37 +00:00
// cause an error. Any returned error will be of type *url.Error. The
2022-10-27 18:55:19 +00:00
// url.Error value's Timeout method will report true if the request
// timed out.
2014-09-21 17:33:12 +00:00
//
// When err is nil, resp always contains a non-nil resp.Body.
// Caller should close resp.Body when done reading from it.
2017-04-10 11:32:00 +00:00
//
// To make a request with custom headers, use NewRequest and Client.Do.
2022-10-27 18:55:19 +00:00
//
// To make a request with a specified context.Context, use NewRequestWithContext
// and Client.Do.
2014-09-21 17:33:12 +00:00
func ( c * Client ) Get ( url string ) ( resp * Response , err error ) {
2012-03-27 23:13:14 +00:00
req , err := NewRequest ( "GET" , url , nil )
if err != nil {
return nil , err
}
2017-10-07 00:16:47 +00:00
return c . Do ( req )
2012-03-27 23:13:14 +00:00
}
2017-04-10 11:32:00 +00:00
func alwaysFalse ( ) bool { return false }
2017-10-07 00:16:47 +00:00
// ErrUseLastResponse can be returned by Client.CheckRedirect hooks to
// control how redirects are processed. If returned, the next request
// is not sent and the most recent response is returned with its body
// unclosed.
var ErrUseLastResponse = errors . New ( "net/http: use last response" )
// checkRedirect calls either the user's configured CheckRedirect
// function, or the default.
func ( c * Client ) checkRedirect ( req * Request , via [ ] * Request ) error {
fn := c . CheckRedirect
if fn == nil {
fn = defaultCheckRedirect
2012-03-27 23:13:14 +00:00
}
2017-10-07 00:16:47 +00:00
return fn ( req , via )
}
2012-03-27 23:13:14 +00:00
2017-10-07 00:16:47 +00:00
// redirectBehavior describes what should happen when the
// client encounters a 3xx status code from the server
func redirectBehavior ( reqMethod string , resp * Response , ireq * Request ) ( redirectMethod string , shouldRedirect , includeBody bool ) {
switch resp . StatusCode {
case 301 , 302 , 303 :
redirectMethod = reqMethod
shouldRedirect = true
includeBody = false
// RFC 2616 allowed automatic redirection only with GET and
// HEAD requests. RFC 7231 lifts this restriction, but we still
// restrict other methods to GET to maintain compatibility.
// See Issue 18570.
if reqMethod != "GET" && reqMethod != "HEAD" {
redirectMethod = "GET"
}
case 307 , 308 :
redirectMethod = reqMethod
shouldRedirect = true
includeBody = true
// Treat 307 and 308 specially, since they're new in
// Go 1.8, and they also require re-sending the request body.
if resp . Header . Get ( "Location" ) == "" {
// 308s have been observed in the wild being served
// without Location headers. Since Go 1.7 and earlier
// didn't follow these codes, just stop here instead
// of returning an error.
// See Issue 17773.
shouldRedirect = false
break
}
if ireq . GetBody == nil && ireq . outgoingLength ( ) != 0 {
// We had a request body, and 307/308 require
// re-sending it, but GetBody is not defined. So just
// return this response to the user instead of an
// error, like we did in Go 1.7 and earlier.
shouldRedirect = false
}
}
return redirectMethod , shouldRedirect , includeBody
}
2019-06-02 15:48:37 +00:00
// urlErrorOp returns the (*url.Error).Op value to use for the
// provided (*Request).Method value.
func urlErrorOp ( method string ) string {
if method == "" {
return "Get"
}
2022-10-27 18:55:19 +00:00
if lowerMethod , ok := ascii . ToLower ( method ) ; ok {
return method [ : 1 ] + lowerMethod [ 1 : ]
}
return method
2019-06-02 15:48:37 +00:00
}
2017-10-07 00:16:47 +00:00
// Do sends an HTTP request and returns an HTTP response, following
// policy (such as redirects, cookies, auth) as configured on the
// client.
//
// An error is returned if caused by client policy (such as
// CheckRedirect), or failure to speak HTTP (such as a network
// connectivity problem). A non-2xx status code doesn't cause an
// error.
//
// If the returned error is nil, the Response will contain a non-nil
2019-06-02 15:48:37 +00:00
// Body which the user is expected to close. If the Body is not both
// read to EOF and closed, the Client's underlying RoundTripper
// (typically Transport) may not be able to re-use a persistent TCP
// connection to the server for a subsequent "keep-alive" request.
2017-10-07 00:16:47 +00:00
//
// The request Body, if non-nil, will be closed by the underlying
// Transport, even on errors.
//
// On error, any Response can be ignored. A non-nil Response with a
// non-nil error only occurs when CheckRedirect fails, and even then
// the returned Response.Body is already closed.
//
// Generally Get, Post, or PostForm will be used instead of Do.
//
// If the server replies with a redirect, the Client first uses the
// CheckRedirect function to determine whether the redirect should be
// followed. If permitted, a 301, 302, or 303 redirect causes
// subsequent requests to use HTTP method GET
// (or HEAD if the original request was HEAD), with no body.
// A 307 or 308 redirect preserves the original HTTP method and body,
// provided that the Request.GetBody function is defined.
// The NewRequest function automatically sets GetBody for common
// standard library body types.
2019-06-02 15:48:37 +00:00
//
// Any returned error will be of type *url.Error. The url.Error
2022-10-27 18:55:19 +00:00
// value's Timeout method will report true if the request timed out.
2017-10-07 00:16:47 +00:00
func ( c * Client ) Do ( req * Request ) ( * Response , error ) {
2019-06-02 15:48:37 +00:00
return c . do ( req )
}
var testHookClientDoResult func ( retres * Response , reterr error )
func ( c * Client ) do ( req * Request ) ( retres * Response , reterr error ) {
if testHookClientDoResult != nil {
defer func ( ) { testHookClientDoResult ( retres , reterr ) } ( )
}
2017-10-07 00:16:47 +00:00
if req . URL == nil {
req . closeBody ( )
2019-06-02 15:48:37 +00:00
return nil , & url . Error {
Op : urlErrorOp ( req . Method ) ,
Err : errors . New ( "http: nil Request.URL" ) ,
}
2012-03-27 23:13:14 +00:00
}
2017-10-07 00:16:47 +00:00
var (
2018-12-28 15:30:48 +00:00
deadline = c . deadline ( )
reqs [ ] * Request
resp * Response
copyHeaders = c . makeHeadersCopier ( req )
reqBodyClosed = false // have we closed the current req.Body?
2015-08-28 15:33:40 +00:00
2017-10-07 00:16:47 +00:00
// Redirect behavior:
redirectMethod string
includeBody bool
)
uerr := func ( err error ) error {
2018-12-28 15:30:48 +00:00
// the body may have been closed already by c.send()
if ! reqBodyClosed {
req . closeBody ( )
}
2017-10-07 00:16:47 +00:00
var urlStr string
if resp != nil && resp . Request != nil {
2019-06-02 15:48:37 +00:00
urlStr = stripPassword ( resp . Request . URL )
2017-10-07 00:16:47 +00:00
} else {
2019-06-02 15:48:37 +00:00
urlStr = stripPassword ( req . URL )
2017-10-07 00:16:47 +00:00
}
return & url . Error {
2019-06-02 15:48:37 +00:00
Op : urlErrorOp ( reqs [ 0 ] . Method ) ,
2017-10-07 00:16:47 +00:00
URL : urlStr ,
Err : err ,
}
}
for {
// For all but the first request, create the next
// request hop and replace req.
if len ( reqs ) > 0 {
loc := resp . Header . Get ( "Location" )
if loc == "" {
2018-12-28 15:30:48 +00:00
resp . closeBody ( )
2017-10-07 00:16:47 +00:00
return nil , uerr ( fmt . Errorf ( "%d response missing Location header" , resp . StatusCode ) )
2014-09-21 17:33:12 +00:00
}
2017-10-07 00:16:47 +00:00
u , err := req . URL . Parse ( loc )
2012-03-27 23:13:14 +00:00
if err != nil {
2018-12-28 15:30:48 +00:00
resp . closeBody ( )
2017-10-07 00:16:47 +00:00
return nil , uerr ( fmt . Errorf ( "failed to parse Location header %q: %v" , loc , err ) )
2012-03-27 23:13:14 +00:00
}
2018-12-28 15:30:48 +00:00
host := ""
if req . Host != "" && req . Host != req . URL . Host {
// If the caller specified a custom Host header and the
// redirect location is relative, preserve the Host header
// through the redirect. See issue #22233.
if u , _ := url . Parse ( loc ) ; u != nil && ! u . IsAbs ( ) {
host = req . Host
}
}
2017-10-07 00:16:47 +00:00
ireq := reqs [ 0 ]
req = & Request {
Method : redirectMethod ,
Response : resp ,
URL : u ,
Header : make ( Header ) ,
2018-12-28 15:30:48 +00:00
Host : host ,
2017-10-07 00:16:47 +00:00
Cancel : ireq . Cancel ,
ctx : ireq . ctx ,
}
if includeBody && ireq . GetBody != nil {
req . Body , err = ireq . GetBody ( )
2012-03-27 23:13:14 +00:00
if err != nil {
2018-12-28 15:30:48 +00:00
resp . closeBody ( )
2017-10-07 00:16:47 +00:00
return nil , uerr ( err )
2012-03-27 23:13:14 +00:00
}
2017-10-07 00:16:47 +00:00
req . ContentLength = ireq . ContentLength
2012-03-27 23:13:14 +00:00
}
2017-10-07 00:16:47 +00:00
// Copy original headers before setting the Referer,
// in case the user set Referer on their first request.
// If they really want to override, they can do it in
// their CheckRedirect func.
copyHeaders ( req )
// Add the Referer header from the most recent
// request URL to the new one, if it's not https->http:
if ref := refererForURL ( reqs [ len ( reqs ) - 1 ] . URL , req . URL ) ; ref != "" {
req . Header . Set ( "Referer" , ref )
}
err = c . checkRedirect ( req , reqs )
// Sentinel error to let users select the
// previous response, without closing its
// body. See Issue 10069.
if err == ErrUseLastResponse {
return resp , nil
2017-04-10 11:32:00 +00:00
}
2012-03-27 23:13:14 +00:00
2017-10-07 00:16:47 +00:00
// Close the previous response's body. But
// read at least some of the body so if it's
// small the underlying TCP connection will be
// re-used. No need to check for errors: if it
// fails, the Transport won't reuse it anyway.
2015-08-28 15:33:40 +00:00
const maxBodySlurpSize = 2 << 10
if resp . ContentLength == - 1 || resp . ContentLength <= maxBodySlurpSize {
2022-10-27 18:55:19 +00:00
io . CopyN ( io . Discard , resp . Body , maxBodySlurpSize )
2015-08-28 15:33:40 +00:00
}
2014-09-21 17:33:12 +00:00
resp . Body . Close ( )
2017-10-07 00:16:47 +00:00
if err != nil {
// Special case for Go 1 compatibility: return both the response
// and an error if the CheckRedirect function failed.
// See https://golang.org/issue/3795
// The resp.Body has already been closed.
ue := uerr ( err )
ue . ( * url . Error ) . URL = loc
return resp , ue
2012-03-27 23:13:14 +00:00
}
}
2017-10-07 00:16:47 +00:00
reqs = append ( reqs , req )
var err error
var didTimeout func ( ) bool
if resp , didTimeout , err = c . send ( req , deadline ) ; err != nil {
2018-12-28 15:30:48 +00:00
// c.send() always closes req.Body
reqBodyClosed = true
2017-10-07 00:16:47 +00:00
if ! deadline . IsZero ( ) && didTimeout ( ) {
err = & httpError {
err : err . Error ( ) + " (Client.Timeout exceeded while awaiting headers)" ,
timeout : true ,
}
}
return nil , uerr ( err )
}
var shouldRedirect bool
redirectMethod , shouldRedirect , includeBody = redirectBehavior ( req . Method , resp , reqs [ 0 ] )
if ! shouldRedirect {
return resp , nil
}
req . closeBody ( )
2012-03-27 23:13:14 +00:00
}
2017-10-07 00:16:47 +00:00
}
2014-09-21 17:33:12 +00:00
2017-10-07 00:16:47 +00:00
// makeHeadersCopier makes a function that copies headers from the
// initial Request, ireq. For every redirect, this function must be called
// so that it can copy headers into the upcoming Request.
func ( c * Client ) makeHeadersCopier ( ireq * Request ) func ( * Request ) {
// The headers to copy are from the very initial request.
// We use a closured callback to keep a reference to these original headers.
var (
2022-10-27 18:55:19 +00:00
ireqhdr = cloneOrMakeHeader ( ireq . Header )
2017-10-07 00:16:47 +00:00
icookies map [ string ] [ ] * Cookie
)
if c . Jar != nil && ireq . Header . Get ( "Cookie" ) != "" {
icookies = make ( map [ string ] [ ] * Cookie )
for _ , c := range ireq . Cookies ( ) {
icookies [ c . Name ] = append ( icookies [ c . Name ] , c )
}
2014-09-21 17:33:12 +00:00
}
2017-10-07 00:16:47 +00:00
preq := ireq // The previous request
return func ( req * Request ) {
// If Jar is present and there was some initial cookies provided
// via the request header, then we may need to alter the initial
// cookies as we follow redirects since each redirect may end up
// modifying a pre-existing cookie.
//
// Since cookies already set in the request header do not contain
// information about the original domain and path, the logic below
// assumes any new set cookies override the original cookie
// regardless of domain or path.
//
// See https://golang.org/issue/17494
if c . Jar != nil && icookies != nil {
var changed bool
resp := req . Response // The response that caused the upcoming redirect
for _ , c := range resp . Cookies ( ) {
if _ , ok := icookies [ c . Name ] ; ok {
delete ( icookies , c . Name )
changed = true
}
}
if changed {
ireqhdr . Del ( "Cookie" )
var ss [ ] string
for _ , cs := range icookies {
for _ , c := range cs {
ss = append ( ss , c . Name + "=" + c . Value )
}
}
sort . Strings ( ss ) // Ensure deterministic headers
ireqhdr . Set ( "Cookie" , strings . Join ( ss , "; " ) )
}
}
// Copy the initial request's Header values
// (at least the safe ones).
for k , vv := range ireqhdr {
if shouldCopyHeaderOnRedirect ( k , preq . URL , req . URL ) {
req . Header [ k ] = vv
}
}
preq = req // Update previous Request with the current request
2014-09-21 17:33:12 +00:00
}
2012-03-27 23:13:14 +00:00
}
func defaultCheckRedirect ( req * Request , via [ ] * Request ) error {
if len ( via ) >= 10 {
return errors . New ( "stopped after 10 redirects" )
}
return nil
}
// Post issues a POST to the specified URL.
//
2014-09-21 17:33:12 +00:00
// Caller should close resp.Body when done reading from it.
2012-03-27 23:13:14 +00:00
//
2017-04-10 11:32:00 +00:00
// If the provided body is an io.Closer, it is closed after the
// request.
//
// Post is a wrapper around DefaultClient.Post.
//
// To set custom headers, use NewRequest and DefaultClient.Do.
2017-10-07 00:16:47 +00:00
//
// See the Client.Do method documentation for details on how redirects
// are handled.
2022-10-27 18:55:19 +00:00
//
// To make a request with a specified context.Context, use NewRequestWithContext
// and DefaultClient.Do.
2019-06-02 15:48:37 +00:00
func Post ( url , contentType string , body io . Reader ) ( resp * Response , err error ) {
2017-10-07 00:16:47 +00:00
return DefaultClient . Post ( url , contentType , body )
2012-03-27 23:13:14 +00:00
}
// Post issues a POST to the specified URL.
//
2014-09-21 17:33:12 +00:00
// Caller should close resp.Body when done reading from it.
//
2017-04-10 11:32:00 +00:00
// If the provided body is an io.Closer, it is closed after the
2015-08-28 15:33:40 +00:00
// request.
2017-04-10 11:32:00 +00:00
//
// To set custom headers, use NewRequest and Client.Do.
2017-10-07 00:16:47 +00:00
//
2022-10-27 18:55:19 +00:00
// To make a request with a specified context.Context, use NewRequestWithContext
// and Client.Do.
//
2017-10-07 00:16:47 +00:00
// See the Client.Do method documentation for details on how redirects
// are handled.
2019-06-02 15:48:37 +00:00
func ( c * Client ) Post ( url , contentType string , body io . Reader ) ( resp * Response , err error ) {
2012-03-27 23:13:14 +00:00
req , err := NewRequest ( "POST" , url , body )
if err != nil {
return nil , err
}
2017-10-07 00:16:47 +00:00
req . Header . Set ( "Content-Type" , contentType )
return c . Do ( req )
2012-03-27 23:13:14 +00:00
}
2014-09-21 17:33:12 +00:00
// PostForm issues a POST to the specified URL, with data's keys and
// values URL-encoded as the request body.
2012-03-27 23:13:14 +00:00
//
2017-04-10 11:32:00 +00:00
// The Content-Type header is set to application/x-www-form-urlencoded.
// To set other headers, use NewRequest and DefaultClient.Do.
//
2014-09-21 17:33:12 +00:00
// When err is nil, resp always contains a non-nil resp.Body.
// Caller should close resp.Body when done reading from it.
2012-03-27 23:13:14 +00:00
//
2017-04-10 11:32:00 +00:00
// PostForm is a wrapper around DefaultClient.PostForm.
2017-10-07 00:16:47 +00:00
//
// See the Client.Do method documentation for details on how redirects
// are handled.
2022-10-27 18:55:19 +00:00
//
// To make a request with a specified context.Context, use NewRequestWithContext
// and DefaultClient.Do.
2014-09-21 17:33:12 +00:00
func PostForm ( url string , data url . Values ) ( resp * Response , err error ) {
2012-03-27 23:13:14 +00:00
return DefaultClient . PostForm ( url , data )
}
2014-09-21 17:33:12 +00:00
// PostForm issues a POST to the specified URL,
2017-04-10 11:32:00 +00:00
// with data's keys and values URL-encoded as the request body.
//
// The Content-Type header is set to application/x-www-form-urlencoded.
2018-12-28 15:30:48 +00:00
// To set other headers, use NewRequest and Client.Do.
2012-03-27 23:13:14 +00:00
//
2014-09-21 17:33:12 +00:00
// When err is nil, resp always contains a non-nil resp.Body.
// Caller should close resp.Body when done reading from it.
2017-10-07 00:16:47 +00:00
//
// See the Client.Do method documentation for details on how redirects
// are handled.
2022-10-27 18:55:19 +00:00
//
// To make a request with a specified context.Context, use NewRequestWithContext
// and Client.Do.
2014-09-21 17:33:12 +00:00
func ( c * Client ) PostForm ( url string , data url . Values ) ( resp * Response , err error ) {
2012-03-27 23:13:14 +00:00
return c . Post ( url , "application/x-www-form-urlencoded" , strings . NewReader ( data . Encode ( ) ) )
}
2017-10-07 00:16:47 +00:00
// Head issues a HEAD to the specified URL. If the response is one of
2017-04-10 11:32:00 +00:00
// the following redirect codes, Head follows the redirect, up to a
// maximum of 10 redirects:
2012-03-27 23:13:14 +00:00
//
// 301 (Moved Permanently)
// 302 (Found)
// 303 (See Other)
// 307 (Temporary Redirect)
2017-10-07 00:16:47 +00:00
// 308 (Permanent Redirect)
2012-03-27 23:13:14 +00:00
//
// Head is a wrapper around DefaultClient.Head
2022-10-27 18:55:19 +00:00
//
// To make a request with a specified context.Context, use NewRequestWithContext
// and DefaultClient.Do.
2014-09-21 17:33:12 +00:00
func Head ( url string ) ( resp * Response , err error ) {
2012-03-27 23:13:14 +00:00
return DefaultClient . Head ( url )
}
2017-10-07 00:16:47 +00:00
// Head issues a HEAD to the specified URL. If the response is one of the
2012-03-27 23:13:14 +00:00
// following redirect codes, Head follows the redirect after calling the
2017-04-10 11:32:00 +00:00
// Client's CheckRedirect function:
2012-03-27 23:13:14 +00:00
//
// 301 (Moved Permanently)
// 302 (Found)
// 303 (See Other)
// 307 (Temporary Redirect)
2017-10-07 00:16:47 +00:00
// 308 (Permanent Redirect)
2022-10-27 18:55:19 +00:00
//
// To make a request with a specified context.Context, use NewRequestWithContext
// and Client.Do.
2014-09-21 17:33:12 +00:00
func ( c * Client ) Head ( url string ) ( resp * Response , err error ) {
2012-03-27 23:13:14 +00:00
req , err := NewRequest ( "HEAD" , url , nil )
if err != nil {
return nil , err
}
2017-10-07 00:16:47 +00:00
return c . Do ( req )
2012-03-27 23:13:14 +00:00
}
2015-08-28 15:33:40 +00:00
2019-06-02 15:48:37 +00:00
// CloseIdleConnections closes any connections on its Transport which
// were previously connected from previous requests but are now
// sitting idle in a "keep-alive" state. It does not interrupt any
// connections currently in use.
//
// If the Client's Transport does not have a CloseIdleConnections method
// then this method does nothing.
func ( c * Client ) CloseIdleConnections ( ) {
type closeIdler interface {
CloseIdleConnections ( )
}
if tr , ok := c . transport ( ) . ( closeIdler ) ; ok {
tr . CloseIdleConnections ( )
}
}
2017-04-10 11:32:00 +00:00
// cancelTimerBody is an io.ReadCloser that wraps rc with two features:
2022-10-27 18:55:19 +00:00
// 1) On Read error or close, the stop func is called.
2017-10-07 00:16:47 +00:00
// 2) On Read failure, if reqDidTimeout is true, the error is wrapped and
2017-04-10 11:32:00 +00:00
// marked as net.Error that hit its timeout.
2015-08-28 15:33:40 +00:00
type cancelTimerBody struct {
2017-10-07 00:16:47 +00:00
stop func ( ) // stops the time.Timer waiting to cancel the request
rc io . ReadCloser
reqDidTimeout func ( ) bool
2015-08-28 15:33:40 +00:00
}
func ( b * cancelTimerBody ) Read ( p [ ] byte ) ( n int , err error ) {
n , err = b . rc . Read ( p )
2017-04-10 11:32:00 +00:00
if err == nil {
return n , nil
}
2015-08-28 15:33:40 +00:00
if err == io . EOF {
2017-04-10 11:32:00 +00:00
return n , err
}
2017-10-07 00:16:47 +00:00
if b . reqDidTimeout ( ) {
2017-04-10 11:32:00 +00:00
err = & httpError {
2022-10-27 18:55:19 +00:00
err : err . Error ( ) + " (Client.Timeout or context cancellation while reading body)" ,
2017-04-10 11:32:00 +00:00
timeout : true ,
}
2015-08-28 15:33:40 +00:00
}
2017-04-10 11:32:00 +00:00
return n , err
2015-08-28 15:33:40 +00:00
}
func ( b * cancelTimerBody ) Close ( ) error {
err := b . rc . Close ( )
2017-04-10 11:32:00 +00:00
b . stop ( )
2015-08-28 15:33:40 +00:00
return err
}
2017-10-07 00:16:47 +00:00
func shouldCopyHeaderOnRedirect ( headerKey string , initial , dest * url . URL ) bool {
switch CanonicalHeaderKey ( headerKey ) {
case "Authorization" , "Www-Authenticate" , "Cookie" , "Cookie2" :
// Permit sending auth/cookie headers from "foo.com"
// to "sub.foo.com".
// Note that we don't send all cookies to subdomains
// automatically. This function is only used for
// Cookies set explicitly on the initial outgoing
// client request. Cookies automatically added via the
// CookieJar mechanism continue to follow each
// cookie's scope as set by Set-Cookie. But for
// outgoing requests with the Cookie header set
// directly, we don't know their scope, so we assume
// it's for *.domain.com.
2018-12-28 15:30:48 +00:00
ihost := canonicalAddr ( initial )
dhost := canonicalAddr ( dest )
2017-10-07 00:16:47 +00:00
return isDomainOrSubdomain ( dhost , ihost )
}
// All other headers are copied:
return true
}
// isDomainOrSubdomain reports whether sub is a subdomain (or exact
// match) of the parent domain.
//
// Both domains must already be in canonical form.
func isDomainOrSubdomain ( sub , parent string ) bool {
if sub == parent {
return true
}
// If sub is "foo.example.com" and parent is "example.com",
// that means sub must end in "."+parent.
// Do it without allocating.
if ! strings . HasSuffix ( sub , parent ) {
return false
}
return sub [ len ( sub ) - len ( parent ) - 1 ] == '.'
}
2019-06-02 15:48:37 +00:00
func stripPassword ( u * url . URL ) string {
2022-10-27 18:55:19 +00:00
_ , passSet := u . User . Password ( )
2019-06-02 15:48:37 +00:00
if passSet {
2022-10-27 18:55:19 +00:00
return strings . Replace ( u . String ( ) , u . User . String ( ) + "@" , u . User . Username ( ) + ":***@" , 1 )
2019-06-02 15:48:37 +00:00
}
return u . String ( )
}