wekan/packages/wekan-request/request.js

var optional = require('./lib/optional')
  , http = require('http')
  , https = optional('https')
  , tls = optional('tls')
  , url = require('url')
  , util = require('util')
  , stream = require('stream')
  , qs = require('qs')
  , querystring = require('querystring')
  , crypto = require('crypto')
  , zlib = require('zlib')

  , oauth = optional('oauth-sign')
  , hawk = optional('hawk')
  , aws = optional('aws-sign2')
  , httpSignature = optional('http-signature')
  , uuid = require('node-uuid')
  , mime = require('mime-types')
  , tunnel = optional('tunnel-agent')
  , _safeStringify = require('json-stringify-safe')
  , stringstream = optional('stringstream')

  , ForeverAgent = require('forever-agent')
  , FormData = optional('form-data')

  , cookies = require('./lib/cookies')
  , globalCookieJar = cookies.jar()

  , copy = require('./lib/copy')
  , debug = require('./lib/debug')
  , getSafe = require('./lib/getSafe')
  , net = require('net')
  ;

function safeStringify (obj) {
  var ret
  try { ret = JSON.stringify(obj) }
  catch (e) { ret = _safeStringify(obj) }
  return ret
}

var globalPool = {}
var isUrl = /^https?:|^unix:/


// Hacky fix for pre-0.4.4 https
if (https && !https.Agent) {
  https.Agent = function (options) {
    http.Agent.call(this, options)
  }
  util.inherits(https.Agent, http.Agent)
  https.Agent.prototype._getConnection = function (host, port, cb) {
    var s = tls.connect(port, host, this.options, function () {
      // do other checks here?
      if (cb) cb()
    })
    return s
  }
}

function isReadStream (rs) {
  return rs.readable && rs.path && rs.mode;
}

function toBase64 (str) {
  return (new Buffer(str || "", "ascii")).toString("base64")
}

function md5 (str) {
  return crypto.createHash('md5').update(str).digest('hex')
}

function Request (options) {
  stream.Stream.call(this)
  this.readable = true
  this.writable = true

  if (typeof options === 'string') {
    options = {uri:options}
  }

  var reserved = Object.keys(Request.prototype)
  for (var i in options) {
    if (reserved.indexOf(i) === -1) {
      this[i] = options[i]
    } else {
      if (typeof options[i] === 'function') {
        delete options[i]
      }
    }
  }

  if (options.method) {
    this.explicitMethod = true
  }

  this.canTunnel = options.tunnel !== false && tunnel;

  this.init(options)
}
util.inherits(Request, stream.Stream)
Request.prototype.init = function (options) {
  // init() contains all the code to setup the request object.
  // the actual outgoing request is not started until start() is called
  // this function is called from both the constructor and on redirect.
  var self = this
  if (!options) options = {}

  if (!self.method) self.method = options.method || 'GET'
  self.localAddress = options.localAddress

  debug(options)
  if (!self.pool && self.pool !== false) self.pool = globalPool
  self.dests = self.dests || []
  self.__isRequestRequest = true

  // Protect against double callback
  if (!self._callback && self.callback) {
    self._callback = self.callback
    self.callback = function () {
      if (self._callbackCalled) return // Print a warning maybe?
      self._callbackCalled = true
      self._callback.apply(self, arguments)
    }
    self.on('error', self.callback.bind())
    self.on('complete', self.callback.bind(self, null))
  }

  if (self.url && !self.uri) {
    // People use this property instead all the time so why not just support it.
    self.uri = self.url
    delete self.url
  }

  if (!self.uri) {
    // this will throw if unhandled but is handleable when in a redirect
    return self.emit('error', new Error("options.uri is a required argument"))
  } else {
    if (typeof self.uri == "string") self.uri = url.parse(self.uri)
  }

  if (self.strictSSL === false) {
    self.rejectUnauthorized = false
  }

  if(!self.hasOwnProperty('proxy')) {
    // check for HTTP(S)_PROXY environment variables
    if(self.uri.protocol == "http:") {
        self.proxy = process.env.HTTP_PROXY || process.env.http_proxy || null;
    } else if(self.uri.protocol == "https:") {
        self.proxy = process.env.HTTPS_PROXY || process.env.https_proxy || 
                     process.env.HTTP_PROXY || process.env.http_proxy || null;
    }
  }
  
  if (self.proxy) {
    if (typeof self.proxy == 'string') self.proxy = url.parse(self.proxy)

    // do the HTTP CONNECT dance using koichik/node-tunnel
    if (http.globalAgent && self.uri.protocol === "https:" && self.canTunnel) {
      var tunnelFn = self.proxy.protocol === "http:"
                   ? tunnel.httpsOverHttp : tunnel.httpsOverHttps

      var tunnelOptions = { proxy: { host: self.proxy.hostname
                                   , port: +self.proxy.port
                                   , proxyAuth: self.proxy.auth
                                   , headers: { Host: self.uri.hostname + ':' +
                                        (self.uri.port || self.uri.protocol === 'https:' ? 443 : 80) }}
                          , rejectUnauthorized: self.rejectUnauthorized
                          , ca: this.ca
                          , cert:this.cert
                          , key: this.key}

      self.agent = tunnelFn(tunnelOptions)
      self.tunnel = true
    }
  }

  if (!self.uri.pathname) {self.uri.pathname = '/'}

  if (!self.uri.host && !self.protocol=='unix:') {
    // Invalid URI: it may generate lot of bad errors, like "TypeError: Cannot call method 'indexOf' of undefined" in CookieJar
    // Detect and reject it as soon as possible
    var faultyUri = url.format(self.uri)
    var message = 'Invalid URI "' + faultyUri + '"'
    if (Object.keys(options).length === 0) {
      // No option ? This can be the sign of a redirect
      // As this is a case where the user cannot do anything (they didn't call request directly with this URL)
      // they should be warned that it can be caused by a redirection (can save some hair)
      message += '. This can be caused by a crappy redirection.'
    }
    self.emit('error', new Error(message))
    return // This error was fatal
  }

  self._redirectsFollowed = self._redirectsFollowed || 0
  self.maxRedirects = (self.maxRedirects !== undefined) ? self.maxRedirects : 10
  self.followRedirect = (self.followRedirect !== undefined) ? self.followRedirect : true
  self.followAllRedirects = (self.followAllRedirects !== undefined) ? self.followAllRedirects : false
  if (self.followRedirect || self.followAllRedirects)
    self.redirects = self.redirects || []

  self.headers = self.headers ? copy(self.headers) : {}

  self.setHost = false
  if (!self.hasHeader('host')) {
    self.setHeader('host', self.uri.hostname)
    if (self.uri.port) {
      if ( !(self.uri.port === 80 && self.uri.protocol === 'http:') &&
           !(self.uri.port === 443 && self.uri.protocol === 'https:') )
      self.setHeader('host', self.getHeader('host') + (':'+self.uri.port) )
    }
    self.setHost = true
  }

  self.jar(self._jar || options.jar)

  if (!self.uri.port) {
    if (self.uri.protocol == 'http:') {self.uri.port = 80}
    else if (self.uri.protocol == 'https:') {self.uri.port = 443}
  }

  if (self.proxy && !self.tunnel) {
    self.port = self.proxy.port
    self.host = self.proxy.hostname
  } else {
    self.port = self.uri.port
    self.host = self.uri.hostname
  }

  self.clientErrorHandler = function (error) {
    if (self._aborted) return
    if (self.req && self.req._reusedSocket && error.code === 'ECONNRESET'
        && self.agent.addRequestNoreuse) {
      self.agent = { addRequest: self.agent.addRequestNoreuse.bind(self.agent) }
      self.start()
      self.req.end()
      return
    }
    if (self.timeout && self.timeoutTimer) {
      clearTimeout(self.timeoutTimer)
      self.timeoutTimer = null
    }
    self.emit('error', error)
  }

  self._parserErrorHandler = function (error) {
    if (this.res) {
      if (this.res.request) {
        this.res.request.emit('error', error)
      } else {
        this.res.emit('error', error)
      }
    } else {
      this._httpMessage.emit('error', error)
    }
  }

  self._buildRequest = function(){
    var self = this;

    if (options.form) {
      self.form(options.form)
    }

    if (options.qs) self.qs(options.qs)

    if (self.uri.path) {
      self.path = self.uri.path
    } else {
      self.path = self.uri.pathname + (self.uri.search || "")
    }

    if (self.path.length === 0) self.path = '/'


    // Auth must happen last in case signing is dependent on other headers
    if (options.oauth) {
      self.oauth(options.oauth)
    }

    if (options.aws) {
      self.aws(options.aws)
    }

    if (options.hawk) {
      self.hawk(options.hawk)
    }

    if (options.httpSignature) {
      self.httpSignature(options.httpSignature)
    }

    if (options.auth) {
      if (Object.prototype.hasOwnProperty.call(options.auth, 'username')) options.auth.user = options.auth.username
      if (Object.prototype.hasOwnProperty.call(options.auth, 'password')) options.auth.pass = options.auth.password

      self.auth(
        options.auth.user,
        options.auth.pass,
        options.auth.sendImmediately,
        options.auth.bearer
      )
    }

    if (self.gzip && !self.hasHeader('accept-encoding')) {
      self.setHeader('accept-encoding', 'gzip')
    }

    if (self.uri.auth && !self.hasHeader('authorization')) {
      var authPieces = self.uri.auth.split(':').map(function(item){ return querystring.unescape(item) })
      self.auth(authPieces[0], authPieces.slice(1).join(':'), true)
    }
    if (self.proxy && self.proxy.auth && !self.hasHeader('proxy-authorization') && !self.tunnel) {
      self.setHeader('proxy-authorization', "Basic " + toBase64(self.proxy.auth.split(':').map(function(item){ return querystring.unescape(item)}).join(':')))
    }


    if (self.proxy && !self.tunnel) self.path = (self.uri.protocol + '//' + self.uri.host + self.path)

    if (options.json) {
      self.json(options.json)
    } else if (options.multipart) {
      self.boundary = uuid()
      self.multipart(options.multipart)
    }

    if (self.body) {
      var length = 0
      if (!Buffer.isBuffer(self.body)) {
        if (Array.isArray(self.body)) {
          for (var i = 0; i < self.body.length; i++) {
            length += self.body[i].length
          }
        } else {
          self.body = new Buffer(self.body)
          length = self.body.length
        }
      } else {
        length = self.body.length
      }
      if (length) {
        if (!self.hasHeader('content-length')) self.setHeader('content-length', length)
      } else {
        throw new Error('Argument error, options.body.')
      }
    }

    var protocol = self.proxy && !self.tunnel ? self.proxy.protocol : self.uri.protocol
      , defaultModules = {'http:':http, 'https:':https, 'unix:':http}
      , httpModules = self.httpModules || {}
      ;
    self.httpModule = httpModules[protocol] || defaultModules[protocol]

    if (!self.httpModule) return this.emit('error', new Error("Invalid protocol: " + protocol))

    if (options.ca) self.ca = options.ca

    if (!self.agent) {
      if (options.agentOptions) self.agentOptions = options.agentOptions

      if (options.agentClass) {
        self.agentClass = options.agentClass
      } else if (options.forever) {
        self.agentClass = protocol === 'http:' ? ForeverAgent : ForeverAgent.SSL
      } else {
        self.agentClass = self.httpModule.Agent
      }
    }

    if (self.pool === false) {
      self.agent = false
    } else {
      self.agent = self.agent || self.getAgent()
      if (self.maxSockets) {
        // Don't use our pooling if node has the refactored client
        self.agent.maxSockets = self.maxSockets
      }
      if (self.pool.maxSockets) {
        // Don't use our pooling if node has the refactored client
        self.agent.maxSockets = self.pool.maxSockets
      }
    }

    self.on('pipe', function (src) {
      if (self.ntick && self._started) throw new Error("You cannot pipe to this stream after the outbound request has started.")
      self.src = src
      if (isReadStream(src)) {
        if (!self.hasHeader('content-type')) self.setHeader('content-type', mime.lookup(src.path))
      } else {
        if (src.headers) {
          for (var i in src.headers) {
            if (!self.hasHeader(i)) {
              self.setHeader(i, src.headers[i])
            }
          }
        }
        if (self._json && !self.hasHeader('content-type'))
          self.setHeader('content-type', 'application/json')
        if (src.method && !self.explicitMethod) {
          self.method = src.method
        }
      }

      // self.on('pipe', function () {
      //   console.error("You have already piped to this stream. Pipeing twice is likely to break the request.")
      // })
    })

    process.nextTick(function () {
      if (self._aborted) return

      var end = function () {
        if (self._form) {
          self._form.pipe(self)
        }
        if (self.body) {
          if (Array.isArray(self.body)) {
            self.body.forEach(function (part) {
              self.write(part)
            })
          } else {
            self.write(self.body)
          }
          self.end()
        } else if (self.requestBodyStream) {
          console.warn("options.requestBodyStream is deprecated, please pass the request object to stream.pipe.")
          self.requestBodyStream.pipe(self)
        } else if (!self.src) {
          if (self.method !== 'GET' && typeof self.method !== 'undefined') {
            self.setHeader('content-length', 0)
          }
          self.end()
        }
      }

      if (self._form && !self.hasHeader('content-length')) {
        // Before ending the request, we had to compute the length of the whole form, asyncly
        self.setHeaders(self._form.getHeaders())
        self._form.getLength(function (err, length) {
          if (!err) {
            self.setHeader('content-length', length)
          }
          end()
        })
      } else {
        end()
      }

      self.ntick = true
    })

  } // End _buildRequest

  self._handleUnixSocketURI = function(self){
    // Parse URI and extract a socket path (tested as a valid socket using net.connect), and a http style path suffix
    // Thus http requests can be made to a socket using the uri unix://tmp/my.socket/urlpath
    // and a request for '/urlpath' will be sent to the unix socket at /tmp/my.socket

    self.unixsocket = true;

    var full_path = self.uri.href.replace(self.uri.protocol+'/', '');

    var lookup = full_path.split('/');
    var error_connecting = true;

    var lookup_table = {};
    do { lookup_table[lookup.join('/')]={} } while(lookup.pop())
    for (r in lookup_table){
      try_next(r);
    }

    function try_next(table_row){
      var client = net.connect( table_row );
      client.path = table_row
      client.on('error', function(){ lookup_table[this.path].error_connecting=true; this.end(); });
      client.on('connect', function(){ lookup_table[this.path].error_connecting=false; this.end(); });
      table_row.client = client;
    }

    wait_for_socket_response();

    response_counter = 0;

    function wait_for_socket_response(){
      var detach;
      if('undefined' == typeof setImmediate ) detach = process.nextTick
      else detach = setImmediate;
      detach(function(){
        // counter to prevent infinite blocking waiting for an open socket to be found.
        response_counter++;
        var trying = false;
        for (r in lookup_table){
          //console.log(r, lookup_table[r], lookup_table[r].error_connecting)
          if('undefined' == typeof lookup_table[r].error_connecting)
            trying = true;
        }
        if(trying && response_counter<1000)
          wait_for_socket_response()
        else
          set_socket_properties();
      })
    }

    function set_socket_properties(){
      var host;
      for (r in lookup_table){
        if(lookup_table[r].error_connecting === false){
          host = r
        }
      }
      if(!host){
        self.emit('error', new Error("Failed to connect to any socket in "+full_path))
      }
      var path = full_path.replace(host, '')

      self.socketPath = host
      self.uri.pathname = path
      self.uri.href = path
      self.uri.path = path
      self.host = ''
      self.hostname = ''
      delete self.host
      delete self.hostname
      self._buildRequest();
    }
  }

  // Intercept UNIX protocol requests to change properties to match socket
  if(/^unix:/.test(self.uri.protocol)){
    self._handleUnixSocketURI(self);
  } else {
    self._buildRequest();
  }

}

// Must call this when following a redirect from https to http or vice versa
// Attempts to keep everything as identical as possible, but update the
// httpModule, Tunneling agent, and/or Forever Agent in use.
Request.prototype._updateProtocol = function () {
  var self = this
  var protocol = self.uri.protocol

  if (protocol === 'https:') {
    // previously was doing http, now doing https
    // if it's https, then we might need to tunnel now.
    if (self.proxy && self.canTunnel) {
      self.tunnel = true
      var tunnelFn = self.proxy.protocol === 'http:'
                   ? tunnel.httpsOverHttp : tunnel.httpsOverHttps
      var tunnelOptions = { proxy: { host: self.proxy.hostname
                                   , port: +self.proxy.port
                                   , proxyAuth: self.proxy.auth }
                          , rejectUnauthorized: self.rejectUnauthorized
                          , ca: self.ca }
      self.agent = tunnelFn(tunnelOptions)
      return
    }

    self.httpModule = https
    switch (self.agentClass) {
      case ForeverAgent:
        self.agentClass = ForeverAgent.SSL
        break
      case http.Agent:
        self.agentClass = https.Agent
        break
      default:
        // nothing we can do.  Just hope for the best.
        return
    }

    // if there's an agent, we need to get a new one.
    if (self.agent) self.agent = self.getAgent()

  } else {
    // previously was doing https, now doing http
    // stop any tunneling.
    if (self.tunnel) self.tunnel = false
    self.httpModule = http
    switch (self.agentClass) {
      case ForeverAgent.SSL:
        self.agentClass = ForeverAgent
        break
      case https.Agent:
        self.agentClass = http.Agent
        break
      default:
        // nothing we can do.  just hope for the best
        return
    }

    // if there's an agent, then get a new one.
    if (self.agent) {
      self.agent = null
      self.agent = self.getAgent()
    }
  }
}

Request.prototype.getAgent = function () {
  var Agent = this.agentClass
  var options = {}
  if (this.agentOptions) {
    for (var i in this.agentOptions) {
      options[i] = this.agentOptions[i]
    }
  }
  if (this.ca) options.ca = this.ca
  if (this.ciphers) options.ciphers = this.ciphers
  if (this.secureProtocol) options.secureProtocol = this.secureProtocol
  if (this.secureOptions) options.secureOptions = this.secureOptions
  if (typeof this.rejectUnauthorized !== 'undefined') options.rejectUnauthorized = this.rejectUnauthorized

  if (this.cert && this.key) {
    options.key = this.key
    options.cert = this.cert
  }

  var poolKey = ''

  // different types of agents are in different pools
  if (Agent !== this.httpModule.Agent) {
    poolKey += Agent.name
  }

  if (!this.httpModule.globalAgent) {
    // node 0.4.x
    options.host = this.host
    options.port = this.port
    if (poolKey) poolKey += ':'
    poolKey += this.host + ':' + this.port
  }

  // ca option is only relevant if proxy or destination are https
  var proxy = this.proxy
  if (typeof proxy === 'string') proxy = url.parse(proxy)
  var isHttps = (proxy && proxy.protocol === 'https:') || this.uri.protocol === 'https:'
  if (isHttps) {
    if (options.ca) {
      if (poolKey) poolKey += ':'
      poolKey += options.ca
    }

    if (typeof options.rejectUnauthorized !== 'undefined') {
      if (poolKey) poolKey += ':'
      poolKey += options.rejectUnauthorized
    }

    if (options.cert)
      poolKey += options.cert.toString('ascii') + options.key.toString('ascii')

    if (options.ciphers) {
      if (poolKey) poolKey += ':'
      poolKey += options.ciphers
    }

    if (options.secureProtocol) {
      if (poolKey) poolKey += ':'
      poolKey += options.secureProtocol
    }

    if (options.secureOptions) {
      if (poolKey) poolKey += ':'
      poolKey += options.secureOptions
    }
  }

  if (this.pool === globalPool && !poolKey && Object.keys(options).length === 0 && this.httpModule.globalAgent) {
    // not doing anything special.  Use the globalAgent
    return this.httpModule.globalAgent
  }

  // we're using a stored agent.  Make sure it's protocol-specific
  poolKey = this.uri.protocol + poolKey

  // already generated an agent for this setting
  if (this.pool[poolKey]) return this.pool[poolKey]

  return this.pool[poolKey] = new Agent(options)
}

Request.prototype.start = function () {
  // start() is called once we are ready to send the outgoing HTTP request.
  // this is usually called on the first write(), end() or on nextTick()
  var self = this

  if (self._aborted) return

  self._started = true
  self.method = self.method || 'GET'
  self.href = self.uri.href

  if (self.src && self.src.stat && self.src.stat.size && !self.hasHeader('content-length')) {
    self.setHeader('content-length', self.src.stat.size)
  }
  if (self._aws) {
    self.aws(self._aws, true)
  }

  // We have a method named auth, which is completely different from the http.request
  // auth option.  If we don't remove it, we're gonna have a bad time.
  var reqOptions = copy(self)
  delete reqOptions.auth

  debug('make request', self.uri.href)
  self.req = self.httpModule.request(reqOptions, self.onResponse.bind(self))

  if (self.timeout && !self.timeoutTimer) {
    self.timeoutTimer = setTimeout(function () {
      self.req.abort()
      var e = new Error("ETIMEDOUT")
      e.code = "ETIMEDOUT"
      self.emit("error", e)
    }, self.timeout)

    // Set additional timeout on socket - in case if remote
    // server freeze after sending headers
    if (self.req.setTimeout) { // only works on node 0.6+
      self.req.setTimeout(self.timeout, function () {
        if (self.req) {
          self.req.abort()
          var e = new Error("ESOCKETTIMEDOUT")
          e.code = "ESOCKETTIMEDOUT"
          self.emit("error", e)
        }
      })
    }
  }

  self.req.on('error', self.clientErrorHandler)
  self.req.on('drain', function() {
    self.emit('drain')
  })
  self.on('end', function() {
    if ( self.req.connection ) self.req.connection.removeListener('error', self._parserErrorHandler)
  })
  self.emit('request', self.req)
}
Request.prototype.onResponse = function (response) {
  var self = this
  debug('onResponse', self.uri.href, response.statusCode, response.headers)
  response.on('end', function() {
    debug('response end', self.uri.href, response.statusCode, response.headers)
  });

  // The check on response.connection is a workaround for browserify.
  if (response.connection && response.connection.listeners('error').indexOf(self._parserErrorHandler) === -1) {
    response.connection.setMaxListeners(0)
    response.connection.once('error', self._parserErrorHandler)
  }
  if (self._aborted) {
    debug('aborted', self.uri.href)
    response.resume()
    return
  }
  if (self._paused) response.pause()
  // Check that response.resume is defined. Workaround for browserify.
  else response.resume && response.resume()

  self.response = response
  response.request = self
  response.toJSON = toJSON

  // XXX This is different on 0.10, because SSL is strict by default
  if (self.httpModule === https &&
      self.strictSSL &&
      !response.client.authorized) {
    debug('strict ssl error', self.uri.href)
    var sslErr = response.client.authorizationError
    self.emit('error', new Error('SSL Error: '+ sslErr))
    return
  }

  if (self.setHost && self.hasHeader('host')) delete self.headers[self.hasHeader('host')]
  if (self.timeout && self.timeoutTimer) {
    clearTimeout(self.timeoutTimer)
    self.timeoutTimer = null
  }

  var targetCookieJar = (self._jar && self._jar.setCookie)?self._jar:globalCookieJar;
  var addCookie = function (cookie) {
    //set the cookie if it's domain in the href's domain.
    try {
      targetCookieJar.setCookie(cookie, self.uri.href, {ignoreError: true});
    } catch (e) {
      self.emit('error', e);
    }
  }

  if (hasHeader('set-cookie', response.headers) && (!self._disableCookies)) {
    var headerName = hasHeader('set-cookie', response.headers)
    if (Array.isArray(response.headers[headerName])) response.headers[headerName].forEach(addCookie)
    else addCookie(response.headers[headerName])
  }

  var redirectTo = null
  if (response.statusCode >= 300 && response.statusCode < 400 && hasHeader('location', response.headers)) {
    var location = response.headers[hasHeader('location', response.headers)]
    debug('redirect', location)

    if (self.followAllRedirects) {
      redirectTo = location
    } else if (self.followRedirect) {
      switch (self.method) {
        case 'PATCH':
        case 'PUT':
        case 'POST':
        case 'DELETE':
          // Do not follow redirects
          break
        default:
          redirectTo = location
          break
      }
    }
  } else if (response.statusCode == 401 && self._hasAuth && !self._sentAuth) {
    var authHeader = response.headers[hasHeader('www-authenticate', response.headers)]
    var authVerb = authHeader && authHeader.split(' ')[0].toLowerCase()
    debug('reauth', authVerb)

    switch (authVerb) {
      case 'basic':
        self.auth(self._user, self._pass, true)
        redirectTo = self.uri
        break

      case 'bearer':
        self.auth(null, null, true, self._bearer)
        redirectTo = self.uri
        break

      case 'digest':
        // TODO: More complete implementation of RFC 2617.
        //   - check challenge.algorithm
        //   - support algorithm="MD5-sess"
        //   - handle challenge.domain
        //   - support qop="auth-int" only
        //   - handle Authentication-Info (not necessarily?)
        //   - check challenge.stale (not necessarily?)
        //   - increase nc (not necessarily?)
        // For reference:
        // http://tools.ietf.org/html/rfc2617#section-3
        // https://github.com/bagder/curl/blob/master/lib/http_digest.c

        var challenge = {}
        var re = /([a-z0-9_-]+)=(?:"([^"]+)"|([a-z0-9_-]+))/gi
        for (;;) {
          var match = re.exec(authHeader)
          if (!match) break
          challenge[match[1]] = match[2] || match[3];
        }

        var ha1 = md5(self._user + ':' + challenge.realm + ':' + self._pass)
        var ha2 = md5(self.method + ':' + self.uri.path)
        var qop = /(^|,)\s*auth\s*($|,)/.test(challenge.qop) && 'auth'
        var nc = qop && '00000001'
        var cnonce = qop && uuid().replace(/-/g, '')
        var digestResponse = qop ? md5(ha1 + ':' + challenge.nonce + ':' + nc + ':' + cnonce + ':' + qop + ':' + ha2) : md5(ha1 + ':' + challenge.nonce + ':' + ha2)
        var authValues = {
          username: self._user,
          realm: challenge.realm,
          nonce: challenge.nonce,
          uri: self.uri.path,
          qop: qop,
          response: digestResponse,
          nc: nc,
          cnonce: cnonce,
          algorithm: challenge.algorithm,
          opaque: challenge.opaque
        }

        authHeader = []
        for (var k in authValues) {
          if (!authValues[k]) {
            //ignore
          } else if (k === 'qop' || k === 'nc' || k === 'algorithm') {
            authHeader.push(k + '=' + authValues[k])
          } else {
            authHeader.push(k + '="' + authValues[k] + '"')
          }
        }
        authHeader = 'Digest ' + authHeader.join(', ')
        self.setHeader('authorization', authHeader)
        self._sentAuth = true

        redirectTo = self.uri
        break
    }
  }

  if (redirectTo) {
    debug('redirect to', redirectTo)

    // ignore any potential response body.  it cannot possibly be useful
    // to us at this point.
    if (self._paused) response.resume()

    if (self._redirectsFollowed >= self.maxRedirects) {
      self.emit('error', new Error("Exceeded maxRedirects. Probably stuck in a redirect loop "+self.uri.href))
      return
    }
    self._redirectsFollowed += 1

    if (!isUrl.test(redirectTo)) {
      redirectTo = url.resolve(self.uri.href, redirectTo)
    }

    var uriPrev = self.uri
    self.uri = url.parse(redirectTo)

    // handle the case where we change protocol from https to http or vice versa
    if (self.uri.protocol !== uriPrev.protocol) {
      self._updateProtocol()
    }

    self.redirects.push(
      { statusCode : response.statusCode
      , redirectUri: redirectTo
      }
    )
    if (self.followAllRedirects && response.statusCode != 401 && response.statusCode != 307) self.method = 'GET'
    // self.method = 'GET' // Force all redirects to use GET || commented out fixes #215
    delete self.src
    delete self.req
    delete self.agent
    delete self._started
    if (response.statusCode != 401 && response.statusCode != 307) {
      // Remove parameters from the previous response, unless this is the second request
      // for a server that requires digest authentication.
      delete self.body
      delete self._form
      if (self.headers) {
        if (self.hasHeader('host')) delete self.headers[self.hasHeader('host')]
        if (self.hasHeader('content-type')) delete self.headers[self.hasHeader('content-type')]
        if (self.hasHeader('content-length')) delete self.headers[self.hasHeader('content-length')]
      }
    }

    self.emit('redirect');

    self.init()
    return // Ignore the rest of the response
  } else {
    self._redirectsFollowed = self._redirectsFollowed || 0
    // Be a good stream and emit end when the response is finished.
    // Hack to emit end on close because of a core bug that never fires end
    response.on('close', function () {
      if (!self._ended) self.response.emit('end')
    })

    var dataStream
    if (self.gzip) {
      var contentEncoding = response.headers["content-encoding"] || "identity"
      contentEncoding = contentEncoding.trim().toLowerCase()

      if (contentEncoding === "gzip") {
        dataStream = zlib.createGunzip()
        response.pipe(dataStream)
      } else {
        // Since previous versions didn't check for Content-Encoding header,
        // ignore any invalid values to preserve backwards-compatibility
        if (contentEncoding !== "identity") {
          debug("ignoring unrecognized Content-Encoding " + contentEncoding)
        }
        dataStream = response
      }
    } else {
      dataStream = response
    }

    if (self.encoding) {
      if (self.dests.length !== 0) {
        console.error("Ignoring encoding parameter as this stream is being piped to another stream which makes the encoding option invalid.")
      } else if (dataStream.setEncoding) {
        dataStream.setEncoding(self.encoding)
      } else {
        // Should only occur on node pre-v0.9.4 (joyent/node@9b5abe5) with
        // zlib streams.
        // If/When support for 0.9.4 is dropped, this should be unnecessary.
        dataStream = dataStream.pipe(stringstream(self.encoding))
      }
    }

    self.emit('response', response)

    self.dests.forEach(function (dest) {
      self.pipeDest(dest)
    })

    dataStream.on("data", function (chunk) {
      var emitted = self.emit("data", chunk)
      if (emitted) {
        self._destdata = true
      } else {
        // pause URL stream until we pipe it
        dataStream.pause()
        dataStream.unshift(chunk)
      }
    })
    dataStream.on("end", function (chunk) {
      self._ended = true
      self.emit("end", chunk)
    })
    dataStream.on("close", function () {self.emit("close")})

    if (self.callback) {
      var buffer = []
      var bodyLen = 0
      self.on("data", function (chunk) {
        buffer.push(chunk)
        bodyLen += chunk.length
      })
      self.on("end", function () {
        debug('end event', self.uri.href)
        if (self._aborted) {
          debug('aborted', self.uri.href)
          return
        }

        if (buffer.length && Buffer.isBuffer(buffer[0])) {
          debug('has body', self.uri.href, bodyLen)
          var body = new Buffer(bodyLen)
          var i = 0
          buffer.forEach(function (chunk) {
            chunk.copy(body, i, 0, chunk.length)
            i += chunk.length
          })
          if (self.encoding === null) {
            response.body = body
          } else {
            response.body = body.toString(self.encoding)
          }
        } else if (buffer.length) {
          // The UTF8 BOM [0xEF,0xBB,0xBF] is converted to [0xFE,0xFF] in the JS UTC16/UCS2 representation.
          // Strip this value out when the encoding is set to 'utf8', as upstream consumers won't expect it and it breaks JSON.parse().
          if (self.encoding === 'utf8' && buffer[0].length > 0 && buffer[0][0] === "\uFEFF") {
            buffer[0] = buffer[0].substring(1)
          }
          response.body = buffer.join('')
        }

        if (self._json) {
          try {
            response.body = JSON.parse(response.body)
          } catch (e) {}
        }
        debug('emitting complete', self.uri.href)
        if(response.body == undefined && !self._json) {
          response.body = "";
        }
        self.emit('complete', response, response.body)
      })
    }
    //if no callback
    else{
      self.on("end", function () {
        if (self._aborted) {
          debug('aborted', self.uri.href)
          return
        }
        self.emit('complete', response);
      });
    }
  }
  debug('finish init function', self.uri.href)
}

Request.prototype.abort = function () {
  this._aborted = true

  if (this.req) {
    this.req.abort()
  }
  else if (this.response) {
    this.response.abort()
  }

  this.emit("abort")
}

Request.prototype.pipeDest = function (dest) {
  var response = this.response
  // Called after the response is received
  if (dest.headers && !dest.headersSent) {
    if (hasHeader('content-type', response.headers)) {
      var ctname = hasHeader('content-type', response.headers)
      if (dest.setHeader) dest.setHeader(ctname, response.headers[ctname])
      else dest.headers[ctname] = response.headers[ctname]
    }

    if (hasHeader('content-length', response.headers)) {
      var clname = hasHeader('content-length', response.headers)
      if (dest.setHeader) dest.setHeader(clname, response.headers[clname])
      else dest.headers[clname] = response.headers[clname]
    }
  }
  if (dest.setHeader && !dest.headersSent) {
    for (var i in response.headers) {
      // If the response content is being decoded, the Content-Encoding header
      // of the response doesn't represent the piped content, so don't pass it.
      if (!this.gzip || i !== 'content-encoding') {
        dest.setHeader(i, response.headers[i])
      }
    }
    dest.statusCode = response.statusCode
  }
  if (this.pipefilter) this.pipefilter(response, dest)
}

// Composable API
Request.prototype.setHeader = function (name, value, clobber) {
  if (clobber === undefined) clobber = true
  if (clobber || !this.hasHeader(name)) this.headers[name] = value
  else this.headers[this.hasHeader(name)] += ',' + value
  return this
}
Request.prototype.setHeaders = function (headers) {
  for (var i in headers) {this.setHeader(i, headers[i])}
  return this
}
Request.prototype.hasHeader = function (header, headers) {
  var headers = Object.keys(headers || this.headers)
    , lheaders = headers.map(function (h) {return h.toLowerCase()})
    ;
  header = header.toLowerCase()
  for (var i=0;i<lheaders.length;i++) {
    if (lheaders[i] === header) return headers[i]
  }
  return false
}

var hasHeader = Request.prototype.hasHeader

Request.prototype.qs = function (q, clobber) {
  var base
  if (!clobber && this.uri.query) base = qs.parse(this.uri.query)
  else base = {}

  for (var i in q) {
    base[i] = q[i]
  }

  if (qs.stringify(base) === ''){
    return this
  }

  this.uri = url.parse(this.uri.href.split('?')[0] + '?' + qs.stringify(base))
  this.url = this.uri
  this.path = this.uri.path

  return this
}
Request.prototype.form = function (form) {
  if (form) {
    this.setHeader('content-type', 'application/x-www-form-urlencoded; charset=utf-8')
    this.body = (typeof form === 'string') ? form.toString('utf8') : qs.stringify(form).toString('utf8')
    return this
  }
  // create form-data object
  this._form = new FormData()
  return this._form
}
Request.prototype.multipart = function (multipart) {
  var self = this
  self.body = []

  if (!self.hasHeader('content-type')) {
    self.setHeader('content-type', 'multipart/related; boundary=' + self.boundary)
  } else {
    var headerName = self.hasHeader('content-type');
    self.setHeader(headerName, self.headers[headerName].split(';')[0] + '; boundary=' + self.boundary)
  }

  if (!multipart.forEach) throw new Error('Argument error, options.multipart.')

  if (self.preambleCRLF) {
    self.body.push(new Buffer('\r\n'))
  }

  multipart.forEach(function (part) {
    var body = part.body
    if(body == null) throw Error('Body attribute missing in multipart.')
    delete part.body
    var preamble = '--' + self.boundary + '\r\n'
    Object.keys(part).forEach(function (key) {
      preamble += key + ': ' + part[key] + '\r\n'
    })
    preamble += '\r\n'
    self.body.push(new Buffer(preamble))
    self.body.push(new Buffer(body))
    self.body.push(new Buffer('\r\n'))
  })
  self.body.push(new Buffer('--' + self.boundary + '--'))
  return self
}
Request.prototype.json = function (val) {
  var self = this

  if (!self.hasHeader('accept')) self.setHeader('accept', 'application/json')

  this._json = true
  if (typeof val === 'boolean') {
    if (typeof this.body === 'object') {
      this.body = safeStringify(this.body)
      if (!self.hasHeader('content-type'))
        self.setHeader('content-type', 'application/json')
    }
  } else {
    this.body = safeStringify(val)
    if (!self.hasHeader('content-type'))
      self.setHeader('content-type', 'application/json')
  }

  return this
}
Request.prototype.getHeader = function (name, headers) {
  var result, re, match
  if (!headers) headers = this.headers
  Object.keys(headers).forEach(function (key) {
    if (key.length !== name.length) return
    re = new RegExp(name, 'i')
    match = key.match(re)
    if (match) result = headers[key]
  })
  return result
}
var getHeader = Request.prototype.getHeader

Request.prototype.auth = function (user, pass, sendImmediately, bearer) {
  if (bearer !== undefined) {
    this._bearer = bearer
    this._hasAuth = true
    if (sendImmediately || typeof sendImmediately == 'undefined') {
      if (typeof bearer === 'function') {
        bearer = bearer()
      }
      this.setHeader('authorization', 'Bearer ' + bearer)
      this._sentAuth = true
    }
    return this
  }
  if (typeof user !== 'string' || (pass !== undefined && typeof pass !== 'string')) {
    throw new Error('auth() received invalid user or password')
  }
  this._user = user
  this._pass = pass
  this._hasAuth = true
  var header = typeof pass !== 'undefined' ? user + ':' + pass : user
  if (sendImmediately || typeof sendImmediately == 'undefined') {
    this.setHeader('authorization', 'Basic ' + toBase64(header))
    this._sentAuth = true
  }
  return this
}
Request.prototype.aws = function (opts, now) {
  if (!now) {
    this._aws = opts
    return this
  }
  var date = new Date()
  this.setHeader('date', date.toUTCString())
  var auth =
    { key: opts.key
    , secret: opts.secret
    , verb: this.method.toUpperCase()
    , date: date
    , contentType: this.getHeader('content-type') || ''
    , md5: this.getHeader('content-md5') || ''
    , amazonHeaders: aws.canonicalizeHeaders(this.headers)
    }
  if (opts.bucket && this.path) {
    auth.resource = '/' + opts.bucket + this.path
  } else if (opts.bucket && !this.path) {
    auth.resource = '/' + opts.bucket
  } else if (!opts.bucket && this.path) {
    auth.resource = this.path
  } else if (!opts.bucket && !this.path) {
    auth.resource = '/'
  }
  auth.resource = aws.canonicalizeResource(auth.resource)
  this.setHeader('authorization', aws.authorization(auth))

  return this
}
Request.prototype.httpSignature = function (opts) {
  var req = this
  httpSignature.signRequest({
    getHeader: function(header) {
      return getHeader(header, req.headers)
    },
    setHeader: function(header, value) {
      req.setHeader(header, value)
    },
    method: this.method,
    path: this.path
  }, opts)
  debug('httpSignature authorization', this.getHeader('authorization'))

  return this
}

Request.prototype.hawk = function (opts) {
  this.setHeader('Authorization', hawk.client.header(this.uri, this.method, opts).field)
}

Request.prototype.oauth = function (_oauth) {
  var form
  if (this.hasHeader('content-type') &&
      this.getHeader('content-type').slice(0, 'application/x-www-form-urlencoded'.length) ===
        'application/x-www-form-urlencoded'
     ) {
    form = qs.parse(this.body)
  }
  if (this.uri.query) {
    form = qs.parse(this.uri.query)
  }
  if (!form) form = {}
  var oa = {}
  for (var i in form) oa[i] = form[i]
  for (var i in _oauth) oa['oauth_'+i] = _oauth[i]
  if (!oa.oauth_version) oa.oauth_version = '1.0'
  if (!oa.oauth_timestamp) oa.oauth_timestamp = Math.floor( Date.now() / 1000 ).toString()
  if (!oa.oauth_nonce) oa.oauth_nonce = uuid().replace(/-/g, '')

  oa.oauth_signature_method = 'HMAC-SHA1'

  var consumer_secret = oa.oauth_consumer_secret
  delete oa.oauth_consumer_secret
  var token_secret = oa.oauth_token_secret
  delete oa.oauth_token_secret
  var timestamp = oa.oauth_timestamp

  var baseurl = this.uri.protocol + '//' + this.uri.host + this.uri.pathname
  var signature = oauth.hmacsign(this.method, baseurl, oa, consumer_secret, token_secret)

  // oa.oauth_signature = signature
  for (var i in form) {
    if ( i.slice(0, 'oauth_') in _oauth) {
      // skip
    } else {
      delete oa['oauth_'+i]
      if (i !== 'x_auth_mode') delete oa[i]
    }
  }
  oa.oauth_timestamp = timestamp
  var authHeader = 'OAuth '+Object.keys(oa).sort().map(function (i) {return i+'="'+oauth.rfc3986(oa[i])+'"'}).join(',')
  authHeader += ',oauth_signature="' + oauth.rfc3986(signature) + '"'
  this.setHeader('Authorization', authHeader)
  return this
}
Request.prototype.jar = function (jar) {
  var cookies

  if (this._redirectsFollowed === 0) {
    this.originalCookieHeader = this.getHeader('cookie')
  }

  if (!jar) {
    // disable cookies
    cookies = false
    this._disableCookies = true
  } else {
    var targetCookieJar = (jar && jar.getCookieString)?jar:globalCookieJar;
    var urihref = this.uri.href
    //fetch cookie in the Specified host
    if (targetCookieJar) {
      cookies = targetCookieJar.getCookieString(urihref);
    }
  }

  //if need cookie and cookie is not empty
  if (cookies && cookies.length) {
    if (this.originalCookieHeader) {
      // Don't overwrite existing Cookie header
      this.setHeader('cookie', this.originalCookieHeader + '; ' + cookies)
    } else {
      this.setHeader('cookie', cookies)
    }
  }
  this._jar = jar
  return this
}


// Stream API
Request.prototype.pipe = function (dest, opts) {
  if (this.response) {
    if (this._destdata) {
      throw new Error("You cannot pipe after data has been emitted from the response.")
    } else if (this._ended) {
      throw new Error("You cannot pipe after the response has been ended.")
    } else {
      stream.Stream.prototype.pipe.call(this, dest, opts)
      this.pipeDest(dest)
      return dest
    }
  } else {
    this.dests.push(dest)
    stream.Stream.prototype.pipe.call(this, dest, opts)
    return dest
  }
}
Request.prototype.write = function () {
  if (!this._started) this.start()
  return this.req.write.apply(this.req, arguments)
}
Request.prototype.end = function (chunk) {
  if (chunk) this.write(chunk)
  if (!this._started) this.start()
  this.req.end()
}
Request.prototype.pause = function () {
  if (!this.response) this._paused = true
  else this.response.pause.apply(this.response, arguments)
}
Request.prototype.resume = function () {
  if (!this.response) this._paused = false
  else this.response.resume.apply(this.response, arguments)
}
Request.prototype.destroy = function () {
  if (!this._ended) this.end()
  else if (this.response) this.response.destroy()
}

function toJSON () {
  return getSafe(this, '__' + (((1+Math.random())*0x10000)|0).toString(16))
}

Request.prototype.toJSON = toJSON

/* Fix per https://github.com/mikeal/request/issues/887#issuecomment-48831952 */
var superOn = Request.prototype.on;
Request.prototype.on = function (eventName) {
  if (eventName === "data") {
    this.resume()
  }
  superOn.apply(this, arguments)
}

module.exports = Request