123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365 |
- 'use strict'
- const { URL } = require('url')
- const http = require('http')
- const https = require('https')
- const zlib = require('minizlib')
- const Minipass = require('minipass')
-
- const Body = require('./body.js')
- const { writeToStream, getTotalBytes } = Body
- const Response = require('./response.js')
- const Headers = require('./headers.js')
- const { createHeadersLenient } = Headers
- const Request = require('./request.js')
- const { getNodeRequestOptions } = Request
- const FetchError = require('./fetch-error.js')
- const AbortError = require('./abort-error.js')
-
- // XXX this should really be split up and unit-ized for easier testing
- // and better DRY implementation of data/http request aborting
- const fetch = async (url, opts) => {
- if (/^data:/.test(url)) {
- const request = new Request(url, opts)
- // delay 1 promise tick so that the consumer can abort right away
- return Promise.resolve().then(() => new Promise((resolve, reject) => {
- let type, data
- try {
- const { pathname, search } = new URL(url)
- const split = pathname.split(',')
- if (split.length < 2) {
- throw new Error('invalid data: URI')
- }
- const mime = split.shift()
- const base64 = /;base64$/.test(mime)
- type = base64 ? mime.slice(0, -1 * ';base64'.length) : mime
- const rawData = decodeURIComponent(split.join(',') + search)
- data = base64 ? Buffer.from(rawData, 'base64') : Buffer.from(rawData)
- } catch (er) {
- return reject(new FetchError(`[${request.method}] ${
- request.url} invalid URL, ${er.message}`, 'system', er))
- }
-
- const { signal } = request
- if (signal && signal.aborted) {
- return reject(new AbortError('The user aborted a request.'))
- }
-
- const headers = { 'Content-Length': data.length }
- if (type) {
- headers['Content-Type'] = type
- }
- return resolve(new Response(data, { headers }))
- }))
- }
-
- return new Promise((resolve, reject) => {
- // build request object
- const request = new Request(url, opts)
- let options
- try {
- options = getNodeRequestOptions(request)
- } catch (er) {
- return reject(er)
- }
-
- const send = (options.protocol === 'https:' ? https : http).request
- const { signal } = request
- let response = null
- const abort = () => {
- const error = new AbortError('The user aborted a request.')
- reject(error)
- if (Minipass.isStream(request.body) &&
- typeof request.body.destroy === 'function') {
- request.body.destroy(error)
- }
- if (response && response.body) {
- response.body.emit('error', error)
- }
- }
-
- if (signal && signal.aborted) {
- return abort()
- }
-
- const abortAndFinalize = () => {
- abort()
- finalize()
- }
-
- const finalize = () => {
- req.abort()
- if (signal) {
- signal.removeEventListener('abort', abortAndFinalize)
- }
- clearTimeout(reqTimeout)
- }
-
- // send request
- const req = send(options)
-
- if (signal) {
- signal.addEventListener('abort', abortAndFinalize)
- }
-
- let reqTimeout = null
- if (request.timeout) {
- req.once('socket', socket => {
- reqTimeout = setTimeout(() => {
- reject(new FetchError(`network timeout at: ${
- request.url}`, 'request-timeout'))
- finalize()
- }, request.timeout)
- })
- }
-
- req.on('error', er => {
- // if a 'response' event is emitted before the 'error' event, then by the
- // time this handler is run it's too late to reject the Promise for the
- // response. instead, we forward the error event to the response stream
- // so that the error will surface to the user when they try to consume
- // the body. this is done as a side effect of aborting the request except
- // for in windows, where we must forward the event manually, otherwise
- // there is no longer a ref'd socket attached to the request and the
- // stream never ends so the event loop runs out of work and the process
- // exits without warning.
- // coverage skipped here due to the difficulty in testing
- // istanbul ignore next
- if (req.res) {
- req.res.emit('error', er)
- }
- reject(new FetchError(`request to ${request.url} failed, reason: ${
- er.message}`, 'system', er))
- finalize()
- })
-
- req.on('response', res => {
- clearTimeout(reqTimeout)
-
- const headers = createHeadersLenient(res.headers)
-
- // HTTP fetch step 5
- if (fetch.isRedirect(res.statusCode)) {
- // HTTP fetch step 5.2
- const location = headers.get('Location')
-
- // HTTP fetch step 5.3
- const locationURL = location === null ? null
- : (new URL(location, request.url)).toString()
-
- // HTTP fetch step 5.5
- if (request.redirect === 'error') {
- reject(new FetchError('uri requested responds with a redirect, ' +
- `redirect mode is set to error: ${request.url}`, 'no-redirect'))
- finalize()
- return
- } else if (request.redirect === 'manual') {
- // node-fetch-specific step: make manual redirect a bit easier to
- // use by setting the Location header value to the resolved URL.
- if (locationURL !== null) {
- // handle corrupted header
- try {
- headers.set('Location', locationURL)
- } catch (err) {
- /* istanbul ignore next: nodejs server prevent invalid
- response headers, we can't test this through normal
- request */
- reject(err)
- }
- }
- } else if (request.redirect === 'follow' && locationURL !== null) {
- // HTTP-redirect fetch step 5
- if (request.counter >= request.follow) {
- reject(new FetchError(`maximum redirect reached at: ${
- request.url}`, 'max-redirect'))
- finalize()
- return
- }
-
- // HTTP-redirect fetch step 9
- if (res.statusCode !== 303 &&
- request.body &&
- getTotalBytes(request) === null) {
- reject(new FetchError(
- 'Cannot follow redirect with body being a readable stream',
- 'unsupported-redirect'
- ))
- finalize()
- return
- }
-
- // Update host due to redirection
- request.headers.set('host', (new URL(locationURL)).host)
-
- // HTTP-redirect fetch step 6 (counter increment)
- // Create a new Request object.
- const requestOpts = {
- headers: new Headers(request.headers),
- follow: request.follow,
- counter: request.counter + 1,
- agent: request.agent,
- compress: request.compress,
- method: request.method,
- body: request.body,
- signal: request.signal,
- timeout: request.timeout,
- }
-
- // if the redirect is to a new hostname, strip the authorization and cookie headers
- const parsedOriginal = new URL(request.url)
- const parsedRedirect = new URL(locationURL)
- if (parsedOriginal.hostname !== parsedRedirect.hostname) {
- requestOpts.headers.delete('authorization')
- requestOpts.headers.delete('cookie')
- }
-
- // HTTP-redirect fetch step 11
- if (res.statusCode === 303 || (
- (res.statusCode === 301 || res.statusCode === 302) &&
- request.method === 'POST'
- )) {
- requestOpts.method = 'GET'
- requestOpts.body = undefined
- requestOpts.headers.delete('content-length')
- }
-
- // HTTP-redirect fetch step 15
- resolve(fetch(new Request(locationURL, requestOpts)))
- finalize()
- return
- }
- } // end if(isRedirect)
-
- // prepare response
- res.once('end', () =>
- signal && signal.removeEventListener('abort', abortAndFinalize))
-
- const body = new Minipass()
- // if an error occurs, either on the response stream itself, on one of the
- // decoder streams, or a response length timeout from the Body class, we
- // forward the error through to our internal body stream. If we see an
- // error event on that, we call finalize to abort the request and ensure
- // we don't leave a socket believing a request is in flight.
- // this is difficult to test, so lacks specific coverage.
- body.on('error', finalize)
- // exceedingly rare that the stream would have an error,
- // but just in case we proxy it to the stream in use.
- res.on('error', /* istanbul ignore next */ er => body.emit('error', er))
- res.on('data', (chunk) => body.write(chunk))
- res.on('end', () => body.end())
-
- const responseOptions = {
- url: request.url,
- status: res.statusCode,
- statusText: res.statusMessage,
- headers: headers,
- size: request.size,
- timeout: request.timeout,
- counter: request.counter,
- trailer: new Promise(resolveTrailer =>
- res.on('end', () => resolveTrailer(createHeadersLenient(res.trailers)))),
- }
-
- // HTTP-network fetch step 12.1.1.3
- const codings = headers.get('Content-Encoding')
-
- // HTTP-network fetch step 12.1.1.4: handle content codings
-
- // in following scenarios we ignore compression support
- // 1. compression support is disabled
- // 2. HEAD request
- // 3. no Content-Encoding header
- // 4. no content response (204)
- // 5. content not modified response (304)
- if (!request.compress ||
- request.method === 'HEAD' ||
- codings === null ||
- res.statusCode === 204 ||
- res.statusCode === 304) {
- response = new Response(body, responseOptions)
- resolve(response)
- return
- }
-
- // Be less strict when decoding compressed responses, since sometimes
- // servers send slightly invalid responses that are still accepted
- // by common browsers.
- // Always using Z_SYNC_FLUSH is what cURL does.
- const zlibOptions = {
- flush: zlib.constants.Z_SYNC_FLUSH,
- finishFlush: zlib.constants.Z_SYNC_FLUSH,
- }
-
- // for gzip
- if (codings === 'gzip' || codings === 'x-gzip') {
- const unzip = new zlib.Gunzip(zlibOptions)
- response = new Response(
- // exceedingly rare that the stream would have an error,
- // but just in case we proxy it to the stream in use.
- body.on('error', /* istanbul ignore next */ er => unzip.emit('error', er)).pipe(unzip),
- responseOptions
- )
- resolve(response)
- return
- }
-
- // for deflate
- if (codings === 'deflate' || codings === 'x-deflate') {
- // handle the infamous raw deflate response from old servers
- // a hack for old IIS and Apache servers
- const raw = res.pipe(new Minipass())
- raw.once('data', chunk => {
- // see http://stackoverflow.com/questions/37519828
- const decoder = (chunk[0] & 0x0F) === 0x08
- ? new zlib.Inflate()
- : new zlib.InflateRaw()
- // exceedingly rare that the stream would have an error,
- // but just in case we proxy it to the stream in use.
- body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
- response = new Response(decoder, responseOptions)
- resolve(response)
- })
- return
- }
-
- // for br
- if (codings === 'br') {
- // ignoring coverage so tests don't have to fake support (or lack of) for brotli
- // istanbul ignore next
- try {
- var decoder = new zlib.BrotliDecompress()
- } catch (err) {
- reject(err)
- finalize()
- return
- }
- // exceedingly rare that the stream would have an error,
- // but just in case we proxy it to the stream in use.
- body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
- response = new Response(decoder, responseOptions)
- resolve(response)
- return
- }
-
- // otherwise, use response as-is
- response = new Response(body, responseOptions)
- resolve(response)
- })
-
- writeToStream(req, request)
- })
- }
-
- module.exports = fetch
-
- fetch.isRedirect = code =>
- code === 301 ||
- code === 302 ||
- code === 303 ||
- code === 307 ||
- code === 308
-
- fetch.Headers = Headers
- fetch.Request = Request
- fetch.Response = Response
- fetch.FetchError = FetchError
- fetch.AbortError = AbortError
|