You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

index.js 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365
  1. 'use strict'
  2. const { URL } = require('url')
  3. const http = require('http')
  4. const https = require('https')
  5. const zlib = require('minizlib')
  6. const Minipass = require('minipass')
  7. const Body = require('./body.js')
  8. const { writeToStream, getTotalBytes } = Body
  9. const Response = require('./response.js')
  10. const Headers = require('./headers.js')
  11. const { createHeadersLenient } = Headers
  12. const Request = require('./request.js')
  13. const { getNodeRequestOptions } = Request
  14. const FetchError = require('./fetch-error.js')
  15. const AbortError = require('./abort-error.js')
  16. // XXX this should really be split up and unit-ized for easier testing
  17. // and better DRY implementation of data/http request aborting
  18. const fetch = async (url, opts) => {
  19. if (/^data:/.test(url)) {
  20. const request = new Request(url, opts)
  21. // delay 1 promise tick so that the consumer can abort right away
  22. return Promise.resolve().then(() => new Promise((resolve, reject) => {
  23. let type, data
  24. try {
  25. const { pathname, search } = new URL(url)
  26. const split = pathname.split(',')
  27. if (split.length < 2) {
  28. throw new Error('invalid data: URI')
  29. }
  30. const mime = split.shift()
  31. const base64 = /;base64$/.test(mime)
  32. type = base64 ? mime.slice(0, -1 * ';base64'.length) : mime
  33. const rawData = decodeURIComponent(split.join(',') + search)
  34. data = base64 ? Buffer.from(rawData, 'base64') : Buffer.from(rawData)
  35. } catch (er) {
  36. return reject(new FetchError(`[${request.method}] ${
  37. request.url} invalid URL, ${er.message}`, 'system', er))
  38. }
  39. const { signal } = request
  40. if (signal && signal.aborted) {
  41. return reject(new AbortError('The user aborted a request.'))
  42. }
  43. const headers = { 'Content-Length': data.length }
  44. if (type) {
  45. headers['Content-Type'] = type
  46. }
  47. return resolve(new Response(data, { headers }))
  48. }))
  49. }
  50. return new Promise((resolve, reject) => {
  51. // build request object
  52. const request = new Request(url, opts)
  53. let options
  54. try {
  55. options = getNodeRequestOptions(request)
  56. } catch (er) {
  57. return reject(er)
  58. }
  59. const send = (options.protocol === 'https:' ? https : http).request
  60. const { signal } = request
  61. let response = null
  62. const abort = () => {
  63. const error = new AbortError('The user aborted a request.')
  64. reject(error)
  65. if (Minipass.isStream(request.body) &&
  66. typeof request.body.destroy === 'function') {
  67. request.body.destroy(error)
  68. }
  69. if (response && response.body) {
  70. response.body.emit('error', error)
  71. }
  72. }
  73. if (signal && signal.aborted) {
  74. return abort()
  75. }
  76. const abortAndFinalize = () => {
  77. abort()
  78. finalize()
  79. }
  80. const finalize = () => {
  81. req.abort()
  82. if (signal) {
  83. signal.removeEventListener('abort', abortAndFinalize)
  84. }
  85. clearTimeout(reqTimeout)
  86. }
  87. // send request
  88. const req = send(options)
  89. if (signal) {
  90. signal.addEventListener('abort', abortAndFinalize)
  91. }
  92. let reqTimeout = null
  93. if (request.timeout) {
  94. req.once('socket', socket => {
  95. reqTimeout = setTimeout(() => {
  96. reject(new FetchError(`network timeout at: ${
  97. request.url}`, 'request-timeout'))
  98. finalize()
  99. }, request.timeout)
  100. })
  101. }
  102. req.on('error', er => {
  103. // if a 'response' event is emitted before the 'error' event, then by the
  104. // time this handler is run it's too late to reject the Promise for the
  105. // response. instead, we forward the error event to the response stream
  106. // so that the error will surface to the user when they try to consume
  107. // the body. this is done as a side effect of aborting the request except
  108. // for in windows, where we must forward the event manually, otherwise
  109. // there is no longer a ref'd socket attached to the request and the
  110. // stream never ends so the event loop runs out of work and the process
  111. // exits without warning.
  112. // coverage skipped here due to the difficulty in testing
  113. // istanbul ignore next
  114. if (req.res) {
  115. req.res.emit('error', er)
  116. }
  117. reject(new FetchError(`request to ${request.url} failed, reason: ${
  118. er.message}`, 'system', er))
  119. finalize()
  120. })
  121. req.on('response', res => {
  122. clearTimeout(reqTimeout)
  123. const headers = createHeadersLenient(res.headers)
  124. // HTTP fetch step 5
  125. if (fetch.isRedirect(res.statusCode)) {
  126. // HTTP fetch step 5.2
  127. const location = headers.get('Location')
  128. // HTTP fetch step 5.3
  129. const locationURL = location === null ? null
  130. : (new URL(location, request.url)).toString()
  131. // HTTP fetch step 5.5
  132. if (request.redirect === 'error') {
  133. reject(new FetchError('uri requested responds with a redirect, ' +
  134. `redirect mode is set to error: ${request.url}`, 'no-redirect'))
  135. finalize()
  136. return
  137. } else if (request.redirect === 'manual') {
  138. // node-fetch-specific step: make manual redirect a bit easier to
  139. // use by setting the Location header value to the resolved URL.
  140. if (locationURL !== null) {
  141. // handle corrupted header
  142. try {
  143. headers.set('Location', locationURL)
  144. } catch (err) {
  145. /* istanbul ignore next: nodejs server prevent invalid
  146. response headers, we can't test this through normal
  147. request */
  148. reject(err)
  149. }
  150. }
  151. } else if (request.redirect === 'follow' && locationURL !== null) {
  152. // HTTP-redirect fetch step 5
  153. if (request.counter >= request.follow) {
  154. reject(new FetchError(`maximum redirect reached at: ${
  155. request.url}`, 'max-redirect'))
  156. finalize()
  157. return
  158. }
  159. // HTTP-redirect fetch step 9
  160. if (res.statusCode !== 303 &&
  161. request.body &&
  162. getTotalBytes(request) === null) {
  163. reject(new FetchError(
  164. 'Cannot follow redirect with body being a readable stream',
  165. 'unsupported-redirect'
  166. ))
  167. finalize()
  168. return
  169. }
  170. // Update host due to redirection
  171. request.headers.set('host', (new URL(locationURL)).host)
  172. // HTTP-redirect fetch step 6 (counter increment)
  173. // Create a new Request object.
  174. const requestOpts = {
  175. headers: new Headers(request.headers),
  176. follow: request.follow,
  177. counter: request.counter + 1,
  178. agent: request.agent,
  179. compress: request.compress,
  180. method: request.method,
  181. body: request.body,
  182. signal: request.signal,
  183. timeout: request.timeout,
  184. }
  185. // if the redirect is to a new hostname, strip the authorization and cookie headers
  186. const parsedOriginal = new URL(request.url)
  187. const parsedRedirect = new URL(locationURL)
  188. if (parsedOriginal.hostname !== parsedRedirect.hostname) {
  189. requestOpts.headers.delete('authorization')
  190. requestOpts.headers.delete('cookie')
  191. }
  192. // HTTP-redirect fetch step 11
  193. if (res.statusCode === 303 || (
  194. (res.statusCode === 301 || res.statusCode === 302) &&
  195. request.method === 'POST'
  196. )) {
  197. requestOpts.method = 'GET'
  198. requestOpts.body = undefined
  199. requestOpts.headers.delete('content-length')
  200. }
  201. // HTTP-redirect fetch step 15
  202. resolve(fetch(new Request(locationURL, requestOpts)))
  203. finalize()
  204. return
  205. }
  206. } // end if(isRedirect)
  207. // prepare response
  208. res.once('end', () =>
  209. signal && signal.removeEventListener('abort', abortAndFinalize))
  210. const body = new Minipass()
  211. // if an error occurs, either on the response stream itself, on one of the
  212. // decoder streams, or a response length timeout from the Body class, we
  213. // forward the error through to our internal body stream. If we see an
  214. // error event on that, we call finalize to abort the request and ensure
  215. // we don't leave a socket believing a request is in flight.
  216. // this is difficult to test, so lacks specific coverage.
  217. body.on('error', finalize)
  218. // exceedingly rare that the stream would have an error,
  219. // but just in case we proxy it to the stream in use.
  220. res.on('error', /* istanbul ignore next */ er => body.emit('error', er))
  221. res.on('data', (chunk) => body.write(chunk))
  222. res.on('end', () => body.end())
  223. const responseOptions = {
  224. url: request.url,
  225. status: res.statusCode,
  226. statusText: res.statusMessage,
  227. headers: headers,
  228. size: request.size,
  229. timeout: request.timeout,
  230. counter: request.counter,
  231. trailer: new Promise(resolveTrailer =>
  232. res.on('end', () => resolveTrailer(createHeadersLenient(res.trailers)))),
  233. }
  234. // HTTP-network fetch step 12.1.1.3
  235. const codings = headers.get('Content-Encoding')
  236. // HTTP-network fetch step 12.1.1.4: handle content codings
  237. // in following scenarios we ignore compression support
  238. // 1. compression support is disabled
  239. // 2. HEAD request
  240. // 3. no Content-Encoding header
  241. // 4. no content response (204)
  242. // 5. content not modified response (304)
  243. if (!request.compress ||
  244. request.method === 'HEAD' ||
  245. codings === null ||
  246. res.statusCode === 204 ||
  247. res.statusCode === 304) {
  248. response = new Response(body, responseOptions)
  249. resolve(response)
  250. return
  251. }
  252. // Be less strict when decoding compressed responses, since sometimes
  253. // servers send slightly invalid responses that are still accepted
  254. // by common browsers.
  255. // Always using Z_SYNC_FLUSH is what cURL does.
  256. const zlibOptions = {
  257. flush: zlib.constants.Z_SYNC_FLUSH,
  258. finishFlush: zlib.constants.Z_SYNC_FLUSH,
  259. }
  260. // for gzip
  261. if (codings === 'gzip' || codings === 'x-gzip') {
  262. const unzip = new zlib.Gunzip(zlibOptions)
  263. response = new Response(
  264. // exceedingly rare that the stream would have an error,
  265. // but just in case we proxy it to the stream in use.
  266. body.on('error', /* istanbul ignore next */ er => unzip.emit('error', er)).pipe(unzip),
  267. responseOptions
  268. )
  269. resolve(response)
  270. return
  271. }
  272. // for deflate
  273. if (codings === 'deflate' || codings === 'x-deflate') {
  274. // handle the infamous raw deflate response from old servers
  275. // a hack for old IIS and Apache servers
  276. const raw = res.pipe(new Minipass())
  277. raw.once('data', chunk => {
  278. // see http://stackoverflow.com/questions/37519828
  279. const decoder = (chunk[0] & 0x0F) === 0x08
  280. ? new zlib.Inflate()
  281. : new zlib.InflateRaw()
  282. // exceedingly rare that the stream would have an error,
  283. // but just in case we proxy it to the stream in use.
  284. body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
  285. response = new Response(decoder, responseOptions)
  286. resolve(response)
  287. })
  288. return
  289. }
  290. // for br
  291. if (codings === 'br') {
  292. // ignoring coverage so tests don't have to fake support (or lack of) for brotli
  293. // istanbul ignore next
  294. try {
  295. var decoder = new zlib.BrotliDecompress()
  296. } catch (err) {
  297. reject(err)
  298. finalize()
  299. return
  300. }
  301. // exceedingly rare that the stream would have an error,
  302. // but just in case we proxy it to the stream in use.
  303. body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
  304. response = new Response(decoder, responseOptions)
  305. resolve(response)
  306. return
  307. }
  308. // otherwise, use response as-is
  309. response = new Response(body, responseOptions)
  310. resolve(response)
  311. })
  312. writeToStream(req, request)
  313. })
  314. }
  315. module.exports = fetch
  316. fetch.isRedirect = code =>
  317. code === 301 ||
  318. code === 302 ||
  319. code === 303 ||
  320. code === 307 ||
  321. code === 308
  322. fetch.Headers = Headers
  323. fetch.Request = Request
  324. fetch.Response = Response
  325. fetch.FetchError = FetchError
  326. fetch.AbortError = AbortError