You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

path-reservations.js 4.3KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156
  1. // A path exclusive reservation system
  2. // reserve([list, of, paths], fn)
  3. // When the fn is first in line for all its paths, it
  4. // is called with a cb that clears the reservation.
  5. //
  6. // Used by async unpack to avoid clobbering paths in use,
  7. // while still allowing maximal safe parallelization.
  8. const assert = require('assert')
  9. const normalize = require('./normalize-unicode.js')
  10. const stripSlashes = require('./strip-trailing-slashes.js')
  11. const { join } = require('path')
  12. const platform = process.env.TESTING_TAR_FAKE_PLATFORM || process.platform
  13. const isWindows = platform === 'win32'
  14. module.exports = () => {
  15. // path => [function or Set]
  16. // A Set object means a directory reservation
  17. // A fn is a direct reservation on that path
  18. const queues = new Map()
  19. // fn => {paths:[path,...], dirs:[path, ...]}
  20. const reservations = new Map()
  21. // return a set of parent dirs for a given path
  22. // '/a/b/c/d' -> ['/', '/a', '/a/b', '/a/b/c', '/a/b/c/d']
  23. const getDirs = path => {
  24. const dirs = path.split('/').slice(0, -1).reduce((set, path) => {
  25. if (set.length) {
  26. path = join(set[set.length - 1], path)
  27. }
  28. set.push(path || '/')
  29. return set
  30. }, [])
  31. return dirs
  32. }
  33. // functions currently running
  34. const running = new Set()
  35. // return the queues for each path the function cares about
  36. // fn => {paths, dirs}
  37. const getQueues = fn => {
  38. const res = reservations.get(fn)
  39. /* istanbul ignore if - unpossible */
  40. if (!res) {
  41. throw new Error('function does not have any path reservations')
  42. }
  43. return {
  44. paths: res.paths.map(path => queues.get(path)),
  45. dirs: [...res.dirs].map(path => queues.get(path)),
  46. }
  47. }
  48. // check if fn is first in line for all its paths, and is
  49. // included in the first set for all its dir queues
  50. const check = fn => {
  51. const { paths, dirs } = getQueues(fn)
  52. return paths.every(q => q[0] === fn) &&
  53. dirs.every(q => q[0] instanceof Set && q[0].has(fn))
  54. }
  55. // run the function if it's first in line and not already running
  56. const run = fn => {
  57. if (running.has(fn) || !check(fn)) {
  58. return false
  59. }
  60. running.add(fn)
  61. fn(() => clear(fn))
  62. return true
  63. }
  64. const clear = fn => {
  65. if (!running.has(fn)) {
  66. return false
  67. }
  68. const { paths, dirs } = reservations.get(fn)
  69. const next = new Set()
  70. paths.forEach(path => {
  71. const q = queues.get(path)
  72. assert.equal(q[0], fn)
  73. if (q.length === 1) {
  74. queues.delete(path)
  75. } else {
  76. q.shift()
  77. if (typeof q[0] === 'function') {
  78. next.add(q[0])
  79. } else {
  80. q[0].forEach(fn => next.add(fn))
  81. }
  82. }
  83. })
  84. dirs.forEach(dir => {
  85. const q = queues.get(dir)
  86. assert(q[0] instanceof Set)
  87. if (q[0].size === 1 && q.length === 1) {
  88. queues.delete(dir)
  89. } else if (q[0].size === 1) {
  90. q.shift()
  91. // must be a function or else the Set would've been reused
  92. next.add(q[0])
  93. } else {
  94. q[0].delete(fn)
  95. }
  96. })
  97. running.delete(fn)
  98. next.forEach(fn => run(fn))
  99. return true
  100. }
  101. const reserve = (paths, fn) => {
  102. // collide on matches across case and unicode normalization
  103. // On windows, thanks to the magic of 8.3 shortnames, it is fundamentally
  104. // impossible to determine whether two paths refer to the same thing on
  105. // disk, without asking the kernel for a shortname.
  106. // So, we just pretend that every path matches every other path here,
  107. // effectively removing all parallelization on windows.
  108. paths = isWindows ? ['win32 parallelization disabled'] : paths.map(p => {
  109. // don't need normPath, because we skip this entirely for windows
  110. return stripSlashes(join(normalize(p))).toLowerCase()
  111. })
  112. const dirs = new Set(
  113. paths.map(path => getDirs(path)).reduce((a, b) => a.concat(b))
  114. )
  115. reservations.set(fn, { dirs, paths })
  116. paths.forEach(path => {
  117. const q = queues.get(path)
  118. if (!q) {
  119. queues.set(path, [fn])
  120. } else {
  121. q.push(fn)
  122. }
  123. })
  124. dirs.forEach(dir => {
  125. const q = queues.get(dir)
  126. if (!q) {
  127. queues.set(dir, [new Set([fn])])
  128. } else if (q[q.length - 1] instanceof Set) {
  129. q[q.length - 1].add(fn)
  130. } else {
  131. q.push(new Set([fn]))
  132. }
  133. })
  134. return run(fn)
  135. }
  136. return { check, reserve }
  137. }