Gist

Processing a list of operations sequentially - JS fundamentals

JavaScript guide to processing tasks sequentially (without rate-limit issues). Learn why async/await with forEach fails, and use for...of loops or a batched promise runner to run operations with delays.

When trying to write migration scripts, one of the most common patterns that one would want to solve for is not hitting API rate limits and also wait enough before processing the next item.

Problem

// list of tasks
const tasks = [1,2,3,4,5,6,7,8,9,10]

// simple processor to print the item, can by async
const processor = (item) => {
	console.log(item)
}

async function main() {
	await tasks.forEach( async (item) => {
    processor(item)
    await new Promise(r => setTimeout(r, 2000));
  })
  console.log('complete')
}

main()

Why will above not work

  1. forEach does not await.
  2. await in processor is redundant and has no impact

Solution 1 - using a simple for loop

const tasks = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10];

const processor = (item) => {
  console.log(item);
};

async function main() {
  for (const item of tasks) {
    processor(item);
    await new Promise((r) => setTimeout(r, 2000)); // Wait 2 seconds before processing the next item
  }
  console.log('complete');
}

main();

Solution 2 - create a batched function call

const tasks = [1,2,3,4,5,6,7,8,9,10]

// simple processor to print the item, can by async
const processor = (item) => {
	console.log(item)
}

const batchedCall = (list, cb, interval) => {
	
	async function processIndex(idx, list, cb, interval, r) {
		// resolve if the list is complete.
  	if (idx >=list.length) {
    	r();
      return;
    }
    // process the task, await here is optional
  	await cb(list[idx]);
  	// wait for the mentioned timeout
    await new Promise(r => setTimeout(r, interval));
    // process the next index
    processIndex(idx+1, list,cb,interval,r)
  }
  
  // call the processIndex function with first index
  return new Promise( r => processIndex(0, list,cb,interval, r));
}

async function main() {
  await batchedCall(tasks,processor,1000)
  console.log('complete')
}

main();