Limit Q promise concurrency in Node js

Is there any way to limit the number of concurrent Q promises to be executed at once in node js?

I am building a web scrapper, which must request and parse more 3000+ pages and without throttle some of the requests i make aren't responded to on time, so the connection rests and the needed response (html code) becomes unavailable.

To counter act this, i found that limiting the number of requests my problem goes away.

---

I have tried the following methods but to no avail:

Concurrency limit in Q promises - node

How can I limit Q promise concurrency?

https://gist.github.com/gaearon/7930162

https://github.com/ForbesLindesay/throat

I need to request an array of urls, doing only 1 request at a time and when all urls in the array have completed, then return the results in a array.

function processWebsite() {
  //computed by this stage
  urls = [u1,u2,u3,u4,l5,u6,u7,u8,u9];

  var promises = throttle(urls,1,myfunction);

  // myfunction returns a Q promise and takes a considerable 
  // amount of time to resolve (approximately 2-5 minutes)

  Q.all(promises).then(function(results){
      //work with the results of the promises array
  });
}

---

You can request a new url in a then() block

myFunction(urls[0]).then(function(result) {
  myFunction(urls[1]).then(function(result) {
    myFunction(urls[2]).then(function(result) {
      ...
    });
  });
});

Of course, this would be its dynamic behaviour. I'd mantain a queue and dequeue a single url once a promise is resolved. Then make another request. And perhaps have a hash object relating urls to results.

A second take:

var urls = ...;
var limit = ...;
var dequeue = function() {
  return an array containing up to limit
};

var myFunction = function(dequeue) {
  var urls = dequeue();

  $q.all(process urls);
};

myFunction(dequeue).then(function(result) {
  myFunction(dequeue).then(function(result) {
    myFunction(dequeue).then(function(result) {
      ...
    });
  });
});

---

I'd do this, which will iterate over each URL, building a chain of promises that run when the previous one finishes, and resolves with an array of the request results.

return urls.reduce(function(acc, url){
    return acc.then(function(results)
        return myfunction(url).then(function(requestResult){
             return results.concat(requestResult)
        });
    });
}, Q.resolve([]));

You could turn that into a helper too:

var results = map(urls, myfunction);

function map(items, fn){
    return items.reduce(function(acc, item){
        return acc.then(function(results)
            return fn(item).then(function(result){
                 return results.concat(result)
            });
        });
    }, Q.resolve([])
}

Note, the bluebird promise library has a helper to simplify this kind of thing.

return Bluebird.map(urls, myfunction, {concurrency: 1});

---

Here is my stab at making a throttled map function for Q.

function qMap(items, worker, concurrent) {
    var result = Q.defer();
    var work = [];
    var working = 0;
    var done = 0;

    concurrent = parseInt(concurrent, 10) || 1;

    function getNextIndex() {
        var i;
        for (i = 0; i < items.length; i++) {
            if (typeof work[i] === "undefined") return i;
        }
    }
    function doneWorking() {
        working--;
        done++;
        result.notify( +((100 * done / items.length).toFixed(1)) );
        if (!startWorking() && done === items.length) {
            result.resolve(work);
        }
    }
    function startWorking() {
        var index = getNextIndex();
        if (typeof index !== "undefined" && working < concurrent) {
            working++;
            work[index] = worker(items[index]).finally(doneWorking);
            return true;
        }
    }
    while (startWorking());
    return result.promise;
}

It accepts

an array of items to work on (URLs, in your case),

a worker (which must be a function that accepts an item and returns a promise)

and a maximum value of concurrent items to work on at any given time.

It returns

a promise and

resolves to an array of settled promises when all workers have finished.

It does not fail, you must inspect the individual promises to determine the overall state of the operation.

In your case you would use it like that, for example with 15 concurrent requests:

// myfunction returns a Q promise and takes a considerable 
// amount of time to resolve (approximately 2-5 minutes)

qMap(urls, myfunction, 15)
.progress(function (percentDone) {
    console.log("progress: " + percentDone);
})
.done(function (urlPromises) {
    console.log("all done: " + urlPromises);
});

链接地址: http://www.djcxy.com/p/23466.html

上一篇: 热与冷mysql模式迁移并提高速度

下一篇: 限制Q在Node js中承诺并发