cURL asynchronous requests using curl_multi and callbacks
<?php
/**
* Make asynchronous requests to different resources as fast as possible and process the results as they are ready.
*/
class Requests
{
public $handle;
public function __construct()
{
$this->handle = curl_multi_init();
}
public function process($urls, $callback)
{
foreach($urls as $url)
{
$ch = curl_init($url);
curl_setopt_array($ch, array(CURLOPT_RETURNTRANSFER => TRUE));
curl_multi_add_handle($this->handle, $ch);
}
do {
$mrc = curl_multi_exec($this->handle, $active);
if ($state = curl_multi_info_read($this->handle))
{
//print_r($state);
$info = curl_getinfo($state['handle']);
//print_r($info);
$callback(curl_multi_getcontent($state['handle']), $info);
curl_multi_remove_handle($this->handle, $state['handle']);
}
usleep(10000); // stop wasting CPU cycles and rest for a couple ms
} while ($mrc == CURLM_CALL_MULTI_PERFORM || $active);
}
public function __destruct()
{
curl_multi_close($this->handle);
}
}
<?php
require('Requests.php');
// We want the requests to be mixed fast/slow so we can see if it's working
$slow = 'http://localhost/curl_multi/slow.php';
$fast = 'http://localhost/curl_multi/fast.php';
$urls = array();
for ($i = 0; $i < 10; $i++)
{
if($i % 2)
{
print "Slow\n";
$urls[] = $slow;
}
else
{
print "Fast\n";
$urls[] = $fast;
}
}
$callback = function($data, $info)
{
print "Callback: ";
print_r($data);
print_r($info);
};
$requests = new Requests();
$requests->process($urls, $callback);
<?php
usleep(mt_rand(10000, 200000));
print basename(__FILE__) . "\n";
<?php
// This limits the request rate to the size of the $rolling_window
// http://www.onlineaspect.com/2009/01/26/how-to-use-curl_multi-without-blocking/
function rolling_curl($urls, $callback, $custom_options = null)
{
// make sure the rolling window isn't greater than the # of urls
$rolling_window = 5;
$rolling_window = (sizeof($urls) < $rolling_window) ? sizeof($urls) : $rolling_window;
$master = curl_multi_init();
$curl_arr = array();
// add additional curl options here
$std_options = array(
CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_MAXREDIRS => 5
);
$options = ($custom_options) ? ($std_options + $custom_options) : $std_options;
// start the first batch of requests
for ($i = 0; $i < $rolling_window; $i++)
{
$ch = curl_init();
$options[CURLOPT_URL] = $urls[$i];
curl_setopt_array($ch,$options);
curl_multi_add_handle($master, $ch);
}
do {
while(($execrun = curl_multi_exec($master, $running)) == CURLM_CALL_MULTI_PERFORM);
if($execrun != CURLM_OK)
break;
// a request was just completed -- find out which one
while($done = curl_multi_info_read($master))
{
$info = curl_getinfo($done['handle']);
$output = curl_multi_getcontent($done['handle']);
// request successful. process output using the callback function.
$callback($output, $info);
if(isset($urls[$i + 1]))
{
// start a new request (it's important to do this before removing the old one)
$ch = curl_init();
$options[CURLOPT_URL] = $urls[$i++]; // increment i
curl_setopt_array($ch,$options);
curl_multi_add_handle($master, $ch);
}
// remove the curl handle that just completed
curl_multi_remove_handle($master, $done['handle']);
}
usleep(10000); // stop wasting CPU cycles and rest for a couple ms
} while ($running);
curl_multi_close($master);
}
<?php
usleep(mt_rand(100000, 600000));
print basename(__FILE__) . "\n";