User:Dsimic/Traffic stats calculation

<?php

define('FETCH_MONTH',   '01');                  // MM
define('FETCH_YEAR',    '2016');                // YYYY
define('FETCH_PROJECT', 'en.wikipedia.org');    // "en.wikipedia.org", "de.wikipedia.org", etc.

$articles = array('Stagefright (bug)',
                  'Row hammer',
                  'Address generation unit',
                  'UniDIMM',
                  'kdump (Linux)',
                  'kernfs (BSD)',
                  'kernfs (Linux)',
                  'ftrace',
                  'Android Runtime',
                  'WebScaleSQL',
                  'Intel X99',
                  'HipHop Virtual Machine',
                  'kpatch',
                  'kGraft',
                  'CoreOS',
                  'ARM Cortex-A17',
                  'Solid-state storage',
                  'Port Control Protocol',
                  'zswap',
                  'Emdebian Grip',
                  'ThinkPad 8',
                  'Laravel',
                  'OpenLMI',
                  'Open vSwitch',
                  'Distributed Overlay Virtual Ethernet',
                  'Management Component Transport Protocol',
                  'Buildroot',
                  'dm-cache',
                  'bcache',
                  'SATA Express',
                  'OpenZFS',
                  'List of Eurocrem packages',
                  'M.2',
                  'Eurocrem');

// ---------------------------------------------
// obviously, configurable stuff ends here
// ---------------------------------------------

define('CHUNK_SIZE',  10);    // articles, imposed by the Pageview API rate limit (see below)
define('CHUNK_SLEEP', 1);     // seconds, also related to the API rate limit

define('EXIT_SUCCESS', 0);    // program exit codes
define('EXIT_FAILURE', 1);

set_time_limit(0);
ini_set('memory_limit', 67108864);
ini_set('default_socket_timeout', 90);

// a few short helper functions

function plural_output($value, $unit) {
    return (number_format($value) . " {$unit}" . ((abs($value) != 1) ? 's' : ''));
}

function progress_message($message = '.') {
    static $last_message = null;

    $now     = microtime(true);
    $ret_val = false;

    if (($last_message === null) ||
        (($now - $last_message) > 0.5)) {    // one message every 0.5 seconds
        echo($message);

        $last_message = $now;
        $ret_val      = true;    // the message has been printed
    }

    return ($ret_val);
}

// prepare the cURL handles for all articles

echo("\nFetching statistics data: ");

$start_time     = microtime(true);
$handles        = array();

$articles_total = count($articles);
$day_of_month   = @date('j');
$current_month  = (FETCH_MONTH == @date('m'));

if ($articles_total == 0) {    // a small sanity check
    echo("no articles specified!\n");
    exit(EXIT_FAILURE);
}

if ($current_month && ($day_of_month == 1)) {       // account only the whole days, also knowing
    echo("no elapsed days in current month!\n");    // that the Pageview API rejects invalid dates
    exit(EXIT_FAILURE);
}

$days_total  = !$current_month
               ? cal_days_in_month(CAL_GREGORIAN, FETCH_MONTH, FETCH_YEAR)
               : ($day_of_month - 1);

$fetch_range = FETCH_YEAR . FETCH_MONTH . '01/' .
               FETCH_YEAR . FETCH_MONTH . sprintf('%02d', $days_total);

for ($id = 0; $id < $articles_total; $id++) {
    $handles[$id] = curl_init();

    curl_setopt($handles[$id], CURLOPT_URL, 'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/' .
                                            FETCH_PROJECT . '/all-access/user/' .
                                            rawurlencode(ucfirst($articles[$id])) . "/daily/{$fetch_range}");

    curl_setopt($handles[$id], CURLOPT_HEADER, false);
    curl_setopt($handles[$id], CURLOPT_RETURNTRANSFER, true);
    curl_setopt($handles[$id], CURLOPT_SSL_VERIFYPEER, false);

    curl_setopt($handles[$id], CURLOPT_CONNECTTIMEOUT, 20);
    curl_setopt($handles[$id], CURLOPT_TIMEOUT, 60);
    curl_setopt($handles[$id], CURLOPT_DNS_CACHE_TIMEOUT, 3600);

    curl_setopt($handles[$id], CURLOPT_FORBID_REUSE, false);
    curl_setopt($handles[$id], CURLOPT_FRESH_CONNECT, false);
    curl_setopt($handles[$id], CURLOPT_MAXCONNECTS, 10);
    
    curl_setopt($handles[$id], CURLOPT_USERAGENT, 'https://en.wikipedia.org/wiki/User_talk:Dsimic');
}

progress_message();

// run the cURL handles in chunks because the Pageview API imposes a rate limit,
// which, as of June 1, 2016, is specified at 10 requests per second, although
// it seems to be happily handling *much* higher rates

$handle_all     = curl_multi_init();
$chunks         = ceil(1.0 * $articles_total / CHUNK_SIZE);
$output         = array();
$error_messages = array('Parsing JSON data failed' => -1);

$views_total    = 0;
$failures       = 0;
$days_available = array();
$php_version    = explode('.', phpversion(), 3);

if (($php_version[0] >= 5) &&    // available since PHP 5.5.0
    ($php_version[1] >= 5)) {
    curl_multi_setopt($handle_all, CURLMOPT_PIPELINING, true);
    curl_multi_setopt($handle_all, CURLMOPT_MAXCONNECTS, 10);
}

for ($chunk = 0; $chunk < $chunks; $chunk++) {    // fetch one chunk at a time
    $id_limit = min(($chunk + 1) * CHUNK_SIZE, $articles_total);

    for ($id = $chunk * CHUNK_SIZE; $id < $id_limit; $id++)    // all articles in this chunk
        curl_multi_add_handle($handle_all, $handles[$id]);

    do {    // fetch the articles stats data in JSON format...
        $status = curl_multi_exec($handle_all, $running);
        progress_message();
    } while (($status == CURLM_CALL_MULTI_PERFORM) || 
             ($running > 0));

    for ($id = $chunk * CHUNK_SIZE; $id < $id_limit; $id++) {    // ... and process it
        $json = curl_multi_getcontent($handles[$id]);

        if (($json == '') ||    // is the JSON Ok?
            (($json = json_decode($json, true)) === null) ||
            !array_key_exists('items', $json) ||
            !is_array($json['items'])) {

            ++$failures;

            if (($message = curl_error($handles[$id])) != '') {        // for some reason, curl_errno()
                if (!array_key_exists($message, $error_messages)) {    // always returns zero here
                    $errno = -1 * count($error_messages) - 1;
                    $error_messages[$message] = $errno;
                }
                else    // already seen
                    $errno = $error_messages[$message];
            }
            else    // below -1 are the cURL errors
                $errno = -1;

            $output[$id] = $errno;
        }
        else {    // fetched JSON data is Ok
            $views = 0;

            foreach ($json['items'] as $json_item) {
                $views += $json_item['views'];

                if ($json_item['views'] > 0)    // complete days may be missing
                    $days_available[$json_item['timestamp']] = true;
            }

            $views_total += $views;
            $output[$id]  = $views;
        }

        curl_multi_remove_handle($handle_all, $handles[$id]);
        curl_close($handles[$id]);

        progress_message();    // done with this chunk
    }

    if ($chunk != ($chunks - 1)) {    // don't sleep after the last chunk
        $message = '#';               // all this results in smooth progress messages
        $limit   = CHUNK_SLEEP * 4;

        for ($i = 0; $i <= $limit; $i++) {
            if (progress_message($message) === true)    // print only one "marker"
                $message = '.';

            usleep(250000);
        }
    }
}

curl_multi_close($handle_all);
echo(" done.\n\n");

// done fetching all chunks of the stats data, generate and print the output...

arsort($output, SORT_NUMERIC);

$error_messages = array_flip($error_messages);
$articles_ok    = $articles_total - $failures;
$first_error    = true;

foreach ($output as $id => $views)
    if ($views >= 0)
        echo("- {$articles[$id]}: total " . plural_output($views, 'view') . "\n");
    else {
        if ($first_error && ($articles_ok > 0)) {    // display an empty line before
            echo("\n");                              // the first failure message
            $first_error = false;
        }

        echo("> {$articles[$id]}: failure ({$error_messages[$views]})\n");
    }

// ... and the final summary

$days_missing = $days_total - count($days_available);
$month_name   = @date('F', @strtotime(FETCH_YEAR . '-' . FETCH_MONTH . '-01'));

$elapsed_time = microtime(true) - $start_time;
$elapsed_min  = intval($elapsed_time / 60);
$elapsed_sec  = round($elapsed_time - $elapsed_min * 60);

echo("\nDone, {$month_name} " . FETCH_YEAR . ' statistics for ' . plural_output($articles_ok, 'article') .
     ' fetched in ' . (($elapsed_min > 0)
                       ? (plural_output($elapsed_min, 'minute') . ' and ')
                       : '') .
     plural_output($elapsed_sec, 'second') . ".\n" .
     (($failures > 0)
      ? ('Fetching the views statistics failed for ' . plural_output($failures, 'article') . ".\n")
      : ''));

if ($days_total > $days_missing) {                                          // it's entirely possible that
    $views_daily = intval($views_total / ($days_total - $days_missing));    // all days were missing

    echo('Total ' . plural_output($views_total, 'view') . ', averaging in ' .
         plural_output($views_daily, 'view') . ' per day (' .
         plural_output($days_total, ($current_month ? 'whole ' : '') . 'day') .
         ' in ' . ($current_month ? 'the current' : 'that') . ' month' .
         (($days_missing > 0)
          ? (', with the statistics unavailable for ' . plural_output($days_missing, 'day'))
          : '') .
         ").\n");
} else {                                                                    // no statistics data
    echo('Sorry, no statistics data is available at the moment for ' .
         ($current_month ? 'the current' : 'that') .  " month.\n");

    $errno = ((($days_total != $days_missing) ? 10 : 0) +    // just in case, perform some additional
              (($views_total != 0) ? 20 : 0));               // sanity checks on the internal logic

    if ($errno > 0) {
        echo("\nInternal errors detected (error code: {$errno}), please report on " .
             "https://en.wikipedia.org/wiki/User_talk:Dsimic by providing complete program output.\n");

        exit(EXIT_FAILURE);
    }
}

exit(EXIT_SUCCESS);

?>