steffen-wirth
4/28/2017 - 6:29 AM

cache warmer precache crawler by INCHOO

cache warmer precache crawler by INCHOO

<?php
/**
 * Inchoo
 *
 * NOTICE OF LICENSE
 *
 * This source file is subject to the Open Software License (OSL 3.0)
 * that is bundled with this package in the file LICENSE.txt.
 * It is also available through the world-wide-web at this URL:
 * http://opensource.org/licenses/osl-3.0.php
 * If you did not receive a copy of the license and are unable to
 * obtain it through the world-wide-web, please send an email
 * to license@magentocommerce.com so we can send you a copy immediately.
 *
 * DISCLAIMER
 *
 * Please do not edit or add to this file if you wish to upgrade
 * Magento or this extension to newer versions in the future.
 ** Inchoo *give their best to conform to
 * "non-obtrusive, best Magento practices" style of coding.
 * However,* Inchoo *guarantee functional accuracy of
 * specific extension behavior. Additionally we take no responsibility
 * for any possible issue(s) resulting from extension usage.
 * We reserve the full right not to provide any kind of support for our free extensions.
 * Thank you for your understanding.
 *
 * @category Inchoo
 * @package Precache
 * @author Marko Martinović <marko.martinovic@inchoo.net>
 * @copyright Copyright (c) Inchoo (http://inchoo.net/)
 * @license http://opensource.org/licenses/osl-3.0.php Open Software License (OSL 3.0)
 */

require_once 'abstract.php';

class Inchoo_Shell_Precache extends Mage_Shell_Abstract
{
    protected $_precacheStores = array();

    protected $_precacheCategories = array();

    protected $_precachePCount = 0;
    protected $_precacheCCount = 0;
    protected $_precacheSCount = 0;

    protected $_precacheBaseUrl;
    protected $_precacheProductSuffix;
    protected $_precacheCategorySuffix;

    public function __construct() {
        parent::__construct();

        set_time_limit(0);

        $this->_precacheBaseUrl = Mage::getBaseUrl(Mage_Core_Model_Store::URL_TYPE_WEB);

        $this->_precacheProductSuffix = Mage::helper('catalog/product')
            ->getProductUrlSuffix();

        $this->_precacheCategorySuffix = Mage::helper('catalog/category')
            ->getCategoryUrlSuffix();

        if($this->getArg('stores')) {
            $this->_precacheStores = array_merge(
                $this->_precacheStores,
                array_map(
                    'trim',
                    explode(',', $this->getArg('stores'))
                )
            );
        }

        if($this->getArg('categories')) {
            $this->_precacheCategories = array_merge(
                $this->_precacheCategories,
                array_map(
                    'trim',
                    explode(',', $this->getArg('categories'))
                )
            );
        }
    }

    public function run() {

        try {

            if(!empty($this->_precacheStores)) {
                $selectedStores = '"'.implode('", "', $this->_precacheStores).'"';
            } else {
                $selectedStores = 'All';
            }

            printf(
                'Selected stores: %s'."\n",
                $selectedStores
            );

            if(!empty($this->_precacheCategories)) {
                $selectedCategories = '"'.implode('", "', $this->_precacheCategories).'"';
            } else {
                $selectedCategories = 'All';
            }

            printf(
                'Selected categories: %s'."\n",
                $selectedCategories
            );

            echo "\n";

            $stores = Mage::app()->getStores();
            foreach ($stores as $store) {
                $this->_precacheProcessStore($store);
            }

            printf(
                'Done processing.'."\n"
                .'Total processed stores count: %d'."\n"
                .'Total processed categories count: %d'."\n"
                .'Total processed products count: %d'."\n",
                $this->_precacheSCount, $this->_precacheCCount, $this->_precachePCount
            );

        } catch (Exception $e) {
            echo $e->getMessage().'@'.time();
        }

    }

    public function usageHelp()
    {
        return <<<USAGE
Usage:  php -f precache.php -- [options]

  --stores <names>       Process only these stores (comma-separated)
  --categories <names>   Process only these categories (comma-separated)

  help                   This help

USAGE;
    }

    protected function _precacheProcessStore($store)
    {
        $storeName = $store->getName();

        if(!empty($this->_precacheStores) &&
            !in_array($storeName, $this->_precacheStores)) {
            continue;
        }

        printf('Processing "%s" store'."\n", $storeName);

        $this->_precacheSCount++;

        Mage::app()->setCurrentStore($store->getId());

        $rootCategory = Mage::getModel('catalog/category')
            ->load($store->getRootCategoryId());

        $this->_precacheProcessCategory($rootCategory, $store);

        echo "\n";
    }

    protected function _precacheProcessCategory($category, $store)
    {
        $categoryName = $category->getName();

        printf('Processing "%s" category'."\n", $categoryName);

        $this->_precacheCCount++;

        if($category->getId() !== $store->getRootCategoryId()) {
            $categoryUrl = $category->getUrl();

            printf(
                "\t".'Category URL: %s [%d]'."\n",
                $categoryUrl,
                $this->_precacheHttpRequest($categoryUrl)
            );
        }

        $productCollection = $category->getProductCollection()
            ->addAttributeToSelect('name')
            ->addAttributeToSelect('url_key');

        Mage::getSingleton('catalog/product_visibility')
            ->addVisibleInCatalogFilterToCollection($productCollection);
        Mage::getSingleton('catalog/product_status')
            ->addVisibleFilterToCollection($productCollection);

        if(!($psize = $productCollection->getSize())) {
            printf('No enabled and visible in catalog products inside this category. Continue...'."\n", $categoryName);
        }

        printf('Total product count inside this category is %d'."\n", $psize);

        foreach ($productCollection as $product) {
            $this->_precacheProcessProduct($product, $category, $store);
        }

        $categoryCollection = Mage::getModel('catalog/category')
            ->getCollection()
            ->addNameToResult()
            ->addUrlRewriteToResult()
            ->addIsActiveFilter()
            ->addAttributeToFilter('parent_id', $category->getId());

        if(!empty($this->_precacheCategories)) {
            $categoryCollection
                ->addAttributeToFilter(array(
                    array(
                        'attribute' => 'name',
                        'in' => $this->_precacheCategories,
                    )
                ));
        }

        if(!($csize = $categoryCollection->getSize())) {
            echo 'No active subcategories match inside this category. Continue...'."\n";

            return;
        }

        printf('Total subcategories count match inside this category is %d'."\n", $csize);

        foreach ($categoryCollection as $childCategory) {
            $this->_precacheProcessCategory($childCategory, $store);
        }

        echo "\n";
    }

    protected function _precacheProcessProduct($product, $category, $store)
    {
        printf('%d. %s:'."\n", ++$this->_precachePCount, $product->getSku());

        $canonicalUrl = $product->getProductUrl();

        printf(
            "\t".'Canonical URL: %s [%d]'."\n",
            $canonicalUrl,
            $this->_precacheHttpRequest($canonicalUrl)
        );

        /*
         * $category->getRequestPath() and $product->getUrlKey()
         * sometimes returns null due to bug with duplicate
         * request_path/url_key in multiple stores.
         *
         * Related Magento bug:
         * http://www.magentocommerce.com/bug-tracking/issue?issue=15035
         *
         */
        if($category->getRequestPath()) {
            $categoryUrlKey = preg_replace(
                '/'. preg_quote($this->_precacheCategorySuffix, '/') . '$/', '',
                $category->getRequestPath()
            );
            /* Fallback - use previous store's category url key if
             * that key exists or else ignore rewrite information
             */
        }

        if($categoryUrlKey && ($productUrlKey = $product->getUrlKey())) {
            // $categoryUrlKey and $productUrlKey is not null
            $categoryUrl = $this->_precacheBaseUrl;

            if($this->_isWebUrlUseStore()) {
                $categoryUrl .= $store->getCode().'/';
            }

            $categoryUrl .= $categoryUrlKey.'/'
                .$productUrlKey
                .$this->_precacheProductSuffix;

            if(!$this->_isWebUrlUseStore()) {
                $categoryUrl .= '?___store='.$store->getCode();
            }
        } else{
            // Fallback - don't use rewrite
            $categoryUrl = Mage::getUrl(
                'catalog/product/view',
                array('id' => $product->getId())
            );
        }

        printf(
            "\t".'Category URL: %s [%d]'."\n",
            $categoryUrl,
            $this->_precacheHttpRequest($categoryUrl)
        );
    }

    protected function _precacheHttpRequest($url)
    {
        $client = new Zend_Http_Client($url, array('timeout' => 60));

        $response = $client->request();

        return (int) $response->getStatus();
    }

    protected function _isWebUrlUseStore()
    {
        return (bool) Mage::getStoreConfig('web/url/use_store');
    }

}

$shell = new Inchoo_Shell_Precache();
$shell->run();