Запрет выдачи роботам

    $userAgent = strtolower($_SERVER['HTTP_USER_AGENT']);
    if (
        substr_count($userAgent, "crawler") or
        substr_count($userAgent, "yandex") or
        substr_count($userAgent, "webalta") or
        substr_count($userAgent, "mail.ru") or
        substr_count($userAgent, "googlebot") or
        substr_count($userAgent, "aport") or
        substr_count($userAgent, "rambler") or
        substr_count($userAgent, "yahoo") or
        substr_count($userAgent, "stackrambler") or
        substr_count($userAgent, "search") or
        substr_count($userAgent, "indexer") or
        substr_count($userAgent, "cuill.com") or
        substr_count($userAgent, "bingbot")
    ){
        header("Status: 404 Not Found", true, 404);
        die();
    }
    

Иначе можно записать так:

        
    $bots = array("crawler","yandex","webalta","mail.ru","googlebot","aport","rambler","yahoo","stackrambler","search","indexer","cuill.com","bingbot");
    $userAgent = $_SERVER['HTTP_USER_AGENT'];
    $isBot = ($userAgent != str_ireplace($bots,"",$userAgent));