Problem with Xenforo and NGINX

[DRaver]

I have a problem with XenForo and I think it's up to my NGINX server configuration.
The problem is that sometimes the Googlebot can not access to resources such as images or scripts.
Googlebot says the images are (Vorrübergehend nicht erreichbar) currently not available.

Here's an example.



It looks as if there is a limit for loading files.

Here my nginx.conf

user www-data;
worker_processes 8;
pid /run/nginx.pid;

events {
    worker_connections 4000;
     multi_accept on;
     use epoll;
}

http {
geoip_country /usr/share/GeoIP/GeoIP.dat;
map $geoip_country_code $allowed_country {
default yes;
CN no;
TW no;
TR no;
RU no;
IR no;
UA no;
GE no;
TH no;
RO no;
PH no;
BA no;
LV no;
LT no;
EE no;
HR no;
AL no;
RS no;
AF no;
IN no;
BR no;

}
    ##
    # Basic Settings
    ##

    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    keepalive_timeout 10;
    types_hash_max_size 2048;
    send_timeout 60;
    server_tokens off;
    client_max_body_size 100m;
    client_body_timeout 30;
    client_header_timeout 30;
  

    # server_names_hash_bucket_size 64;
    # server_name_in_redirect off;

    include /etc/nginx/mime.types;
    default_type application/octet-stream;

# File Cache Settings
    ##
    open_file_cache          max=5000  inactive=20s;
    open_file_cache_valid    30s;
    open_file_cache_min_uses 2;
    open_file_cache_errors   on;
    ##

    ##
    # Logging Settings
    ##

    access_log /var/log/nginx/access.log;
    error_log /var/log/nginx/error.log;

    ##
    # Gzip Settings
    ##

    gzip on;
    gzip_disable "msie6";

     gzip_vary on;
     gzip_proxied any;
     gzip_comp_level 6;
     gzip_buffers 16 8k;
     gzip_http_version 1.1;
     gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;

    ##
    # nginx-naxsi config
    ##
    # Uncomment it if you installed nginx-naxsi
    ##

    #include /etc/nginx/naxsi_core.rules;

    ##
    # Virtual Host Configs
    ##

    include /etc/nginx/conf.d/*.conf;
    include /etc/nginx/sites-enabled/*;
}


#mail {
#    # See sample authentication script at:
#    # http://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
#
#    # auth_http localhost/auth.php;
#    # pop3_capabilities "TOP" "USER";
#    # imap_capabilities "IMAP4rev1" "UIDPLUS";
#
#    server {
#        listen     localhost:110;
#        protocol   pop3;
#        proxy      on;
#    }
#
#    server {
#        listen     localhost:143;
#        protocol   imap;
#        proxy      on;
#    }
#}

and here is my conf file for this domain:

server {
       listen 5.x.x.x:80;
       server_name my-website.de;
      return       301 http://www.my-website.de$request_uri;
}
server {
       listen 5.x.x.x:80;
       server_name www.my-website.de;
    root /var/www/my-website;
    index index.php index.html index.htm;
  access_log  /var/log/nginx/my-website.de.access.log;
    error_log /var/log/nginx/my-website.de.error.log;
    # Make site accessible from http://localhost/
    #server_name localhost;

if ($allowed_country = no) {
return 444;
}

location / {
try_files $uri $uri/ /index.php?$uri&$args;

location ~/admin\.php$ {
                        auth_basic "Administrator Login";
                        auth_basic_user_file /var/www/htpasswd;
                        root /var/www/my-website;
                        try_files $uri =404;
                        fastcgi_pass unix:/var/run/php5-fpm.sock;
                        fastcgi_index index.php;
                        fastcgi_read_timeout 300;
                        fastcgi_param  HTTP_SCHEME https;
                        include fastcgi_params;
                }

  


# Media: images, icons, video, audio, HTC
        location ~* \.(?:jpg|jpeg|gif|png|ico|gz|svg|svgz|mp4|ogg|ogv|webm|htc|woff)$ {
            expires 1M;
            access_log off;
            add_header Cache-Control "public";
    }
  
# CSS and Javascript
        location ~* \.(?:css|js)$ {
            expires 1y;
            access_log off;
            add_header Cache-Control "public";
    }


    location ~ /(internal_data|library) {
         internal;
    }

location /install {
        auth_basic "Administrator Login";
        auth_basic_user_file /var/www/htpasswd;
        index index.php index.html index.htm;
        }

    location ~ \.php$ {
        fastcgi_pass unix:/var/run/php5-fpm.sock;
        fastcgi_index  index.php;
        fastcgi_param  SCRIPT_FILENAME  $document_root$fastcgi_script_name;
        include fastcgi_params;
        fastcgi_read_timeout 300;
        fastcgi_send_timeout 180;
        fastcgi_connect_timeout 60;
        fastcgi_ignore_client_abort off;
        fastcgi_intercept_errors on;
    } 
}

The problem occurs only when the first access. IF I repeat the rendering of the page, there is no problem.

Please can someone help me?

 

[DRaver]

nobody here with NGINX experience?

 

[Mouth]

No limiting in any of those configs. Perhaps googlebot is sourcing from one of your geo blocked countries?
Check your '/var/log/nginx/my-website.de.access.log' for the request from Googlebot and see what your server is returning?

 

[HWS]

Do you allow Gravatar? Sometimes they have issues.

 

[ForumCube]

@Mouth is right about the cause of this issue. You have to exclude Google bots from geo location blocking.

This is pretty common issue and you will easily able to find a lot of solutions on nginx forums but to get you started here's what might help you to exclude Google bots

#Whitelist crawlers
map $http_user_agent $crawler {
default 0;

~*(AdsBot-Google|Googlebot-Mobile|Googlebot-Image|Mediapartners-Google|
bingbot|Feedfetcher-Google|Googlebot|Yahoo\ !Slurp|msnbot|msnbot-media|
YahooCacheSystem) 1;
}
# You'll also probably need to override for specific IP addresses too...
geo $whitelisted {
default $crawler;
}

 

[DRaver]

but to get you started here's what might help you to exclude Google bots

Where should I paste the code?

 

[ForumCube]

Where should I paste the code?

This code is for your nginx.conf file.

 

[DRaver]

I do not get it. The problem is still there.

So I checked the the this currently not available files in the access log an I found vor the Google Search Console bot a code 200. That's crazy.