Slow search index rebuild

[dutchbb]

Can't seem to find the problem of this. With 9mil posts it takes arount 7 hours in batches of 10k from browser.

Before heap_size was set to 1GB, so i increased it to 2GB and is a little faster but not much. We have a fast SSD, 16GB of RAM and 6 x Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz

We are using Elasticsearch 6, CPU usage of elasticsearch is around 1% (even when rebuilding) and RAM 18%. Maybe increase heap_size further to like 5GB?

Server has been optimized and running fast without any problems.

mysqld

performance-schema=0
ignore_db_dirs=cmsetiofiotest
local-infile=0
ignore_db_dirs=lost+found

innodb=ON
back_log = 512
max_connections = 100
key_buffer_size = 256M
myisam_sort_buffer_size = 256M
myisam_max_sort_file_size = 2048M
join_buffer_size = 256K
read_buffer_size = 256K
sort_buffer_size = 256K
table_definition_cache = 8192
table_open_cache = 4096
thread_cache_size = 256
wait_timeout = 1800
connect_timeout = 10
tmp_table_size = 256M
max_heap_table_size = 256M
max_allowed_packet=268435456
max_seeks_for_key = 1000
group_concat_max_len = 1024
max_length_for_sort_data = 1024
net_buffer_length = 16384
max_connect_errors = 100000
concurrent_insert = 2
read_rnd_buffer_size = 512K
bulk_insert_buffer_size = 8M
query_cache_limit = 1024K
query_cache_size = 80M
query_cache_type = 1
query_cache_min_res_unit = 2K
query_prealloc_size = 262144
query_alloc_block_size = 65536
transaction_alloc_block_size = 8192
transaction_prealloc_size = 4096
default-storage-engine = InnoDB


innodb_large_prefix=1
innodb_purge_threads = 4
innodb_file_format = Barracuda
innodb_file_per_table = 1
innodb_open_files = 1000
innodb_data_file_path= ibdata1:10M:autoextend
innodb_buffer_pool_size = 8G

innodb_buffer_pool_instances = 8

innodb_log_files_in_group = 2
innodb_log_file_size = 256M
innodb_log_buffer_size = 8M
innodb_flush_log_at_trx_commit = 2
innodb_lock_wait_timeout=50
innodb_flush_method = O_DIRECT
innodb_support_xa=1

innodb_io_capacity = 600
innodb_io_capacity_max = 1200
innodb_read_io_threads = 4
innodb_write_io_threads = 4

open_files_limit=10000
[mariadb]
userstat = 0
key_cache_segments = 1
aria_group_commit = none
aria_group_commit_interval = 0
aria_log_file_size = 768M
aria_log_purge_type = immediate
aria_pagecache_buffer_size = 768M
aria_sort_buffer_size = 192M

[mariadb-5.5]
innodb_file_format = Barracuda
innodb_file_per_table = 1

query_cache_strip_comments=0

[mysqld_safe]
socket          = /var/lib/mysql/mysql.sock
open-files-limit = 8192

[myisamchk]
key_buffer = 256M
sort_buffer = 32M
read_buffer = 32M
write_buffer = 32M

[mysqlhotcopy]
interactive-timeout

[mariadb-10.0]
innodb_file_format = Barracuda
innodb_file_per_table = 1

innodb_buffer_pool_populate=0
performance_schema=OFF
innodb_stats_on_metadata=OFF
innodb_sort_buffer_size=2M
innodb_online_alter_log_max_size=128M
query_cache_strip_comments=0
log_slow_filter =admin,filesort,filesort_on_disk,full_join,full_scan,query_cache,query_cache_miss,tmp_table,tmp_table_on_disk

 

[dutchbb]

I am thinking these values could be increased for writing to disk? When rebuilding search index i am getting 20 w/s so that could be the cause.

innodb_io_capacity = 600
innodb_io_capacity_max = 1200
innodb_read_io_threads = 4
innodb_write_io_threads = 4