Elasticsearch Data too large Error排查过程

背景

 当时线上的kibana全都连不上，然后不管kibana还是es均在报同样的错误

[parent] Data too large, data for [<transport_request>] would be larger than

OR

[parent] Data too large, data for [<http_request>] would be larger than limit of [23941899878/22.2gb], with { bytes_wanted=23941987633 bytes_limit=23941899878 }"

 大概都是在说数据太大，超出了什么限制。ok，经过一番Google、baidu终于有了一点眉目.

问题排查

 网上写到，出现类似问题的原因在于fielddata的内存被占用完了，其他索引无法分配更多的内存

通过命令查看当前的字段内存分配情况

GET /_stats/fielddata?fields=*
# 然后我们可以得到类似于如下信息
"logstash-2016.04.02": {
  "primaries": {
    "fielddata": {
      "memory_size_in_bytes": 53009116,
      "evictions": 0,
      "fields": {
      }
    }
  },
  "total": {
    "fielddata": {
      "memory_size_in_bytes": 53009116,
      "evictions": 0,
      "fields": {
      }
    }
  }
},
"logstash-2016.04.29": {
  "primaries": {
    "fielddata": {
      "memory_size_in_bytes":0,
      "evictions": 0,
      "fields": {
      }
    }
  },
  "total": {
    "fielddata": {
      "memory_size_in_bytes":0,
      "evictions": 0,
      "fields": {
      }
    }
  }
},

 通过以上信息，我们可以看出logstash-2016.04.02的memory_size_in_bytes为53009116，但logstash-2016.04.29的memory_size_in_bytes为0。我们可以确认之前的索引占用了太多的内存，导致后来的索引没有内存可用，从而导致该索引在查询过程中出现异常。

解决方案

大招1
在es参数中，可以通过设置indices.fielddata.cache.size值来修改单个索引占用缓存的大小，如果超出这个值，该数据将被逐出。这样的方式可以帮助你合理的分配你有限的内存，但是不能改变你的内存。所以终极解决办法还是增加你的内存大小。

## 编辑ES配置文件config/elasticsearch.yml，添加一下配置，然后restart集群
indices.fielddata.cache.size:  20%

 如果你觉得以上方式太暴力，可以通过命令行方式修改配置

PUT _cluster/settings
{
  "persistent" : {
    "indices.breaker.fielddata.limit" : "20%" 
  }
}

大招2
问题解决了吗？什么没有？？？OK,如果以上方式依然没有解决你的问题。第二大招来了。出现以上问题的根源就是ES的fielddata缓存空间被用完了，那么我们可以清除ES缓存

## 清除单个索引缓存
POST twitter/_cache/clear
##  清除多个索引缓存
POST kimchy,elasticsearch/_cache/clear
POST _cache/clear

 好啦，到目前为此，希望你的集群已经健康的变绿。

另外，以下为查询集群的统计信息

GET _cluster/stats
## 得到结果
{  
  "timestamp" : 1401257069496,  
  "cluster_name" : "if2c",  
  "status" : "yellow",  
  "indices" : {  
    "count" : 3,  
    "shards" : {  
      "total" : 3,  
      "primaries" : 3,  
      "replication" : 0.0,  
      "index" : {  
        "shards" : {  
          "min" : 1,  
          "max" : 1,  
          "avg" : 1.0  
        },  
        "primaries" : {  
          "min" : 1,  
          "max" : 1,  
          "avg" : 1.0  
        },  
        "replication" : {  
          "min" : 0.0,  
          "max" : 0.0,  
          "avg" : 0.0  
        }  
      }  
    },  
    "docs" : {                    //文档信息  
      "count" : 39642,  
      "deleted" : 0  
    },  
    "store" : {                   //存储信息  
      "size" : "15.4mb",  
      "size_in_bytes" : 16235013,  
      "throttle_time" : "208.7ms",  
      "throttle_time_in_millis" : 208  
    },  
    "fielddata" : {                //字段信息  
      "memory_size" : "1.7mb",  
      "memory_size_in_bytes" : 1846772,  
      "evictions" : 0  
    },  
    "filter_cache" : {            //缓存  
      "memory_size" : "182.6kb",  
      "memory_size_in_bytes" : 187045,  
      "evictions" : 0  
    },  
    "id_cache" : {  
      "memory_size" : "0b",  
      "memory_size_in_bytes" : 0  
    },  
    "completion" : {  
      "size" : "219.6kb",  
      "size_in_bytes" : 224886  
    },  
    "segments" : {              //段信息，luence段  
      "count" : 10,  
      "memory" : "0b",  
      "memory_in_bytes" : 0  
    }  
  },  
  "nodes" : {                         //节点信息  
    "count" : {  
      "total" : 1,  
      "master_only" : 0,  
      "data_only" : 0,  
      "master_data" : 1,  
      "client" : 0  
    },  
    "versions" : [ "0.90.9" ],      版本  
    "os" : {  
      "available_processors" : 8,  
      "mem" : {                         //内存  
        "total" : "7.6gb",  
        "total_in_bytes" : 8254771200  
      },  
      "cpu" : [ {                     //CPU信息  
        "vendor" : "Intel",  
        "model" : "Xeon",  
        "mhz" : 1995,  
        "total_cores" : 8,  
        "total_sockets" : 4,  
        "cores_per_socket" : 2,  
        "cache_size" : "4kb",  
        "cache_size_in_bytes" : 4096,  
        "count" : 1  
      } ]  
    },  
    "process" : {  
      "cpu" : {  
        "percent" : 0  
      },  
      "avg_open_file_descriptors" : 435  
    },  
    "jvm" : {                                //JVM信息  
      "max_uptime" : "1.8d",  
      "max_uptime_in_millis" : 159452573,  
      "versions" : [ {  
        "version" : "1.7.0_15",  
        "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",  
        "vm_version" : "23.7-b01",  
        "vm_vendor" : "Oracle Corporation",  
        "count" : 1  
      } ],  
      "mem" : {  
        "heap_used" : "337.3mb",  
        "heap_used_in_bytes" : 353722720,  
        "heap_max" : "989.8mb",  
        "heap_max_in_bytes" : 1037959168  
      },  
      "threads" : 112  
    },  
    "fs" : {  
      "total" : "35.2gb",  
      "total_in_bytes" : 37840687104,  
      "free" : "20.2gb",  
      "free_in_bytes" : 21713711104,  
      "available" : "18.4gb",  
      "available_in_bytes" : 19791515648,  
      "disk_reads" : 73151,  
      "disk_writes" : 2445649,  
      "disk_io_op" : 2518800,  
      "disk_read_size" : "1.4gb",  
      "disk_read_size_in_bytes" : 1605862400,  
      "disk_write_size" : "19.6gb",  
      "disk_write_size_in_bytes" : 21100687360,  
      "disk_io_size" : "21.1gb",  
      "disk_io_size_in_bytes" : 22706549760,  
      "disk_queue" : "0",  
      "disk_service_time" : "8.2"  
    },  
    "plugins" : [ {                               //插件信息  
      "name" : "bigdesk",  
      "description" : "No description found for bigdesk.",  
      "url" : "/_plugin/bigdesk/",  
      "jvm" : false,  
      "site" : true  
    },{  
      "name" : "inquisitor",  
      "description" : "No description found for inquisitor.",  
      "url" : "/_plugin/inquisitor/",  
      "jvm" : false,  
      "site" : true  
    }, {  
      "name" : "analysis-ansj",  
      "description" : "ansj analysis",  
      "jvm" : true,  
      "site" : false  
    }, {  
      "name" : "head",  
      "description" : "No description found for head.",  
      "url" : "/_plugin/head/",  
      "jvm" : false,  
      "site" : true  
    } ]  
  }  
}