jjeong

$ curl -XGET 'http://localhost:9200/_nodes?pretty=true'
{
  "cluster_name" : "cluster_node",
  "nodes" : {
    "-Rr1fhFpTy2tYjQk9qWSQw" : {
      "name" : "node2",
      "transport_address" : "inet[/127.0.0.1:9301]",
      "host" : "jeong-ui-MacBook-Pro.local",
      "ip" : "192.168.0.117",
      "version" : "1.0.0",
      "build" : "a46900e",
      "http_address" : "inet[localhost/127.0.0.1:9201]",
      "attributes" : {
        "master" : "true"
      },
      "settings" : {
        "index" : {
          "mapper" : {
            "dynamic" : "true"
          },
          "number_of_replicas" : "0",
          "number_of_shards" : "5",
          "refresh_interval" : "1s"
        },
        "gateway" : {
          "type" : "local"
        },
        "pidfile" : "es.pid",
        "network" : {
          "host" : "localhost"
        },
        "node" : {
          "data" : "true",
          "master" : "true",
          "name" : "node2"
        },
        "http" : {
          "port" : "9201",
          "enabled" : "true"
        },
        "transport" : {
          "tcp" : {
            "compress" : "true",
            "port" : "9301"
          }
        },
        "name" : "node2",
        "action" : {
          "disable_shutdown" : "true",
          "auto_create_index" : "true"
        },
        "path" : {
          "logs" : "/Users/hwjeong/server/app/elasticsearch/node2/logs",
          "home" : "/Users/hwjeong/server/app/elasticsearch/node2"
        },
        "cluster" : {
          "name" : "cluster_node"
        },
        "discovery" : {
          "zen" : {
            "minimum_master_nodes" : "2",
            "ping" : {
              "unicast" : {
                "hosts" : [ "localhost:9300", "localhost:9301", "localhost:9302" ]
              },
              "multicast" : {
                "enabled" : "false"
              }
            }
          }
        },
        "foreground" : "yes"
      },
      "os" : {
        "refresh_interval" : 1000,
        "available_processors" : 8,
        "cpu" : {
          "vendor" : "Intel",
          "model" : "MacBookPro10,1",
          "mhz" : 2400,
          "total_cores" : 8,
          "total_sockets" : 8,
          "cores_per_socket" : 16,
          "cache_size_in_bytes" : 256
        },
        "mem" : {
          "total_in_bytes" : 8589934592
        },
        "swap" : {
          "total_in_bytes" : 5368709120
        }
      },
      "process" : {
        "refresh_interval" : 1000,
        "id" : 2447,
        "max_file_descriptors" : 10240,
        "mlockall" : false
      },
      "jvm" : {
        "pid" : 2447,
        "version" : "1.6.0_65",
        "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
        "vm_version" : "20.65-b04-462",
        "vm_vendor" : "Apple Inc.",
        "start_time" : 1392355185303,
        "mem" : {
          "heap_init_in_bytes" : 268435456,
          "heap_max_in_bytes" : 1060372480,
          "non_heap_init_in_bytes" : 24317952,
          "non_heap_max_in_bytes" : 136314880,
          "direct_max_in_bytes" : 1060372480
        },
        "gc_collectors" : [ "ParNew", "ConcurrentMarkSweep" ],
        "memory_pools" : [ "Code Cache", "Par Eden Space", "Par Survivor Space", "CMS Old Gen", "CMS Perm Gen" ]
      },
      "thread_pool" : {
        "generic" : {
          "type" : "cached",
          "keep_alive" : "30s"
        },
        "index" : {
          "type" : "fixed",
          "min" : 8,
          "max" : 8,
          "queue_size" : "200"
        },
        "get" : {
          "type" : "fixed",
          "min" : 8,
          "max" : 8,
          "queue_size" : "1k"
        },
        "snapshot" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 4,
          "keep_alive" : "5m"
        },
        "merge" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 4,
          "keep_alive" : "5m"
        },
        "suggest" : {
          "type" : "fixed",
          "min" : 8,
          "max" : 8,
          "queue_size" : "1k"
        },
        "bulk" : {
          "type" : "fixed",
          "min" : 8,
          "max" : 8,
          "queue_size" : "50"
        },
        "optimize" : {
          "type" : "fixed",
          "min" : 1,
          "max" : 1
        },
        "warmer" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 4,
          "keep_alive" : "5m"
        },
        "flush" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 4,
          "keep_alive" : "5m"
        },
        "search" : {
          "type" : "fixed",
          "min" : 24,
          "max" : 24,
          "queue_size" : "1k"
        },
        "percolate" : {
          "type" : "fixed",
          "min" : 8,
          "max" : 8,
          "queue_size" : "1k"
        },
        "management" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 5,
          "keep_alive" : "5m"
        },
        "refresh" : {
          "type" : "scaling",
          "min" : 1,
          "max" : 4,
          "keep_alive" : "5m"
        }
      },
      "network" : {
        "refresh_interval" : 5000,
        "primary_interface" : {
          "address" : "192.168.0.117",
          "name" : "en0",
          "mac_address" : "28:CF:E9:14:C5:C9"
        }
      },
      "transport" : {
        "bound_address" : "inet[/127.0.0.1:9301]",
        "publish_address" : "inet[/127.0.0.1:9301]"
      },
      "http" : {
        "bound_address" : "inet[/127.0.0.1:9201]",
        "publish_address" : "inet[/127.0.0.1:9201]",
        "max_content_length_in_bytes" : 104857600
      },
      "plugins" : [ ]
    },
    .................. 요기 부터 클러스터 노드 정보들로 중복 입니다.
    }
  }
}

bin/elasticsearch -Des.pidfile=es.pid > /dev/null 2>&1 &

The main features available in 1.0 are:

• Snapshot/Restore API

  Backup or restore select indices or the whole cluster to a shared filesystem, S3 or HDFS via a simple API.

  Blog: Introducing snapshot & restore, Docs: Snapshot/Restore

• Aggregations

  Aggregations are “facets” reborn, providing more powerful, more flexible real-time analytics. Aggregations can be combined with each other and nested, to slice and dice your data exactly the way you want it. Includes the new geohash grid aggregation for geo-clustering.

  Blog: Data visualization with elasticsearch aggregations and D3, Docs: Aggregations

• Distributed Percolation

  Percolation is search reversed: instead of running a query to find matching docs, we can percolate a document to find matching queries. While percolation was already available before, this release makes percolation distributed, so that it will scale with your cluster. Percolation now supports highlighted search snippets, aggregations and bulk percolation.

  Blog: Redesigned percolator, Docs: Percolator

• cat API

  Easy to read, console-based insight into what is happening in your cluster. Particularly useful to the sysadmin when the alarm goes off at 3am and JSON is too difficult to read.

  Blog: Introducing the cat API, Docs: cat API

• Federated search

  The tribe node joins multiple clusters and acts as a federated client. Almost all operations are supported: distributed search, suggestions, percolation. You can even index into multiple clusters with the tribe node. Alternatively, you can set a tribe node to not allow any write operations, making it read-only.

  Docs: Tribe node

• Doc values

  Some of those beautiful aggregations can use a lot of memory, especially when they involve text fields. Doc values store field values on disk rather than in memory, allowing you to run aggregations on much bigger datasets, at the cost of a little performance.

  Blog: disk-based field data a.k.a. doc values, Docs: Doc Values

• Circuit breaker

  There are a few sharp edges in Elasticsearch, places where you can hurt yourself if you are not careful. We are working on adding “circuit breaker” functionality to prevent you from doing so. The first circuit breaker detects attempts to load too much fielddata into memory, which may cause OOM (out of memory) exceptions. More circuit breakers will follow.

  Docs: Fielddata circuit breaker

- index settings 에서 number_of_shards 설정 정보를 이용합니다.

- 그래서 각 node 들에 순서데로 나눠서 할당을 합니다.

- sharding 된 index shard 에 document 를 어떻게 분배 할지 결정을 합니다.

- 분배는 hash 알고리즘을 이용합니다.

- Math.abs( hash(....) % numberOfShards )

{
    "query" : {
        "term" : {
            "field" : "value"
        }
    }
}

{
    "query" : {
        "filtered" : {
            "query" : {
                "match_all" : {}
            },
            "filter" : {
                "term" : {
                    "field" : "value"
                }
            }
        }
    }
}

{
    "query" : { … },
    "filter" : { … }
}

/{index}/_search
{
    "query" : { … },
    "size" : 0
}

/{index}/_search?search_type=count
{
    "query" : { … }
}

1. Query/filter to quickly find top N results
2. Rescore with complex logic to find top 10

_source.field

_fields.field

doc[field]

input {
    file {
        path => "/server/app/logstash/log/data.json"
        codec => json
        start_position => "beginning"
    }
}

output {
    stdout { debug => true debug_format => "json"}

    elasticsearch_http {
        host => "192.168.0.120"
        port => 9200
    }
}

input {
  file {
    add_field => ... # hash (optional), default: {}
    codec => ... # codec (optional), default: "plain"
    debug => ... # boolean (optional), default: false
    discover_interval => ... # number (optional), default: 15
    exclude => ... # array (optional)
    path => ... # array (required)
    sincedb_path => ... # string (optional)
    sincedb_write_interval => ... # number (optional), default: 15
    start_position => ... # string, one of ["beginning", "end"] (optional), default: "end"
    stat_interval => ... # number (optional), default: 1
    tags => ... # array (optional)
    type => ... # string (optional)
}

}

output {
  elasticsearch_http {
    codec => ... # codec (optional), default: "plain"
    document_id => ... # string (optional), default: nil
    flush_size => ... # number (optional), default: 100
    host => ... # string (required)
    idle_flush_time => ... # number (optional), default: 1
    index => ... # string (optional), default: "logstash-%{+YYYY.MM.dd}"
    index_type => ... # string (optional)
    manage_template => ... # boolean (optional), default: true
    password => ... # password (optional), default: nil
    port => ... # number (optional), default: 9200
    replication => ... # string, one of ["async", "sync"] (optional), default: "sync"
    template => ... # a valid filesystem path (optional)
    template_name => ... # string (optional), default: "logstash"
    template_overwrite => ... # boolean (optional), default: false
    user => ... # string (optional), default: nil
    workers => ... # number (optional), default: 1
}

}

Currently, we support file system, S3, Azure and HDFS repositories.

$ curator.py -h
usage: curator.py [-h] [-v] [--host HOST] [--port PORT] [-t TIMEOUT]
                  [-p PREFIX] [-s SEPARATOR] [-C CURATION_STYLE]
                  [-T TIME_UNIT] [-d DELETE_OLDER] [-c CLOSE_OLDER]
                  [-b BLOOM_OLDER] [-g DISK_SPACE]
                  [--max_num_segments MAX_NUM_SEGMENTS] [-o OPTIMIZE] [-n]
                  [-D] [-l LOG_FILE]

Curator for Elasticsearch indices. Can delete (by space or time), close,
disable bloom filters and optimize (forceMerge) your indices.

optional arguments:
  -h, --help            show this help message and exit
  -v, --version         show program version number and exit
  --host HOST           Elasticsearch host. Default: localhost
  --port PORT           Elasticsearch port. Default: 9200
  -t TIMEOUT, --timeout TIMEOUT
                        Elasticsearch timeout. Default: 30
  -p PREFIX, --prefix PREFIX
                        Prefix for the indices. Indices that do not have this
                        prefix are skipped. Default: logstash-
  -s SEPARATOR, --separator SEPARATOR
                        Time unit separator. Default: .
  -C CURATION_STYLE, --curation-style CURATION_STYLE
                        Curate indices by [time, space] Default: time
  -T TIME_UNIT, --time-unit TIME_UNIT
                        Unit of time to reckon by: [days, hours] Default: days
  -d DELETE_OLDER, --delete DELETE_OLDER
                        Delete indices older than n TIME_UNITs.
  -c CLOSE_OLDER, --close CLOSE_OLDER
                        Close indices older than n TIME_UNITs.
  -b BLOOM_OLDER, --bloom BLOOM_OLDER
                        Disable bloom filter for indices older than n
                        TIME_UNITs.
  -g DISK_SPACE, --disk-space DISK_SPACE
                        Delete indices beyond n GIGABYTES.
  --max_num_segments MAX_NUM_SEGMENTS
                        Maximum number of segments, post-optimize. Default: 2
  -o OPTIMIZE, --optimize OPTIMIZE
                        Optimize (Lucene forceMerge) indices older than n
                        TIME_UNITs. Must increase timeout to stay connected
                        throughout optimize operation, recommend no less than
                        3600.
  -n, --dry-run         If true, does not perform any changes to the
                        Elasticsearch indices.
  -D, --debug           Debug mode
  -l LOG_FILE, --logfile LOG_FILE
                        log file