Elasticsearch (Docker) Install

module.exports =  header: 'Docker ES Install', handler: (options) ->

Identities

  @system.group options.group
  @system.user options.user

Limits

  @system.limits
    header: 'Ulimit'
    user: options.user.name
  , options.user.limits

Kernel

Configure kernel parameters at runtime. There are no properties set by default, here's a suggestion:

• vm.swappiness = 10
• vm.overcommit_memory = 1
• vm.overcommit_ratio = 100
• net.core.somaxconn = 4096 (default socket listen queue size 128)

Note, we might move this middleware to Masson.

  @tools.sysctl
    header: 'Kernel'
    properties: options.sysctl
    merge: true
    comment: true

SSL Certificate

  @file.download
    header: 'SSL CAcert'
    source: options.ssl.cacert.source
    local: options.ssl.cacert.local
    target: options.ssl.dest_cacert
    mode: 0o0640
  @file.download
    header: 'SSL cert'
    source: options.ssl.cert.source
    local: options.ssl.cert.local
    target: options.ssl.dest_cert
    mode: 0o0640
  @file.download
    header: 'SSL key'
    source: options.ssl.key.source
    local: options.ssl.key.local
    target: options.ssl.dest_key
    mode: 0o0640

Write YAML Files

  for es_name,es of options.clusters then do (es_name,es) =>
    docker_services = {}
    docker_networks = {}
    @call header: "#{es_name}: ", ->
      @file.yaml
        header: 'elasticsearch config file'
        target: "/etc/elasticsearch/#{es_name}/conf/elasticsearch.yml"
        content:es.config
        backup: true


      @file.render
        header: 'elasticsearch java policy'
        target: "/etc/elasticsearch/#{es_name}/conf/java.policy"
        source: "#{__dirname}/resources/java.policy.j2"
        local: true
        context: {es: logs_path: "#{es.logs_path}/#{es_name}"}
        backup: true

      @file
        header: 'elasticsearch logging'
        target: "/etc/elasticsearch/#{es_name}/conf/log4j2.properties"
        source: "#{__dirname}/resources/log4j2.properties"
        local: true
        backup: true

      @system.mkdir directory:"#{path}/#{es_name}" ,uid: options.user.name, gid: options.user.name for path in es.data_path
      @system.mkdir directory:"#{es.plugins_path}",uid: options.user.name, gid: options.user.name
      @system.mkdir directory:"#{es.plugins_path}/#{es.es_version}",uid: options.user.name, gid: options.user.name
      @system.mkdir directory:"#{es.logs_path}/#{es_name}", uid: options.user.name, gid: options.user.name
      @system.mkdir directory:"#{es.logs_path}/#{es_name}/logstash",uid: options.user.name, gid: options.user.name
      @system.mkdir directory:"/etc/elasticsearch/#{es_name}/scripts",uid: options.user.name, gid: options.user.name
      @system.mkdir directory:"/etc/elasticsearch/keytabs",uid: options.user.name, gid: options.user.name

      @each es.downloaded_urls, (opts,callback) ->
        extract_target  = if opts.value.indexOf("github") != -1  then "#{es.plugins_path}/#{es.es_version}/" else "#{es.plugins_path}/#{es.es_version}/#{opts.key}"
        @call header: "Plugin #{opts.key} installation...", ->
          @file.download
            cache_file: "./#{opts.key}.zip"
            source: opts.value
            target: "#{es.plugins_path}/#{es.es_version}/#{opts.key}.zip"
            uid: options.user.name
            gid: options.user.name
            shy: true
          @tools.extract
            format: "zip"
            source: "#{es.plugins_path}/#{es.es_version}/#{opts.key}.zip"
            target: extract_target
            shy: true
          es.volumes.push "#{es.plugins_path}/#{es.es_version}/#{options.key}/elasticsearch:/usr/share/elasticsearch/plugins/#{options.key}"
          @system.remove "#{es.plugins_path}/#{es.es_version}/#{options.key}.zip", shy: true
        @next callback

IPTables

      matching_hosts = []
      matching_hosts.push options.hostname for type,es_node of es.nodes when minimatch(options.hostname, es_node.filter)

      @tools.iptables
        header: 'IPTables'
        if: (es.network.mode is 'host') and options.hostname in matching_hosts
        rules: [
          { chain: 'INPUT', jump: 'ACCEPT', dport: es.network.http_port, protocol: 'tcp', state: 'NEW', comment: "ES Docker #{es_name}" }
          { chain: 'INPUT', jump: 'ACCEPT', dport: es.network.tcp_port, protocol: 'tcp', state: 'NEW', comment: "ES Docker #{es_name}" }
        ]

Generate compose file

      #if options.fqdn is options.hosts[options.hosts.length-1]
      if options.fqdn is "#{if es.docker_compose_host then es.docker_compose_host else options.hosts[options.hosts.length-1]}"
        #TODO create overlay network if the network does not exist
        docker_networks["#{es.network.name}"] = external: es.network.external
        master_node = if es.master_nodes > 0
          "#{es.normalized_name}_master"
        else if es.master_data_nodes > 0
          "#{es.normalized_name}_master_data"
        es.volumes = [
          "/etc/elasticsearch/#{es_name}/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml",
          "/etc/elasticsearch/#{es_name}/conf/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties",
          "/etc/elasticsearch/#{es_name}/scripts:/usr/share/elasticsearch/config/scripts",
          "#{es.logs_path}/#{es_name}:#{es.config['path.logs']}",
          "/etc/elasticsearch/#{es_name}/conf/java.policy:/usr/share/elasticsearch/config/java.policy"

        ].concat es.volumes
        es.volumes.push "#{path}/#{es_name}/:#{path}" for path in es.data_path
        # es.volumes.push "#{es.plugins_path}/#{es.es_version}/#{plugin}:/usr/share/elasticsearch/plugins/#{plugin}" for plugin in es.plugins
        for type,es_node of es.nodes
          command = switch type
            when "master" then "elasticsearch -Enode.master=true -Enode.data=false"
            when "master_data" then "elasticsearch -Enode.master=true -Enode.data=true"
            when "data" then "elasticsearch -Enode.master=false -Enode.data=true"
          es.environment.push "constraint:node==#{es_node.filter}" if es_node.filter != ""
          docker_services[type] = {'environment' : [].concat.apply([],[es.environment,"ES_JAVA_OPTS=-Xms#{es_node.heap_size} -Xmx#{es_node.heap_size} -Djava.security.policy=/usr/share/elasticsearch/config/java.policy","bootstrap.memory_lock=true"]) }

          service_def =
            image : es.docker_es_image
            restart: "always"
            command: command
            # networks: [es.network.name]
            user: "elasticsearch"
            volumes: es.volumes
            ports: es.ports
            mem_limit: if es_node.mem_limit? then es_node.mem_limit else es.default_mem
            ulimits:  es.ulimits
            cap_add:  es.cap_add
          
          if es.network.mode is 'host'
            service_def["network_mode"] = 'host'
          else 
            service_def["networks"] = [es.network.name]
          if es_node.cpuset?
            service_def["cpuset"] = es_node.cpuset
          else
            service_def["cpu_quota"] = if es_node.cpu_quota? then es_node.cpu_quota * 1000 else es.default_cpu_quota
          misc.merge docker_services[type], service_def
        if es.kibana?
          docker_services["#{es_name}_kibana"] =
            image: es.docker_kibana_image
            container_name: "#{es_name}_kibana"
            environment: ["ELASTICSEARCH_URL=http://#{master_node}_1:9200"]
            ports: ["#{es.kibana.port}:5601"]
            networks: [es.network.name]
            
        docker_compose_content = {}
        docker_compose_content.version ?= "2"
        docker_compose_content.services ?= docker_services
        docker_compose_content.networks ?= docker_networks if docker_networks[0] is not undefined
        
        @file.yaml
          header: 'docker-compose'
          target: "/etc/elasticsearch/#{es_name}/docker-compose.yml"
          content: docker_compose_content
          backup: true

Run docker compose file

        docker_args =
            host: if options.swarm_manager then options.swarm_manager else ''
            tlsverify:" "
            tlscacert: options.ssl.dest_cacert
            tlscert: options.ssl.dest_cert
            tlskey:options.ssl.dest_key
        export_vars = 
          "export DOCKER_HOST=#{options.swarm_manager};export DOCKER_CERT_PATH=#{options.ssl.dest_dir};export DOCKER_TLS_VERIFY=1"

        for service, node of es.nodes then do (service, node) =>
          @system.execute
            cmd:"""
            #{export_vars}
            pushd /etc/elasticsearch/#{es_name}
            docker-compose --verbose scale #{service}=#{node.number}
            """

        @system.execute
          cmd:"""
          #{export_vars}
          pushd /etc/elasticsearch/#{es_name}
          docker-compose --verbose up -d #{es_name}_kibana
          """
          if: -> es.kibana is true

Dependencies

misc = require '@nikitajs/core/lib/misc'
minimatch = require 'minimatch'