Apache Atlas Install

module.exports = header: 'Atlas Install', handler: (options) ->
  protocol = if options.application.properties['atlas.enableTLS'] is 'true' then 'https' else 'http'
  credential_file = options.application.properties['cert.stores.credential.provider.path'].split('jceks://file')[1]
  credential_name = path.basename credential_file
  credential_dir = path.dirname credential_file

Registry

  @registry.register 'hdp_select', 'ryba/lib/hdp_select'
  @registry.register ['file', 'jaas'], 'ryba/lib/file_jaas'
  @registry.register 'ranger_service', 'ryba/ranger/actions/ranger_service'
  @registry.register 'ranger_policy', 'ryba/ranger/actions/ranger_policy'
  @registry.register 'ranger_service_wait', 'ryba/ranger/actions/ranger_service_wait'
  @registry.register 'ranger_user', 'ryba/ranger/actions/ranger_user'

Wait

  # @call 'masson/core/krb5_client/wait', once: true, options.wait_krb5_client
  # @call 'ryba/zookeeper/server/wait', once: true, options.wait_zookeeper_server
  # @call 'ryba/hbase/master/wait', once: true, options.wait_hbase
  # @call 'ryba/kafka/broker/wait', once: true, options.wait_kafka
  # @call 'ryba/ranger/admin/wait', once: true, options.wait_ranger

Identities

  @system.group header: 'Group',  options.group
  @system.user header: 'User', options.user

IPTables

IPTables rules are only inserted if the parameter "iptables.action" is set to "start" (default value).

  @tools.iptables
    if: options.iptables
    rules: [
      { chain: 'INPUT', jump: 'ACCEPT', dport: options.application.properties["atlas.server.#{protocol}.port"], protocol: 'tcp', state: 'NEW', comment: "Atlas Server #{protocol}" }
    ]

Package & Repository

Install Atlas packages

  @service
    header: 'Atlas Package'
    name: 'atlas-metadata'
  @hdp_select
    name: 'atlas-server'
  @hdp_select
    name: 'atlas-client'
  @service.init
    header: 'Init Script'
    target: '/etc/init.d/atlas-metadata-server'
    source: "#{__dirname}/resources/atlas-metadata-server.j2"
    local: true
    mode: 0o0755
    context: options

Layout && Directories

  @call header: 'Layout Directories', ->
    @system.mkdir
      target: options.log_dir
      uid: options.user.name
      gid: options.group.name
      mode: 0o0750
    @system.mkdir
      target: options.pid_dir
      uid: options.user.name
      gid: options.group.name
      mode: 0o0750
    @system.mkdir
      target: options.conf_dir
      uid: options.user.name
      gid: options.group.name
      mode: 0o0750
    @system.mkdir
      target: options.env['ATLAS_DATA_DIR']
      uid: options.user.name
      gid: options.group.name
      mode: 0o0750
    @system.mkdir
      target: options.env['ATLAS_EXPANDED_WEBAPP_DIR']
      uid: options.user.name
      gid: options.group.name
      mode: 0o0750
    @system.link
      target: options.conf_dir
      source: '/usr/hdp/current/atlas-server/conf'
    @system.tmpfs
      if_os: name: ['redhat','centos'], version: '7'
      mount: options.pid_dir
      uid: options.user.name
      gid: options.group.name
      perm: '0750'

SSL

Import certificates, private and public keys of the host.

  @java.keystore_add
    keystore: options.application.properties['keystore.file']
    storepass: options.ssl.keystore.password
    key: options.ssl.key.source
    cert: options.ssl.cert.source
    keypass: options.ssl.keystore.keypass
    name: options.ssl.key.name
    local: options.ssl.cert.local
    uid: options.user.name
    gid: options.group.name
    mode: 0o0640
  @java.keystore_add
    keystore: options.application.properties['keystore.file']
    storepass: options.ssl.keystore.password
    caname: "hadoop_root_ca"
    cacert: options.ssl.cacert.source
    local: options.ssl.cacert.local
  @java.keystore_add
    keystore: options.application.properties['truststore.file']
    storepass: options.ssl.truststore.password
    caname: "hadoop_root_ca"
    cacert: options.ssl.cacert.source
    local: options.ssl.cacert.local
    uid: options.user.name
    gid: options.group.name
    mode: 0o0644
  @call
    if: -> @status(-3) or @status(-2)
    header: 'Generate Credentials SSL provider file'
  , (_, callback) ->
    ssh = @ssh options.ssh
    ssh.shell (err, stream) =>
      stream.write 'if /usr/hdp/current/atlas-client/bin/cputil.py ;then exit 0; else exit 1;fi\n'
      data = ''
      error = exit = null
      stream.on 'data', (data, extended) =>
        data = data.toString()
        switch
          when /Please enter the full path to the credential provider:/.test data
            @log "prompt: #{data}"
            @log "writing: #{options.application.properties['cert.stores.credential.provider.path'].split('jceks://file')[1]}\n"
            stream.write "#{options.application.properties['cert.stores.credential.provider.path'].split('jceks://file')[1]}\n"
            data = ''
          when /Please enter the password value for keystore.password:/.test data
            @log "prompt: #{data}"
            @log "write: #{options.ssl.keystore.password}"
            stream.write "#{options.ssl.keystore.password}\n"
            data = ''
          when /Please enter the password value for keystore.password again:/.test data
            @log "prompt: #{data}"
            @log "write: #{options.ssl.keystore.password}"
            stream.write "#{options.ssl.keystore.password}\n"
            data = ''
          when /Please enter the password value for truststore.password:/.test data
            @log "prompt: #{data}"
            @log "write: #{options.ssl.truststore.password}"
            stream.write "#{options.ssl.truststore.password}\n"
            data = ''
          when /Please enter the password value for truststore.password again:/.test data
            @log "prompt: #{data}"
            @log "write: #{options.ssl.truststore.password}"
            stream.write "#{options.ssl.truststore.password}\n"
            data = ''
          when /Please enter the password value for password:/.test data
            @log "prompt: #{data}"
            @log "write: #{options.ssl.keystore.keypass}"
            stream.write "#{options.ssl.keystore.keypass}\n"
            data = ''
          when /Please enter the password value for password again:/.test data
            @log "prompt: #{data}"
            @log "write: #{options.ssl.keystore.keypass}"
            stream.write "#{options.ssl.keystore.keypass}\n"
            data = ''
          when /Entry for keystore.password already exists/.test data
            stream.write "y\n"
            data = ''
          when /Entry for truststore.password already exists/.test data
            stream.write "y\n"
            data = ''
          when /Entry for password already exists/.test data
            stream.write "y\n"
            data = ''
          when /Exception in thread.*/.test data
            error = new Error data
            stream.end 'exit\n' unless exit
            exit = true
      stream.on 'exit', =>
        return callback error if error
        callback null, true
  @system.chown
    header: 'Ownership credential'
    target: "#{credential_dir}/#{credential_name}"
    uid: options.user.name
    gid: options.group.name
    mode: 0o770
  @system.chown
    header: 'Ownership crc'
    target: "#{credential_dir}/.#{credential_name}.crc"
    uid: options.user.name
    gid: options.group.name
    mode: 0o770

Kerberos

Add The Kerberos Principal for atlas service and setup a JAAS configuration file for atlas to able to open client connection to solr for its indexing backend.

  @krb5.addprinc options.krb5.admin,
    header: 'Kerberos Atlas Service'
    randkey: true
    principal: options.application.properties['atlas.authentication.principal'].replace '_HOST', options.fqdn
    keytab: options.application.properties['atlas.authentication.keytab']
    uid: options.user.name
    gid: options.group.name
    mode: 0o660
  @krb5.addprinc options.krb5.admin,
    header: 'Kerberos Atlas Service'
    principal: options.application.properties['atlas.http.authentication.kerberos.principal'].replace '_HOST', options.fqdn
    randkey: true
    keytab: options.application.properties['atlas.http.authentication.kerberos.keytab']
    uid: 'root'
    gid: options.hadoop_group.name
    mode: 0o660
    unless: -> @status -1
  @file.jaas
    if: options.atlas_opts['java.security.auth.login.config']?
    header: 'Atlas Service JAAS'
    target: options.atlas_opts['java.security.auth.login.config']
    mode: 0o750
    uid: options.user.name
    gid: options.group.name
    content:
      KafkaClient:
        principal: options.application.properties['atlas.authentication.principal']
        keyTab: options.application.properties['atlas.authentication.keytab']
        useKeyTab: true
        storeKey: true
        serviceName: 'kafka'
        useTicketCache: true
      Client:
        useKeyTab: true
        storeKey: true
        useTicketCache: false
        doNotPrompt: false
        keyTab: options.application.properties['atlas.authentication.keytab']
        principal: options.application.properties['atlas.authentication.principal'].replace '_HOST', options.fqdn
  @krb5.addprinc options.krb5.admin,
    header: 'Kerberos Atlas Service Admin Users'
    principal: options.admin_principal
    randkey: true
    password: options.admin_password

Application Properties

Writes atlas-application.properties file.

  @file.properties
    header: 'Atlas Application Properties'
    target: "#{options.conf_dir}/atlas-application.properties"
    content: options.application.properties
    backup: true
    uid: options.user.name
    gid: options.group.name
    merge: false
    mode: 0o770

Log4 Properties

  @file.download
    header: 'Atlas Log4j Properties'
    target: "#{options.conf_dir}/atlas-log4j.xml"
    source: "#{__dirname}/resources/atlas-log4j.xml"
    local: true
    uid: options.user.name
    gid: options.group.name
    mode: 0o770

Environment

Render the Atlas Environment file

  @call ->
    options.env['METADATA_OPTS'] ?= ''
    options.env['METADATA_OPTS'] += " -D#{k}=#{v} "  for k, v of options.metadata_opts
    options.env['ATLAS_OPTS'] ?= ''
    options.env['ATLAS_OPTS'] += " -D#{k}=#{v} "  for k, v of options.atlas_opts
    writes = for k,v of options.env
      match: RegExp "^.*#{k}=.*$", 'mg'
      replace: "export #{k}=\"#{v}\" # RYBA DON'T OVERWRITE"
      append: true
    @file.render
      header: 'Atlas Env'
      target: "#{options.conf_dir}/atlas-env.sh"
      source: "#{__dirname}/resources/atlas-env.sh.j2"
      backup: true
      uid: options.user.name
      gid: options.group.name
      mode: 0o770
      local: true
      context: options
      write: writes
      unlink: true
      eof: true

Deploy Atlas War

Need to copy the atlas war file if env['ATLAS_EXPANDED_WEBAPP_DIR'] is set to other than the default

  @system.copy
    header: 'Atlas webapp war'
    source: '/usr/hdp/current/atlas-server/server/webapp/atlas.war'
    target: "#{options.env['ATLAS_EXPANDED_WEBAPP_DIR']}/atlas.war"

HBase Layout

  @system.copy
    header: 'HBase Client Site'
    source: "#{options.hbase_conf_dir}/hbase-site.xml"
    target: "#{options.conf_dir}/hbase/hbase-site.xml"
  @system.copy
    header: 'HBase Client Env'
    target: "#{options.conf_dir}/hbase/hbase-env.sh"
    source: "#{options.hbase_conf_dir}/hbase-env.sh"
    uid: options.user.name
    gid: options.group.name
    context: options
    local: false
    eof: true
    # Fix mapreduce looking for "mapreduce.tar.gz"
    write: [
      match: /^export HBASE_OPTS=\"(.*)\$\{HBASE_OPTS\} -Djava.security.auth.login.config(.*)$/m
      replace: "export HBASE_OPTS=\"${HBASE_OPTS} -Dhdp.version=$HDP_VERSION -Djava.security.auth.login.config=#{options.conf_dir}/atlas-server.jaas\" # HDP VERSION FIX RYBA, HBASE CLIENT ONLY"
      append: true
    ]
  @system.copy
    header: 'HBase Client HDFS site'
    source: "/etc/hadoop/conf/hdfs-site.xml"
    target: "#{options.conf_dir}/hbase/hdfs-site.xml"
  @system.execute
    header: 'Create HBase Namespace'
    cmd: mkcmd.hbase options.hbase_admin, """
    hbase shell 2>/dev/null <<-CMD
      create_namespace 'atlas'
    CMD
    """
    if_exec: mkcmd.hbase options.hbase_admin, "hbase shell 2>/dev/null <<< \"list_namespace_tables 'atlas'\" | grep 'ERROR: Unknown namespace atlas!'"

HBase Permission

Grant Permission to atlas for its titan' tables through ranger or from hbase shell.

  @call
    if: -> options.ranger_hbase
    header: 'HBase Atlas Permissions'
  , ->
    @ranger_service_wait
      header: "HBase Plugin Wait"
      username: options.ranger_admin.options.admin.username
      password: options.ranger_admin.options.admin.password
      url: options.ranger_admin.options.install['policymgr_external_url']
      service: options.hbase_policy.service
    @ranger_user
      header: "Ranger admin atlas"
      username: options.ranger_admin.options.admin.username
      password: options.ranger_admin.options.admin.password
      url: options.ranger_admin.options.install['policymgr_external_url']
      user: options.ranger_user
    @ranger_policy
      header: 'HBase Plugin ACL'
      username: options.ranger_admin.options.admin.username
      password: options.ranger_admin.options.admin.password
      url: options.ranger_admin.options.install['policymgr_external_url']
      policy: options.hbase_policy
  @call
    unless: -> options.ranger_hbase
    header: 'HBase Atlas Permissions'
  , ->
    @system.execute
      header: 'Grant Permissions'
      unless_exec: mkcmd.hbase options.hbase_admin, "hbase shell 2>/dev/null <<< \"user_permission '@#{options.application.namespace}'\" |  egrep \"^\\s(#{options.user.name})\\s*(#{options.user.name}).*\\[Permission: actions=(READ|EXEC|WRITE|CREATE|ADMIN|,){9}\\]$\""
      cmd: mkcmd.hbase options.hbase_admin, """
      hbase shell 2>/dev/null <<-CMD
        grant '#{options.user.name}', 'RWCA', '@#{options.application.namespace}'
      CMD
      """
      trap: true

Setup Credentials File

Convert the user_creds object into a file of credentials. See how to generate atlas credential based on file.

  user_creds
    'toto':
      name: 'toto'
      password: 'toto123'
      group: 'user'
    'juju':
      name: 'julie'
      password: 'juju123'
      group: 'user'

  @call
    if: options.application.properties['atlas.authentication.method.file'] is 'true'
    header: 'Render Credentials file'
  , ->
    old_lines = []
    new_lines = []
    content = ''
    @call header: 'Read Current Credential', (_, callback )  ->
      ssh = @ssh options.ssh
      fs.readFile ssh, options.application.properties['atlas.authentication.method.file.filename'], 'utf8', (err, content) ->
        return callback null, true if err and err.code is 'ENOENT'
        return callback err if err
        old_lines = string.lines content
        return if old_lines.length > 0 then callback null, true else callback null, false
    @call
      header: 'Merge user credentials'
      if: -> @status -1
    , ->
      for line in old_lines
        name = line.split(':')[0]
        new_lines.push unless name in Object.keys(options.user_creds)#keep track of old user if not present in current config
    @call header: 'Generate credential file', ->
      @each options.user_creds, (opt, callback) ->
        name = opt.key
        user = opt.value
        line = "#{user.name}=#{user.group}"
        @system.execute
          header: 'Generate new credential'
          cmd: "echo -n '#{user.password}' | sha256sum"
        ,(err, status, stdout) ->
          throw err if err
          [match] = /[a-zA-Z0-9]*/.exec stdout.trim()
          new_lines.push "#{line}::#{match}"
        @next callback
      @call ->
        @file
          content: new_lines.join "/n"
          target: options.application.properties['atlas.authentication.method.file.filename']
          mode: 0o740
          eof: true
          backup: true
          uid: options.user.name
          gid: options.user.name

Kafka Layout

Create the kafka topics needed by Atlas, if the property atlas.notification.create.topics is false. Ryba create the topic base on the channel chosen for atlas. See configure options. kakfa client become an implicit dependance. Its properties can be used.

  @call
    header: "Kafka Topic Layout"
    retry: 3
    if: options.application.properties['atlas.notification.create.topics'] is 'false'
  , ->
    topics = options.application.properties['atlas.notification.topics'].split ','
    for topic in topics
      [ATLAS_HOOK_TOPIC,ATLAS_ENTITIES_TOPIC] = topics
      group_id = null
      switch topic
        when ATLAS_HOOK_TOPIC then group_id = options.application.properties['atlas.kafka.hook.group.id']
        when ATLAS_ENTITIES_TOPIC then group_id = options.application.properties['atlas.kafka.entities.group.id']
      @system.execute
        header: "Create #{topic} (Kerberos)"
        if: options.application.properties['atlas.kafka.security.protocol'] in ['SASL_SSL','SASL_PLAINTEXT']
        cmd: mkcmd.kafka options.kafka_admin, """
        /usr/hdp/current/kafka-broker/bin/kafka-topics.sh --create \
          --zookeeper #{options.application.properties['atlas.kafka.zookeeper.connect']} \
          --partitions #{options.kafka_partitions} --replication-factor #{options.kafka_replication} \
          --topic #{topic}
        """
        unless_exec: mkcmd.kafka options.kafka_admin, """
        /usr/hdp/current/kafka-broker/bin/kafka-topics.sh --list \
        --zookeeper #{options.application.properties['atlas.kafka.zookeeper.connect']} | grep #{topic}
        """
      @system.execute
        header: "Create #{topic} (Simple)"
        unless: options.application.properties['atlas.kafka.security.protocol'] in ['SASL_SSL','SASL_PLAINTEXT']
        cmd: """
        /usr/hdp/current/kafka-broker/bin/kafka-topics.sh --create \
          --zookeeper #{options.application.properties['atlas.kafka.zookeeper.connect']} \
          --partitions #{options.kafka_partitions} \
          --replication-factor #{options.kafka_replication} \
          --topic #{topic}
        """
        unless_exec: """
        /usr/hdp/current/kafka-broker/bin/kafka-topics.sh --list \
        --zookeeper #{options.application.properties['atlas.kafka.zookeeper.connect']} | grep #{topic}
        """

Add Ranger ACL

      @call header: 'KafKa Topic ACL (Ranger)', if: options.ranger_kafka_install?, ->
        @ranger_service_wait
          header: "Kafka Plugin Wait"
          username: options.ranger_admin.options.admin.username
          password: options.ranger_admin.options.admin.password
          url: options.ranger_admin.options.install['policymgr_external_url']
          service: options.kafka_policy.service
        @ranger_user
          header: "Ranger admin"
          username: options.ranger_admin.options.admin.username
          password: options.ranger_admin.options.admin.password
          url: options.ranger_admin.options.install['policymgr_external_url']
          user: options.ranger_user
        @ranger_policy
          header: 'Kafka Plugin ACL'
          username: options.ranger_admin.options.admin.username
          password: options.ranger_admin.options.admin.password
          url: options.ranger_admin.options.install['policymgr_external_url']
          policy: options.kafka_policy
        @wait
          time: 10000
          if: -> @status -1

Add Simple ACL

Need to put ACL, even when Ranger is not configured. Atlas and Hive users needs Authorization to topics. The commands a divided per user, as the hive bridge is not mandatory.

      @system.execute
        header: 'KafKa Topic ACL Atlas User (Simple)'
        cmd: mkcmd.kafka options.kafka_admin, """
          /usr/hdp/current/kafka-broker/bin/kafka-acls.sh \
          --authorizer-properties zookeeper.connect=#{options.application.properties['atlas.kafka.zookeeper.connect']} \
          --add --allow-principal User:#{options.user.name}  --group #{group_id} \
          --operation All --topic #{topic}
        """
        unless_exec: mkcmd.kafka options.kafka_admin, """
          /usr/hdp/current/kafka-broker/bin/kafka-acls.sh  --list \
          --authorizer-properties \
          zookeeper.connect=#{options.application.properties['atlas.kafka.zookeeper.connect']}  \
          --topic #{topic} | grep 'User:#{options.user.name} has Allow permission for operations: Write from hosts: *'
        """

Add Ranger Solr ACL

  # @call
  #   header: 'Titan Solr ACL (Ranger)'
  #   if: options.ranger_solr_install
  # , ->
  #   @ranger_service_wait
  #     header: "Solr Plugin Wait"
  #     username: options.admin.username
  #     password: options.admin.password
  #     url: options.ranger_admin.options.install['policymgr_external_url']
  #     service: options.solr_policy.service
  #   @ranger_user
  #     header: "Ranger admin"
  #     username: options.admin.username
  #     password: options.admin.password
  #     url: options.ranger_admin.options.install['policymgr_external_url']
  #     user: options.ranger_user
  #   @ranger_policy
  #     header: 'Solr Plugin ACL'
  #     username: options.ranger_admin.options.admin.username
  #     password: options.ranger_admin.options.admin.password
  #     url: options.ranger_admin.options.install['policymgr_external_url']
  #     policy: options.solr_policy
  #   @wait
  #     time: 10000
  #     if: -> @status -1

Dependencies

mkcmd = require '../lib/mkcmd'
string = require '@nikitajs/core/lib/misc/string'
path = require 'path'
fs = require 'ssh2-fs'
{merge} = require '@nikitajs/core/lib/misc'