Ranger HDFS Plugin Install

module.exports = header: 'Ranger HDFS Plugin', handler: ({options}) ->
  {hdfs_conf_dir} = options

Wait

  @call 'ryba/ranger/admin/wait', once: true, options.wait_ranger_admin

HDFS Dependencies

  # @call 'ryba/hadoop/hdfs_client/install' #migation solved it with implicy hdfs_client requirement
  @registry.register 'hconfigure', 'ryba/lib/hconfigure'
  @registry.register 'hdfs_mkdir', 'ryba/lib/hdfs_mkdir'
  @registry.register 'ranger_service', 'ryba/ranger/actions/ranger_service'
  @registry.register 'ranger_policy', 'ryba/ranger/actions/ranger_policy'
  @registry.register 'ranger_user', 'ryba/ranger/actions/ranger_user'

HDP version

  version = null
  @system.execute
    header: 'HDP Version'
    shy: true
    cmd: """
    hdp-select versions | tail -1
    """
   , (err, {status, stdout}) ->
      throw err if err
      version = stdout.trim()

Package

  @service
    header: 'Package'
    name: "ranger-hdfs-plugin"

Layout

The value present in "XAAUDIT.HDFS.DESTINATION_DIRECTORY" contains variables such as "%app-type% and %time:yyyyMMdd%".

migration: wdavidw 170918, NameNodes are not yet started.

  @call
    if: options.install['XAAUDIT.HDFS.IS_ENABLED'] is 'true'
    header: 'Audit HDFS Policy'
  , ->
    @ranger_policy
      header: 'HDFS Audit'
      username: options.ranger_admin.username
      password: options.ranger_admin.password
      url: options.install['POLICY_MGR_URL']
      policy: options.policy_hdfs_audit
    @system.mkdir
      header: 'HDFS Spool Dir'
      if: options.install['XAAUDIT.HDFS.IS_ENABLED'] is 'true'
      target: options.install['XAAUDIT.HDFS.FILE_SPOOL_DIR']
      uid: options.hdfs_user.name
      gid: options.hadoop_group.name
      mode: 0o0750
  @system.mkdir
    header: 'SOLR Spool Dir'
    if: options.install['XAAUDIT.SOLR.IS_ENABLED'] is 'true'
    target: options.install['XAAUDIT.SOLR.FILE_SPOOL_DIR']
    uid: options.hdfs_user.name
    gid: options.hadoop_group.name
    mode: 0o0750

Ranger User

  @ranger_user
    if: options.master_fqdn is options.fqdn
    header: 'Ranger User'
    username: options.ranger_admin.username
    password: options.ranger_admin.password
    url: options.install['POLICY_MGR_URL']
    user: options.plugin_user

Service Repository

Based on step 1 in hdfs plugin configuration. Instead of using the web ui we execute this task using the rest api.

  @ranger_service
    username: options.ranger_admin.username
    password: options.ranger_admin.password
    url: options.install['POLICY_MGR_URL']
    service: options.service_repo

Note, by default, we're are using the same Ranger principal for every plugin and the principal is created by the Ranger Admin service. Chances are that a customer user will need specific ACLs but this hasn't been tested.

  # See [#96](https://github.com/ryba-io/ryba/issues/95): Ranger HDFS: should we use a dedicated principal
  @krb5.addprinc options.krb5.admin,
    header: 'Plugin Principal'
    principal: "#{options.service_repo.configs.username}"
    password: options.service_repo.configs.password

Properties

  @call -> @file
    header: 'Properties'
    if: -> version?
    source: "#{__dirname}/../../resources/plugin-install.properties"
    target: "/usr/hdp/#{version}/ranger-hdfs-plugin/install.properties"
    local: true
    eof: true
    backup: true
    write: for k, v of options.install
      match: RegExp "^#{quote k}=.*$", 'mg'
      replace: "#{k}=#{v}"
      append: true

Plugin Scripts

From HDP 2.5 (Ranger 0.6) hdfs plugin need a Client JAAS configuration file to talk with kerberized component. The JAAS configuration can be donne with a jaas file and the Namenonde Env property auth.to.login.conf or can be set by properties in ranger-hdfs-audit.xml file. Not documented be taken from github-source

  @call (_, callback) ->
    files = ['ranger-hdfs-audit.xml','ranger-hdfs-security.xml','ranger-policymgr-ssl.xml', 'hdfs-site.xml']
    sources_props = {}
    current_props = {}
    files_exists = {}
    @system.execute
      cmd: """
      echo '' | keytool -list \
        -storetype jceks \
        -keystore /etc/ranger/#{options.install['REPOSITORY_NAME']}/cred.jceks | egrep '.*ssltruststore|auditdbcred|sslkeystore'
      """
      code_skipped: 1
    @call
      if: -> @status -1 #do not need this if the cred.jceks file is not provisioned
    , ->
      @each files, ({options}, cb) ->
        file = options.key
        target = "#{hdfs_conf_dir}/#{file}"
        ssh = @ssh options.ssh
        fs.exists ssh, target, (err, exists) ->
          return cb err if err
          return cb() unless exists
          files_exists["#{file}"] = exists
          properties.read ssh, target , (err, props) ->
            return cb err if err
            sources_props["#{file}"] = props
            cb()
    @system.link
      source: hdfs_conf_dir
      target: '/usr/hdp/current/hadoop-hdfs-namenode/etc/hadoop'
    @file
      header: 'Fix'
      target: "/usr/hdp/#{version}/ranger-hdfs-plugin/enable-hdfs-plugin.sh"
      write: [
          match: RegExp "^HCOMPONENT_CONF_DIR=.*$", 'mg'
          replace: "HCOMPONENT_CONF_DIR=#{hdfs_conf_dir}"
        ,
          match: RegExp "^HCOMPONENT_INSTALL_DIR_NAME=.*$", 'mg'
          replace: "HCOMPONENT_INSTALL_DIR_NAME=/usr/hdp/current/hadoop-hdfs-namenode"
        ,
          match: RegExp "^HCOMPONENT_LIB_DIR=.*$", 'mg'
          replace: "HCOMPONENT_LIB_DIR=/usr/hdp/current/hadoop-hdfs-namenode/lib"
        ,
          match: RegExp "^HCOMPONENT_ARCHIVE_CONF_DIR==.*$", 'mg'
          replace: "HCOMPONENT_LIB_DIR=#{hdfs_conf_dir}/.archive"
        ,
          match: RegExp "^HCOMPONENT_INSTALL_DIR==.*$", 'mg'
          replace: "HCOMPONENT_INSTALL_DIR=/usr/hdp/current/hadoop-hdfs-namenode"
          
      ]
      backup: true
      mode: 0o750
    @system.execute
      header: 'Execution'
      shy: true
      cmd: """
      export HADOOP_LIBEXEC_DIR=/usr/hdp/current/hadoop-client/libexec
      /usr/hdp/#{version}/ranger-hdfs-plugin/enable-hdfs-plugin.sh
      """
    @hconfigure
      header: 'Fix Conf'
      target: "#{hdfs_conf_dir}/ranger-hdfs-security.xml"
      merge: true
      properties:
        'ranger.plugin.hdfs.policy.rest.ssl.config.file': "#{hdfs_conf_dir}/ranger-policymgr-ssl.xml"
    @hconfigure
      header: 'Solr JAAS'
      target: "#{hdfs_conf_dir}/ranger-hdfs-audit.xml"
      merge: true
      properties: options.audit
    @each files, ({options}, cb) ->
      file = options.key
      target = "#{hdfs_conf_dir}/#{file}"
      ssh = @ssh options.ssh
      fs.exists ssh, target, (err, exists) ->
        return callback err if err
        properties.read ssh, target , (err, props) ->
          return cb err if err
          current_props["#{file}"] = props
          cb()
    @call
      header: 'Diff'
      shy: true
    , ->
      for file in files
        #do not need to go further if the file did not exist
        return callback null, true unless sources_props["#{file}"]?
        for prop, value of current_props["#{file}"]
          return callback null, true unless value is sources_props["#{file}"][prop]
        for prop, value of sources_props["#{file}"]
          return callback null, true unless value is current_props["#{file}"][prop]
        return callback null, false

Dependencies

quote = require 'regexp-quote'
properties = require '../../../lib/properties'
fs = require 'ssh2-fs'