Spark Livy Server Install

Install dockerized Apache Spark Livy Server 0.2 container. The container can be build by ./bin/prepare script or directly downloaded (from local computer only for now, no images available on dockerhub).

Run ryba prepare to create the Docker container.

module.exports = header: 'Spark Livy Server', handler: ->
  {spark} = @config.ryba
  {hadoop_group, hdfs, hive, hbase, hadoop_conf_dir, realm, ssl} = @config.ryba
  krb5 = @config.krb5_client.admin[realm]
  tmp_location = "/var/tmp/ryba/ssl"

Wait

  @call once: true, 'ryba/spark/history_server/wait'
  @call once: true, 'ryba/spark/thrift_server/wait'

IPTables

IPTables rules are only inserted if the parameter "iptables.action" is set to "start" (default value).

  @tools.iptables
    header: 'IPTables'
    rules: [
      { chain: 'INPUT', jump: 'ACCEPT', dport: spark.livy.conf['livy.server.port'], protocol: 'tcp', state: 'NEW', comment: "Spark Livy Server" }
    ]
    if: @config.iptables.action is 'start'

Identities

By default, the "spark" package create the following entries:

cat /etc/passwd | grep spark
spark:x:495:494:Spark:/var/lib/spark:/bin/bash
cat /etc/group | grep spark
spark:x:494:

  @system.group header: 'Group', spark.group
  @system.user header: 'User', spark.user

Startup Script

Write startup script to /etc/init.d/service-hue-docker

  @call header: 'Startup Script', handler: (options) ->
    @service.init
      source: "#{__dirname}/../resources/#{spark.livy.service}"
      local: true
      target: "/etc/init.d/#{spark.livy.service}"
      context: spark.livy
      mode: 0o755
    @system.tmpfs
      if_os: name: ['redhat','centos'], version: '7'
      mount: spark.livy.pid_dir
      uid: spark.user.name
      gid: spark.group.name
      perm: '0750'

Layout

  @call header: 'Layout', handler: ->
    @system.mkdir
      target: spark.livy.pid_dir
      uid: spark.user.name
      gid: spark.group.name
      mode: 0o0770
    @system.mkdir
      target: spark.livy.log_dir
      uid: spark.user.name
      gid: spark.group.name
      mode: 0o0770
    @system.mkdir
      target: spark.livy.conf_dir
      uid: spark.user.name
      gid: spark.group.name

Spark Configuration

  @file.properties
    header: 'Livy Server Configuration'
    target: "#{spark.livy.conf_dir}/livy.conf"
    content: spark.livy.conf
    uid: spark.user.name
    gid: spark.group.name
    backup: true
    eof: true
  @service.restart
    name: 'spark-livy-server'
    if: -> @status -1

Download Container

  @call header: 'Download Container', handler: ->
    tmp = spark.livy.image_dir
    md5 = spark.livy.md5 ?= true
    @file.download
      source: "#{spark.livy.build.directory}/#{spark.livy.build.tar}"
      target: "#{tmp}/#{spark.livy.build.tar}"
      binary: true
      md5: md5
    @docker_load
      input: "#{tmp}/#{spark.livy.build.tar}"

kerberos

  @krb5.addprinc krb5,
    if: spark.livy.conf['livy.server.auth.kerberos.principal']
    header: 'Livy Server principal'
    principal: spark.livy.conf['livy.server.auth.kerberos.principal']
    keytab: spark.livy.conf['livy.server.auth.kerberos.keytab']
    randkey: true
    uid: spark.user.name
    gid: spark.group.name
    mode: 0o0600

SSL

  @call if: spark.livy.ssl_enabled , handler: ->
    @file.download
      source: ssl.cacert
      target: "#{tmp_location}/#{path.basename ssl.cacert}"
      mode: 0o0600
      shy: true
    @file.download
      source: ssl.cert
      target: "#{tmp_location}/#{path.basename ssl.cert}"
      mode: 0o0600
      shy: true
    @file.download
      source: ssl.key
      target: "#{tmp_location}/#{path.basename ssl.key}"
      mode: 0o0600
      shy: true
    @java.keystore_add
      keystore: spark.livy.keystore
      storepass: spark.livy.keystorePassword
      caname: "hadoop_root_ca"
      cacert: "#{tmp_location}/#{path.basename ssl.cacert}"
      key: "#{tmp_location}/#{path.basename ssl.key}"
      cert: "#{tmp_location}/#{path.basename ssl.cert}"
      keypass: spark.livy.keystorePassword
      name: @config.shortname
    @system.remove
      target: "#{tmp_location}/#{path.basename ssl.cacert}"
      shy: true
    @system.remove
      target: "#{tmp_location}/#{path.basename ssl.cert}"
      shy: true
    @system.remove
      target: "#{tmp_location}/#{path.basename ssl.key}"
      shy: true

Docker run

  @docker_service
    header: 'Livy Spark Server Run'
    # force: -> @status -1
    image: "#{spark.livy.image}:#{spark.livy.build.version}"
    volume: [
      "#{hadoop_conf_dir}:#{hadoop_conf_dir}"
      "#{spark.conf_dir}:#{spark.conf_dir}"
      "#{spark.livy.conf_dir}:#{spark.livy.conf_dir}"
      "#{spark.livy.log_dir}:#{spark.livy.log_dir}"
      "#{spark.livy.pid_dir}:#{spark.livy.pid_dir}"
      '/etc/krb5.conf:/etc/krb5.conf'
      '/etc/security/keytabs:/etc/security/keytabs'
      '/usr/hdp/current/spark-client:/usr/hdp/current/spark-client'
    ]
    env: [
      "HADOOP_CONF_DIR=#{spark.livy.env['HADOOP_CONF_DIR']}"
      "SPARK_HOME=#{spark.livy.env['SPARK_HOME']}"
      "SPARK_CONF_DIR=#{spark.livy.env['SPARK_CONF_DIR']}"
      "LIVY_IDENT_STRING=#{spark.livy.env['LIVY_IDENT_STRING']}"
      "LIVY_LOG_DIR=#{spark.livy.env['LIVY_LOG_DIR']}"
      "LIVY_PID_DIR=#{spark.livy.env['LIVY_PID_DIR']}"
      "KRB5CCNAME=FILE:/tmp/krb5cc_#{spark.user.uid}"
    ]
    net: 'host'
    service: true
    name: spark.livy.container

path = require 'path'