From: Wayne Schneider <wayne@indexdata.com>
Date: Mon, 1 Aug 2016 17:44:39 +0000 (-0500)
Subject: Zookeeper install and config.
X-Git-Url: http://sru.miketaylor.org.uk/cgi-bin?a=commitdiff_plain;h=2a9dbcffd0a9062496051b4f1cdb94b0a8c057b1;p=lui-solr.git

Zookeeper install and config.
---

diff --git a/conf/zookeeper/log4j.properties b/conf/zookeeper/log4j.properties
new file mode 100644
index 0000000..f5ec35d
--- /dev/null
+++ b/conf/zookeeper/log4j.properties
@@ -0,0 +1,58 @@
+# Define some default values that can be overridden by system properties
+zookeeper.root.logger=INFO, CONSOLE
+zookeeper.console.threshold=INFO
+zookeeper.log.dir=/var/log/masterkey/lui
+zookeeper.log.file=zookeeper.log
+zookeeper.log.threshold=DEBUG
+zookeeper.tracelog.dir=/var/log/masterkey/lui
+zookeeper.tracelog.file=zookeeper_trace.log
+
+#
+# ZooKeeper Logging Configuration
+#
+
+# Format is "<default threshold> (, <appender>)+
+
+# DEFAULT: console appender only
+log4j.rootLogger=${zookeeper.root.logger}
+
+# Example with rolling log file
+#log4j.rootLogger=DEBUG, CONSOLE, ROLLINGFILE
+
+# Example with rolling log file and tracing
+#log4j.rootLogger=TRACE, CONSOLE, ROLLINGFILE, TRACEFILE
+
+#
+# Log INFO level and above messages to the console
+#
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.Threshold=${zookeeper.console.threshold}
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
+
+#
+# Add ROLLINGFILE to rootLogger to get log file output
+#    Log DEBUG level and above messages to a log file
+log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
+log4j.appender.ROLLINGFILE.Threshold=${zookeeper.log.threshold}
+log4j.appender.ROLLINGFILE.File=${zookeeper.log.dir}/${zookeeper.log.file}
+
+# Max log file size of 10MB
+log4j.appender.ROLLINGFILE.MaxFileSize=10MB
+# uncomment the next line to limit number of backup files
+#log4j.appender.ROLLINGFILE.MaxBackupIndex=10
+
+log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
+log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
+
+
+#
+# Add TRACEFILE to rootLogger to get log file output
+#    Log DEBUG level and above messages to a log file
+log4j.appender.TRACEFILE=org.apache.log4j.FileAppender
+log4j.appender.TRACEFILE.Threshold=TRACE
+log4j.appender.TRACEFILE.File=${zookeeper.tracelog.dir}/${zookeeper.tracelog.file}
+
+log4j.appender.TRACEFILE.layout=org.apache.log4j.PatternLayout
+### Notice we are including log4j's NDC here (%x)
+log4j.appender.TRACEFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L][%x] - %m%n
diff --git a/conf/zookeeper/zoo.cfg b/conf/zookeeper/zoo.cfg
new file mode 100644
index 0000000..61c0460
--- /dev/null
+++ b/conf/zookeeper/zoo.cfg
@@ -0,0 +1,3 @@
+tickTime=2000
+dataDir=/var/lib/masterkey/lui/zookeeper
+clientPort=2181
\ No newline at end of file
diff --git a/conf/zookeeper/zookeeper.default b/conf/zookeeper/zookeeper.default
deleted file mode 100644
index 9eacc73..0000000
--- a/conf/zookeeper/zookeeper.default
+++ /dev/null
@@ -1,11 +0,0 @@
-# See the following page for extensive details on setting
-# up the JVM to accept JMX remote management:
-# http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
-# by default we allow local JMX connections
-#JMXLOCALONLY=false
-
-#JAVA_OPTS=""
-
-# Make sure to include the original classpath since it contains the zookeeper
-# jars
-#CLASSPATH="$CLASSPATH:YOURCLASSPATH"
diff --git a/conf/zookeeper/zookeeper.init.conf b/conf/zookeeper/zookeeper.init.conf
deleted file mode 100644
index 167ae02..0000000
--- a/conf/zookeeper/zookeeper.init.conf
+++ /dev/null
@@ -1,27 +0,0 @@
-description "zookeeper centralized coordination service"
-
-start on runlevel [2345]
-stop on runlevel [!2345]
-
-respawn
-
-limit nofile 8192 8192
-
-pre-start script
-    [ -r "/usr/share/java/zookeeper.jar" ] || exit 0
-    [ -r "/etc/zookeeper/conf/environment" ] || exit 0
-    . /etc/zookeeper/conf/environment
-    [ -d $ZOO_LOG_DIR ] || mkdir -p $ZOO_LOG_DIR
-    chown $USER:$GROUP $ZOO_LOG_DIR
-end script
-
-script
-    . /etc/zookeeper/conf/environment
-    [ -r /etc/default/zookeeper ] && . /etc/default/zookeeper
-    if [ -z "$JMXDISABLE" ]; then
-        JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY"
-    fi
-    exec start-stop-daemon --start -c $USER --exec $JAVA --name zookeeper \
-    	-- -cp $CLASSPATH $JAVA_OPTS -Dzookeeper.log.dir=${ZOO_LOG_DIR} \
-      	-Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG
-end script
diff --git a/debian/lui-zookeeper.default b/debian/lui-zookeeper.default
new file mode 100755
index 0000000..a6a318e
--- /dev/null
+++ b/debian/lui-zookeeper.default
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script should be sourced into other zookeeper
+# scripts to setup the env variables
+
+# We use ZOOCFGDIR if defined,
+# otherwise we use /etc/zookeeper
+# or the conf directory that is
+# a sibling of this script's directory
+
+ZOOBINDIR=/usr/share/masterkey/lui/zookeeper/bin
+ZOOKEEPER_PREFIX="${ZOOBINDIR}/.."
+ZOOCFGDIR=/etc/masterkey/lui/zookeeper
+ZOO_LOG_DIR=/var/log/masterkey/lui
+ZOOPIDDIR=/var/lib/masterkey/lui
+ZOOPIDFILE=${ZOOPIDDIR}/zookeeper_server.pid
+
+if [ "x$ZOOCFGDIR" = "x" ]
+then
+  if [ -e "${ZOOKEEPER_PREFIX}/conf" ]; then
+    ZOOCFGDIR="$ZOOBINDIR/../conf"
+  else
+    ZOOCFGDIR="$ZOOBINDIR/../etc/zookeeper"
+  fi
+fi
+
+if [ "x$ZOOCFG" = "x" ]
+then
+    ZOOCFG="zoo.cfg"
+fi
+
+ZOOCFG="$ZOOCFGDIR/$ZOOCFG"
+
+if [ -f "$ZOOCFGDIR/java.env" ]
+then
+    . "$ZOOCFGDIR/java.env"
+fi
+
+if [ "x${ZOO_LOG_DIR}" = "x" ]
+then
+    ZOO_LOG_DIR="."
+fi
+
+if [ "x${ZOO_LOG4J_PROP}" = "x" ]
+then
+    ZOO_LOG4J_PROP="INFO,CONSOLE"
+fi
+
+if [ "$JAVA_HOME" != "" ]; then
+  JAVA="$JAVA_HOME/bin/java"
+else
+  JAVA=java
+fi
+
+#add the zoocfg dir to classpath
+CLASSPATH="$ZOOCFGDIR:$CLASSPATH"
+
+for i in "$ZOOBINDIR"/../src/java/lib/*.jar
+do
+    CLASSPATH="$i:$CLASSPATH"
+done
+
+#make it work in the binary package
+if [ -e "${ZOOKEEPER_PREFIX}"/share/zookeeper/zookeeper-*.jar ]; then
+  for i in "$ZOOKEEPER_PREFIX"/share/zookeeper/*.jar
+  do
+    CLASSPATH="$i:$CLASSPATH"
+  done
+else
+  #release tarball format
+  for i in "$ZOOBINDIR"/../zookeeper-*.jar
+  do
+    CLASSPATH="$i:$CLASSPATH"
+  done
+  for i in "$ZOOBINDIR"/../lib/*.jar
+  do
+    CLASSPATH="$i:$CLASSPATH"
+  done
+fi
+
+#make it work for developers
+for d in "$ZOOBINDIR"/../build/lib/*.jar
+do
+   CLASSPATH="$d:$CLASSPATH"
+done
+
+#make it work for developers
+CLASSPATH="$ZOOBINDIR/../build/classes:$CLASSPATH"
+
+case "`uname`" in
+    CYGWIN*) cygwin=true ;;
+    *) cygwin=false ;;
+esac
+
+if $cygwin
+then
+    CLASSPATH=`cygpath -wp "$CLASSPATH"`
+fi
+
+#echo "CLASSPATH=$CLASSPATH"
diff --git a/debian/lui-zookeeper.init b/debian/lui-zookeeper.init
new file mode 100755
index 0000000..b9147d2
--- /dev/null
+++ b/debian/lui-zookeeper.init
@@ -0,0 +1,139 @@
+#! /bin/sh
+
+# init script for lui-zookeeper
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+### BEGIN INIT INFO
+# Provides:		lui-zookeeper
+# Required-Start:	$remote_fs $syslog
+# Required-Stop:	$remote_fs $syslog
+# Default-Start:	2 3 4 5
+# Default-Stop:		
+# Short-Description:	Apache ZooKeeper server for Index Data Local Unified Index (LUI)
+### END INIT INFO
+
+set -e
+
+# /etc/init.d/lui-zookeeper: start and stop the Apache ZooKeeper daemon
+
+umask 022
+
+. /etc/default/lui-zookeeper
+
+. /lib/lsb/init-functions
+
+check_privsep_dir() {
+    # Create the PrivSep empty dir if necessary
+    if [ ! -d ${ZOOPIDDIR} ]; then
+	mkdir -p ${ZOOPIDDIR}
+        chown lui-solr:lui-solr ${ZOOPIDDIR}
+	chmod 0775 ${ZOOPIDDIR} 
+    fi
+}
+
+# Are we running from init?
+run_by_init() {
+    ([ "$previous" ] && [ "$runlevel" ]) || [ "$runlevel" = S ]
+}
+
+check_for_no_start() {
+    # forget it if we're trying to start, and /etc/masterkey/lui/zookeeper/zookeeper_not_to_be_run exists
+    if [ -e /etc/masterkey/lui/zookeeper/zookeeper_not_to_be_run ]; then 
+	if [ "$1" = log_end_msg ]; then
+	    log_end_msg 0
+	fi
+	if ! run_by_init; then
+	    log_action_msg "Apache ZooKeeper server not in use (/etc/zookeeper/zookeeper_not_to_be_run)"
+	fi
+	exit 0
+    fi
+}
+
+export PATH="${PATH:+$PATH:}/usr/sbin:/usr/bin"
+
+case "$1" in
+  start)
+	check_for_no_start
+	check_privsep_dir
+	log_daemon_msg "Starting Apache ZooKeeper server" "zookeeper"
+	if start-stop-daemon --start --quiet --oknodo --pidfile ${ZOOPIDFILE} -c lui-solr -x ${ZOOKEEPER_PREFIX}/bin/zkServer.sh start; then
+	    log_end_msg 0
+	else
+	    log_end_msg 1
+	fi
+	;;
+  stop)
+	log_daemon_msg "Stopping Apache ZooKeeper server" "zookeeper"
+	if start-stop-daemon --stop --quiet --oknodo --pidfile ${ZOOPIDFILE}; then
+	    log_end_msg 0
+	else
+	    log_end_msg 1
+	fi
+	;;
+
+  restart)
+	check_privsep_dir
+	log_daemon_msg "Restarting Apache ZooKeeper server" "zookeeper"
+	start-stop-daemon --stop --quiet --oknodo --retry 30 --pidfile ${ZOOPIDFILE}
+	check_for_no_start log_end_msg
+	if start-stop-daemon --start --quiet --oknodo --pidfile ${ZOOPIDFILE} -c lui-solr -x ${ZOOKEEPER_PREFIX}/bin/zkServer.sh start; then
+	    log_end_msg 0
+	else
+	    log_end_msg 1
+	fi
+	;;
+
+  try-restart)
+	check_privsep_dir
+	log_daemon_msg "Restarting Apache ZooKeeper server" "zookeeper"
+	set +e
+	start-stop-daemon --stop --quiet --retry 30 --pidfile ${ZOOPIDFILE}
+	RET="$?"
+	set -e
+	case $RET in
+	    0)
+		# old daemon stopped
+		check_for_no_start log_end_msg
+		if start-stop-daemon --start --quiet --oknodo --pidfile ${ZOOPIDFILE} -c lui-solr -x ${ZOOKEEPER_PREFIX}/bin/zkServer.sh start; then
+		    log_end_msg 0
+		else
+		    log_end_msg 1
+		fi
+		;;
+	    1)
+		# daemon not running
+		log_progress_msg "(not running)"
+		log_end_msg 0
+		;;
+	    *)
+		# failed to stop
+		log_progress_msg "(failed to stop)"
+		log_end_msg 1
+		;;
+	esac
+	;;
+
+  status)
+	status_of_proc -p ${ZOOPIDFILE} ${JAVA_HOME}/bin/java zookeeper && exit 0 || exit $?
+	;;
+
+  *)
+	log_action_msg "Usage: /etc/init.d/lui-zookeeper {start|stop|restart|try-restart|status}"
+	exit 1
+esac
+
+exit 0
diff --git a/dev-deploy-ha/zookeeper.yml b/dev-deploy-ha/zookeeper.yml
index e92fb66..dfb0a37 100644
--- a/dev-deploy-ha/zookeeper.yml
+++ b/dev-deploy-ha/zookeeper.yml
@@ -41,3 +41,27 @@
     - name: Link Zookeeper
       become: yes
       file: src=/usr/share/masterkey/lui/zookeeper-3.4.6 path=/usr/share/masterkey/lui/zookeeper state=link
+
+    - name: Create Zookeeper libexec directory
+      become: yes
+      file: path=/usr/share/masterkey/lui/zookeeper/libexec state=directory
+
+    - name: Link Zookeeper config
+      become: yes
+      file: src=/vagrant/conf/zookeeper path=/etc/masterkey/lui/zookeeper state=link
+
+    - name: Link Zookeeper defaults into /etc/default
+      become: yes
+      file: src=/vagrant/debian/lui-zookeeper.default path=/etc/default/lui-zookeeper state=link
+
+    - name: Link Zookeeper defaults into /usr/share/masterkey/lui/zookeeper/libexec
+      become: yes
+      file: src=/etc/default/lui-zookeeper path=/usr/share/masterkey/lui/zookeeper/libexec/zkEnv.sh state=link
+
+    - name: Link init script
+      become: yes
+      file: src=/vagrant/debian/lui-zookeeper.init path=/etc/init.d/lui-zookeeper state=link
+
+    - name: Enable and start service
+      become: yes
+      service: name=lui-zookeeper enabled=yes state=started
diff --git a/script/zookeeper.initscript.sh b/script/zookeeper.initscript.sh
deleted file mode 100755
index 444789a..0000000
--- a/script/zookeeper.initscript.sh
+++ /dev/null
@@ -1,166 +0,0 @@
-#! /bin/sh
-### BEGIN INIT INFO
-# Provides:          zookeeper
-# Required-Start:    $remote_fs
-# Required-Stop:     $remote_fs
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: centralized coordination service
-# Description:       ZooKeeper is a centralized service for maintaining
-#                    configuration information, naming, providing distributed
-#                    synchronization, and providing group services.
-### END INIT INFO
-
-# Author: Foo Bar <foobar@baz.org>
-#
-# Please remove the "Author" lines above and replace them
-# with your own name if you copy and modify this script.
-
-# Do NOT "set -e"
-
-# Exit if the package is not installed
-# Test that libzookeeper-java is installed
-[ -r "/usr/share/java/zookeeper.jar" ] || exit 0
-# Test that zookeeper is installed and not purged
-[ -r "/etc/zookeeper/conf/environment" ] || exit 0
-. /etc/zookeeper/conf/environment
-
-# PATH should only include /usr/* if it runs after the mountnfs.sh script
-PATH=/sbin:/usr/sbin:/bin:/usr/bin
-DESC="centralized coordination service"
-SCRIPTNAME=/etc/init.d/$NAME
-
-# Read configuration variable file if it is present
-[ -r /etc/default/$NAME ] && . /etc/default/$NAME
-
-# Load the VERBOSE setting and other rcS variables
-. /lib/init/vars.sh
-
-# Define LSB log_* functions.
-# Depend on lsb-base (>= 3.0-6) to ensure that this file is present.
-. /lib/lsb/init-functions
-
-is_running()
-{
-	start-stop-daemon --start --quiet --pidfile $PIDFILE --exec $JAVA --user $USER --test > /dev/null \
-		|| return 1
-    return 0
-}
-
-#
-# Function that starts the daemon/service
-#
-do_start()
-{
-	# Return
-	#   0 if daemon has been started
-	#   1 if daemon was already running
-	#   2 if daemon could not be started
-    is_running || return 1
-
-    if [ ! -d $PIDDIR ]
-    then
-      mkdir -p $PIDDIR
-    fi
-    chown $USER:$GROUP $PIDDIR
-
-    if [ ! -d  $ZOO_LOG_DIR ]
-    then
-      mkdir -p $ZOO_LOG_DIR
-    fi
-    chown $USER:$GROUP $ZOO_LOG_DIR
-
-	start-stop-daemon --start --quiet \
-               --pidfile $PIDFILE \
-               --make-pidfile \
-               --chuid $USER:$GROUP \
-               --background \
-               --exec $JAVA -- \
-                 -cp $CLASSPATH \
-                 $JAVA_OPTS \
-                 -Dzookeeper.log.dir=${ZOO_LOG_DIR} \
-                 -Dzookeeper.root.logger=${ZOO_LOG4J_PROP} \
-                 $ZOOMAIN $ZOOCFG \
-		|| return 2
-}
-
-#
-# Function that stops the daemon/service
-#
-do_stop()
-{
-	# Return
-	#   0 if daemon has been stopped
-	#   1 if daemon was already stopped
-	#   2 if daemon could not be stopped
-	#   other if a failure occurred
-	is_running && return 1
-
-	start-stop-daemon --stop --quiet --retry=TERM/30/KILL/5 --pidfile $PIDFILE
-	RETVAL="$?"
-	[ "$RETVAL" = 2 ] && return 2
-	# Many daemons don't delete their pidfiles when they exit.
-	[ "$RETVAL" = 0 ] && rm -f $PIDFILE
-	return "$RETVAL"
-}
-
-case "$1" in
-  start)
-        if [ "x$JMXDISABLE" = "x" ]
-        then
-            [ "$VERBOSE" != no ] && log_action_msg "$NAME: JMX enabled by default"
-            # for some reason these two options are necessary on jdk6 on Ubuntu
-            #   accord to the docs they are not necessary, but otw jconsole cannot
-            #   do a local attach
-            JAVA_OPTS="$JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=$JMXLOCALONLY"
-        else
-            [ "$VERBOSE" != no ] && log_action_msg "$NAME: JMX disabled by user request"
-        fi
-
-	[ "$VERBOSE" != no ] && log_daemon_msg "Starting $DESC" "$NAME"
-	do_start
-	case "$?" in
-		0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
-		2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
-	esac
-	;;
-  stop)
-	[ "$VERBOSE" != no ] && log_daemon_msg "Stopping $DESC" "$NAME"
-	do_stop
-	case "$?" in
-		0|1) [ "$VERBOSE" != no ] && log_end_msg 0 ;;
-		2) [ "$VERBOSE" != no ] && log_end_msg 1 ;;
-	esac
-	;;
-  status)
-       status_of_proc -p $PIDFILE "$NAME" "$NAME" && exit 0 || exit $?
-       ;;
-  restart|force-reload)
-	#
-	# If the "reload" option is implemented then remove the
-	# 'force-reload' alias
-	#
-	log_daemon_msg "Restarting $DESC" "$NAME"
-	do_stop
-	case "$?" in
-	  0|1)
-		do_start
-		case "$?" in
-			0) log_end_msg 0 ;;
-			1) log_end_msg 1 ;; # Old process is still running
-			*) log_end_msg 1 ;; # Failed to start
-		esac
-		;;
-	  *)
-	  	# Failed to stop
-		log_end_msg 1
-		;;
-	esac
-	;;
-  *)
-	echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2
-	exit 3
-	;;
-esac
-
-: