cdh6.3.2 Spark 多版本共存
tar -zxvf spark-3.3.1-bin-3.0.0-cdh6.3.2.tgz -C /opt/cloudera/parcels/CDH/lib
cd /opt/cloudera/parcels/CDH/lib
mv spark-3.3.1-bin-3.0.0-cdh6.3.2/ spark3
cp /etc/spark/conf/spark-env.sh /opt/cloudera/parcels/CDH/lib/spark3/conf
chmod +x /opt/cloudera/parcels/CDH/lib/spark3/conf/spark-env.sh
vim /opt/cloudera/parcels/CDH/lib/spark3/conf/spark-env.sh
export SPARK_HOME=/opt/cloudera/parcels/CDH/lib/spark3
HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
cp /etc/hive/conf/hive-site.xml /opt/cloudera/parcels/CDH/lib/spark3/conf/
tar -zxvf spark-2.4.0-bin-hadoop2.7.tgz -C /opt/cloudera/parcels/CDH/lib
cd /opt/cloudera/parcels/CDH/lib
mv spark-2.4.0-bin-hadoop2.7/ spark2
cp /etc/spark/conf/spark-env.sh /opt/cloudera/parcels/CDH/lib/spark2/conf
chmod +x /opt/cloudera/parcels/CDH/lib/spark2/conf/spark-env.sh
vim /opt/cloudera/parcels/CDH/lib/spark2/conf/spark-env.sh
export SPARK_HOME=/opt/cloudera/parcels/CDH/lib/spark2
HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/etc/hadoop/conf}
cp /etc/hive/conf/hive-site.xml /opt/cloudera/parcels/CDH/lib/spark2/conf/
vim /opt/cloudera/parcels/CDH/bin/spark3-sql
export HADOOP_CONF_DIR=/etc/hadoop/conf
export YARN_CONF_DIR=/etc/hadoop/conf
SOURCE="${BASH_SOURCE[0]}"
BIN_DIR="$( dirname "$SOURCE" )"
while [ -h "$SOURCE" ]
do
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$BIN_DIR/$SOURCE"
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
done
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
LIB_DIR=$BIN_DIR/../lib
export HADOOP_HOME=$LIB_DIR/hadoop
. $LIB_DIR/bigtop-utils/bigtop-detect-javahome
exec $LIB_DIR/spark3/bin/spark-submit --class org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver "$@"
chmod +x /opt/cloudera/parcels/CDH/bin/spark3-sql
alternatives --install /usr/bin/spark-sql spark-sql /opt/cloudera/parcels/CDH/bin/spark3-sql 1
vim /opt/cloudera/parcels/CDH/bin/spark2-sql
export HADOOP_CONF_DIR=/etc/hadoop/conf
export YARN_CONF_DIR=/etc/hadoop/conf
SOURCE="${BASH_SOURCE[0]}"
BIN_DIR="$( dirname "$SOURCE" )"
while [ -h "$SOURCE" ]
do
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$BIN_DIR/$SOURCE"
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
done
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
LIB_DIR=$BIN_DIR/../lib
export HADOOP_HOME=$LIB_DIR/hadoop
. $LIB_DIR/bigtop-utils/bigtop-detect-javahome
exec $LIB_DIR/spark2/bin/spark-submit --class org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver "$@"
chmod +x /opt/cloudera/parcels/CDH/bin/spark2-sql
alternatives --install /usr/bin/spark-sql spark-sql /opt/cloudera/parcels/CDH/bin/spark2-sql 2
cd /opt/cloudera/parcels/CDH/lib/spark3/conf
mv log4j2.properties.template log4j2.properties
cp /opt/cloudera/parcels/CDH/lib/spark/conf/spark-defaults.conf ./
vim /opt/cloudera/parcels/CDH/lib/spark3/conf/spark-defaults.conf
hadoop fs -mkdir -p /spark/3versionJars
cd /opt/cloudera/parcels/CDH/lib/spark3/jars
hadoop fs -put *.jar /spark/3versionJars
cd /opt/cloudera/parcels/CDH/lib/spark2/conf
mv log4j2.properties.template log4j2.properties
cp /opt/cloudera/parcels/CDH/lib/spark/conf/spark-defaults.conf ./
vim /opt/cloudera/parcels/CDH/lib/spark3/conf/spark-defaults.conf
hadoop fs -mkdir -p /spark/2versionJars
cd /opt/cloudera/parcels/CDH/lib/spark2/jars
hadoop fs -put *.jar /spark/2versionJars
vim /opt/cloudera/parcels/CDH/bin/spark3-submit
export HADOOP_CONF_DIR=/etc/hadoop/conf
export YARN_CONF_DIR=/etc/hadoop/conf
SOURCE="${BASH_SOURCE[0]}"
BIN_DIR="$( dirname "$SOURCE" )"
while [ -h "$SOURCE" ]
do
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$BIN_DIR/$SOURCE"
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
done
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
LIB_DIR=/opt/cloudera/parcels/CDH/lib
export HADOOP_HOME=$LIB_DIR/hadoop
. $LIB_DIR/bigtop-utils/bigtop-detect-javahome
export PYTHONHASHSEED=0
exec $LIB_DIR/spark3/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@"
chmod +755 /opt/cloudera/parcels/CDH/bin/spark3-submit
alternatives --install /usr/bin/spark-submit spark-submit /opt/cloudera/parcels/CDH/bin/spark3-submit 1
vim /opt/cloudera/parcels/CDH/bin/spark2-submit
export HADOOP_CONF_DIR=/etc/hadoop/conf
export YARN_CONF_DIR=/etc/hadoop/conf
SOURCE="${BASH_SOURCE[0]}"
BIN_DIR="$( dirname "$SOURCE" )"
while [ -h "$SOURCE" ]
do
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$BIN_DIR/$SOURCE"
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
done
BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
LIB_DIR=/opt/cloudera/parcels/CDH/lib
export HADOOP_HOME=$LIB_DIR/hadoop
. $LIB_DIR/bigtop-utils/bigtop-detect-javahome
export PYTHONHASHSEED=0
exec $LIB_DIR/spark2/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@"
chmod +755 /opt/cloudera/parcels/CDH/bin/spark2-submit
alternatives --install /usr/bin/spark-submit spark-submit /opt/cloudera/parcels/CDH/bin/spark2-submit 1