Date: Fri, 6 Nov 2015 15:14:47 +0000 (UTC) From: Dmitry Sivachenko <demon@FreeBSD.org> To: ports-committers@freebsd.org, svn-ports-all@freebsd.org, svn-ports-head@freebsd.org Subject: svn commit: r400935 - in head/devel/spark: . files Message-ID: <201511061514.tA6FElqJ048659@repo.freebsd.org>
next in thread | raw e-mail | index | archive | help
Author: demon Date: Fri Nov 6 15:14:47 2015 New Revision: 400935 URL: https://svnweb.freebsd.org/changeset/ports/400935 Log: Update to version 1.5.1. Submitted by: Mark Dixon <mnd999@gmail.com> Modified: head/devel/spark/Makefile head/devel/spark/distinfo head/devel/spark/files/patch-sbin-spark-daemon.sh head/devel/spark/files/spark_master.in head/devel/spark/files/spark_worker.in head/devel/spark/pkg-plist Modified: head/devel/spark/Makefile ============================================================================== --- head/devel/spark/Makefile Fri Nov 6 14:30:04 2015 (r400934) +++ head/devel/spark/Makefile Fri Nov 6 15:14:47 2015 (r400935) @@ -2,7 +2,7 @@ # $FreeBSD$ PORTNAME= spark -PORTVERSION= 1.2.1 +PORTVERSION= 1.5.1 CATEGORIES= devel java MASTER_SITES= APACHE/${PORTNAME}/${PORTNAME}-${PORTVERSION} \ LOCAL/demon/:maven \ @@ -16,7 +16,8 @@ COMMENT= Fast big data processing engine LICENSE= APACHE20 -BUILD_DEPENDS= ${LOCALBASE}/share/java/maven3/bin/mvn:${PORTSDIR}/devel/maven3 +BUILD_DEPENDS= ${LOCALBASE}/share/java/maven33/bin/mvn:${PORTSDIR}/devel/maven33 \ + ${LOCALBASE}/lib/libsnappyjava.so:${PORTSDIR}/archivers/snappy-java RUN_DEPENDS= bash:${PORTSDIR}/shells/bash \ yarn:${PORTSDIR}/devel/hadoop2 @@ -24,7 +25,6 @@ USES= python:2 USE_JAVA= yes JAVA_VERSION= 1.7+ MAKE_ENV+= MAVEN_OPTS="-Xmx2g -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=512m" -NO_ARCH= YES USERS= spark GROUPS= spark @@ -33,8 +33,21 @@ USE_RC_SUBR= spark_master spark_worker PLIST_SUB+= SPARK_USER=spark SPARK_GROUP=spark VER=${PORTVERSION} SUB_LIST+= SPARK_USER=spark SPARK_GROUP=spark +.include <bsd.port.pre.mk> + +.if ${ARCH} == "amd64" +JAVA_ARCH = x86_64 +.elif ${ARCH} == "i386" +JAVA_ARCH = x86 +.else +JAVA_ARCH = ${ARCH} +.endif + do-build: - cd ${WRKSRC} && ${SETENV} ${MAKE_ENV} ${LOCALBASE}/share/java/maven3/bin/mvn --offline -Dmaven.repo.local=${WRKDIR}/m2 clean package -Dhadoop.version=2.4.1 -Pyarn -Phadoop-2.4 -DskipTests + ${MKDIR} ${WRKDIR}/snappy/org/xerial/snappy/native/${OPSYS}/${JAVA_ARCH} + ${CP} ${LOCALBASE}/lib/libsnappyjava.so ${WRKDIR}/snappy/org/xerial/snappy/native/${OPSYS}/${JAVA_ARCH} + cd ${WRKSRC} && ${SETENV} ${MAKE_ENV} ${LOCALBASE}/share/java/maven33/bin/mvn --offline -Dmaven.repo.local=${WRKDIR}/m2 clean package -Dhadoop.version=2.7.1 -Pyarn -Phadoop-2.6 -DskipTests + ${JAR} uvf ${WRKSRC}/assembly/target/scala*/*assembly*hadoop*.jar -C ${WRKDIR}/snappy org post-build: ${RM} ${WRKSRC}/bin/*.cmd ${WRKSRC}/sbin/spark-daemon.sh.orig @@ -52,4 +65,4 @@ do-install: ${MKDIR} ${STAGEDIR}/var/run/spark ${MKDIR} ${STAGEDIR}/var/log/spark -.include <bsd.port.mk> +.include <bsd.port.post.mk> Modified: head/devel/spark/distinfo ============================================================================== --- head/devel/spark/distinfo Fri Nov 6 14:30:04 2015 (r400934) +++ head/devel/spark/distinfo Fri Nov 6 15:14:47 2015 (r400935) @@ -1,4 +1,4 @@ -SHA256 (hadoop/spark-1.2.1.tgz) = 42257cab38e3217fa3b68efdec05bc98c716ad65f8cd3215d4f5718d5865b72c -SIZE (hadoop/spark-1.2.1.tgz) = 10318179 -SHA256 (hadoop/FreeBSD-spark-1.2.1-maven-repository.tar.gz) = c832b440ed2fb9b591308094ee8c1b469b2e0b795c3d39c18b0e2bab43326206 -SIZE (hadoop/FreeBSD-spark-1.2.1-maven-repository.tar.gz) = 262876744 +SHA256 (hadoop/spark-1.5.1.tgz) = 96cfd31ab5c07fe40dfca0475c7df888e2abf15126ff2a3c3af1facd02f1a848 +SIZE (hadoop/spark-1.5.1.tgz) = 74046742 +SHA256 (hadoop/FreeBSD-spark-1.5.1-maven-repository.tar.gz) = 088f7cff25c9b6c9d485061ae004f2e05457d7bc7f05219be3e922b9f92b23a2 +SIZE (hadoop/FreeBSD-spark-1.5.1-maven-repository.tar.gz) = 292317551 Modified: head/devel/spark/files/patch-sbin-spark-daemon.sh ============================================================================== --- head/devel/spark/files/patch-sbin-spark-daemon.sh Fri Nov 6 14:30:04 2015 (r400934) +++ head/devel/spark/files/patch-sbin-spark-daemon.sh Fri Nov 6 15:14:47 2015 (r400935) @@ -1,5 +1,5 @@ ---- sbin/spark-daemon.sh.orig 2014-12-20 14:07:28.000000000 +0300 -+++ sbin/spark-daemon.sh 2014-12-20 14:07:31.000000000 +0300 +--- sbin/spark-daemon.sh.orig 2015-10-26 09:05:25.709519603 +0000 ++++ sbin/spark-daemon.sh 2015-10-26 09:07:31.767513027 +0000 @@ -99,14 +99,6 @@ if [ "$SPARK_LOG_DIR" = "" ]; then export SPARK_LOG_DIR="$SPARK_HOME/logs" @@ -15,12 +15,12 @@ if [ "$SPARK_PID_DIR" = "" ]; then SPARK_PID_DIR=/tmp -@@ -126,8 +118,6 @@ +@@ -125,8 +117,6 @@ + mode="$1" + shift - (start|spark-submit) - -- mkdir -p "$SPARK_PID_DIR" +- mkdir -p "$SPARK_PID_DIR" - - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. + if [ -f "$pid" ]; then + TARGET_ID="$(cat "$pid")" + if [[ $(ps -p "$TARGET_ID" -o comm=) =~ "java" ]]; then Modified: head/devel/spark/files/spark_master.in ============================================================================== --- head/devel/spark/files/spark_master.in Fri Nov 6 14:30:04 2015 (r400934) +++ head/devel/spark/files/spark_master.in Fri Nov 6 15:14:47 2015 (r400935) @@ -12,11 +12,13 @@ rcvar=spark_master_enable load_rc_config $name : ${spark_master_enable:=NO} +: ${spark_master_ip=:`hostname`} : ${spark_master_port:=7077} : ${spark_master_webui_port:=8080} export SPARK_PID_DIR=/var/run/spark export SPARK_LOG_DIR=/var/log/spark +export SPARK_MASTER_IP=${spark_master_ip} export SPARK_MASTER_PORT=${spark_master_port} export SPARK_MASTER_WEBUI_PORT=${spark_master_webui_port} export SPARK_IDENT_STRING=%%SPARK_USER%% Modified: head/devel/spark/files/spark_worker.in ============================================================================== --- head/devel/spark/files/spark_worker.in Fri Nov 6 14:30:04 2015 (r400934) +++ head/devel/spark/files/spark_worker.in Fri Nov 6 15:14:47 2015 (r400935) @@ -21,8 +21,8 @@ export SPARK_IDENT_STRING=%%SPARK_USER%% export SPARK_WORKER_DIR=${spark_worker_dir} pidfile=${SPARK_PID_DIR}/spark-${SPARK_IDENT_STRING}-org.apache.spark.deploy.worker.Worker-1.pid -start_cmd="/usr/bin/su -m %%SPARK_USER%% -c \"%%DATADIR%%/sbin/start-slave.sh 1 ${spark_worker_master}\"" -stop_cmd='/usr/bin/su -m %%SPARK_USER%% -c "%%DATADIR%%/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker 1"' +start_cmd="/usr/bin/su -m %%SPARK_USER%% -c \"%%DATADIR%%/sbin/start-slave.sh ${spark_worker_master}\"" +stop_cmd='/usr/bin/su -m %%SPARK_USER%% -c "%%DATADIR%%/sbin/spark-daemon.sh stop org.apache.spark.deploy.worker.Worker"' export PATH=$PATH:%%LOCALBASE%%/bin Modified: head/devel/spark/pkg-plist ============================================================================== --- head/devel/spark/pkg-plist Fri Nov 6 14:30:04 2015 (r400934) +++ head/devel/spark/pkg-plist Fri Nov 6 15:14:47 2015 (r400935) @@ -1,6 +1,5 @@ %%DATADIR%%/RELEASE %%DATADIR%%/bin/beeline -%%DATADIR%%/bin/compute-classpath.sh %%DATADIR%%/bin/load-spark-env.sh %%DATADIR%%/bin/pyspark %%DATADIR%%/bin/run-example @@ -8,7 +7,8 @@ %%DATADIR%%/bin/spark-shell %%DATADIR%%/bin/spark-sql %%DATADIR%%/bin/spark-submit -%%DATADIR%%/bin/utils.sh +%%DATADIR%%/bin/sparkR +%%DATADIR%%/conf/docker.properties.template %%DATADIR%%/conf/fairscheduler.xml.template %%DATADIR%%/conf/log4j.properties.template %%DATADIR%%/conf/metrics.properties.template @@ -23,20 +23,32 @@ %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/JavaTC.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/JavaWordCount.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaCrossValidatorExample.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaDeveloperApiExample.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleParamsExample.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaSimpleTextClassificationPipeline.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/ml/JavaTrainValidationSplitExample.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaALS.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaDecisionTree.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaFPGrowthExample.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostedTreesRunner.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaKMeans.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaLDAExample.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaLR.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaPowerIterationClusteringExample.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestExample.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQL.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaDirectKafkaWordCount.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaFlumeEventCount.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaKafkaWordCount.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaQueueStream.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecord.java %%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java +%%DATADIR%%/examples/src/main/java/org/apache/spark/examples/streaming/JavaStatefulNetworkWordCount.java %%DATADIR%%/examples/src/main/python/als.py %%DATADIR%%/examples/src/main/python/avro_inputformat.py %%DATADIR%%/examples/src/main/python/cassandra_inputformat.py @@ -45,9 +57,18 @@ %%DATADIR%%/examples/src/main/python/hbase_outputformat.py %%DATADIR%%/examples/src/main/python/kmeans.py %%DATADIR%%/examples/src/main/python/logistic_regression.py +%%DATADIR%%/examples/src/main/python/ml/cross_validator.py +%%DATADIR%%/examples/src/main/python/ml/gradient_boosted_trees.py +%%DATADIR%%/examples/src/main/python/ml/kmeans_example.py +%%DATADIR%%/examples/src/main/python/ml/logistic_regression.py +%%DATADIR%%/examples/src/main/python/ml/random_forest_example.py +%%DATADIR%%/examples/src/main/python/ml/simple_params_example.py +%%DATADIR%%/examples/src/main/python/ml/simple_text_classification_pipeline.py %%DATADIR%%/examples/src/main/python/mllib/correlations.py %%DATADIR%%/examples/src/main/python/mllib/dataset_example.py %%DATADIR%%/examples/src/main/python/mllib/decision_tree_runner.py +%%DATADIR%%/examples/src/main/python/mllib/gaussian_mixture_model.py +%%DATADIR%%/examples/src/main/python/mllib/gradient_boosted_trees.py %%DATADIR%%/examples/src/main/python/mllib/kmeans.py %%DATADIR%%/examples/src/main/python/mllib/logistic_regression.py %%DATADIR%%/examples/src/main/python/mllib/random_forest_example.py @@ -59,12 +80,21 @@ %%DATADIR%%/examples/src/main/python/pi.py %%DATADIR%%/examples/src/main/python/sort.py %%DATADIR%%/examples/src/main/python/sql.py +%%DATADIR%%/examples/src/main/python/status_api_demo.py +%%DATADIR%%/examples/src/main/python/streaming/direct_kafka_wordcount.py +%%DATADIR%%/examples/src/main/python/streaming/flume_wordcount.py %%DATADIR%%/examples/src/main/python/streaming/hdfs_wordcount.py +%%DATADIR%%/examples/src/main/python/streaming/kafka_wordcount.py +%%DATADIR%%/examples/src/main/python/streaming/mqtt_wordcount.py %%DATADIR%%/examples/src/main/python/streaming/network_wordcount.py +%%DATADIR%%/examples/src/main/python/streaming/queue_stream.py %%DATADIR%%/examples/src/main/python/streaming/recoverable_network_wordcount.py +%%DATADIR%%/examples/src/main/python/streaming/sql_network_wordcount.py %%DATADIR%%/examples/src/main/python/streaming/stateful_network_wordcount.py %%DATADIR%%/examples/src/main/python/transitive_closure.py %%DATADIR%%/examples/src/main/python/wordcount.py +%%DATADIR%%/examples/src/main/r/data-manipulation.R +%%DATADIR%%/examples/src/main/r/dataframe.R %%DATADIR%%/examples/src/main/resources/full_user.avsc %%DATADIR%%/examples/src/main/resources/kv1.txt %%DATADIR%%/examples/src/main/resources/people.json @@ -75,6 +105,7 @@ %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/BroadcastTest.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/CassandraCQLTest.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/CassandraTest.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/DFSReadWriteTest.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ExceptionHandlingTest.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/GroupByTest.scala @@ -98,31 +129,43 @@ %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/SparkTC.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/SparkTachyonHdfsLR.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/SparkTachyonPi.scala -%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/bagel/PageRankUtils.scala -%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/bagel/WikipediaPageRank.scala -%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/bagel/WikipediaPageRankStandalone.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/Analytics.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/LiveJournalPageRank.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/graphx/SynthBenchmark.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/CrossValidatorExample.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeExample.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/GBTExample.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/KMeansExample.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionExample.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionExample.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/MovieLensALS.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/OneVsRestExample.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestExample.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/SimpleParamsExample.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/SimpleTextClassificationPipeline.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/ml/TrainValidationSplitExample.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassification.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/Correlations.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/CosineSimilarity.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/DatasetExample.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRunner.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/DenseGaussianMixture.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/DenseKMeans.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/FPGrowthExample.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostedTreesRunner.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/LinearRegression.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/MovieLensALS.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/MultivariateSummarizer.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/PowerIterationClusteringExample.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/RandomRDDGeneration.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/SampledRDDs.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/SparseNaiveBayes.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingLinearRegression.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingLogisticRegression.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/TallSkinnyPCA.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/mllib/TallSkinnySVD.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/pythonconverters/AvroConverters.scala @@ -132,14 +175,17 @@ %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/sql/hive/HiveFromSpark.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/ActorWordCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKafkaWordCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/FlumeEventCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/FlumePollingEventCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/HdfsWordCount.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/KafkaWordCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/MQTTWordCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/QueueStream.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/RawNetworkGrep.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala +%%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/StreamingExamples.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/TwitterAlgebirdCMS.scala @@ -148,41 +194,23 @@ %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/ZeroMQWordCount.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/PageViewGenerator.scala %%DATADIR%%/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/PageViewStream.scala -%%DATADIR%%/lib/spark-assembly-%%VER%%-hadoop2.4.1.jar -%%DATADIR%%/lib/spark-examples-%%VER%%-hadoop2.4.1.jar +%%DATADIR%%/lib/spark-assembly-%%VER%%-hadoop%%PYTHON_VER%%.1.jar +%%DATADIR%%/lib/spark-examples-%%VER%%-hadoop%%PYTHON_VER%%.1.jar %%DATADIR%%/python/.gitignore -%%DATADIR%%/python/build/py4j/__init__.py -%%DATADIR%%/python/build/py4j/compat.py -%%DATADIR%%/python/build/py4j/finalizer.py -%%DATADIR%%/python/build/py4j/java_collections.py -%%DATADIR%%/python/build/py4j/java_gateway.py -%%DATADIR%%/python/build/py4j/protocol.py -%%DATADIR%%/python/build/py4j/tests/__init__.py -%%DATADIR%%/python/build/py4j/tests/byte_string_test.py -%%DATADIR%%/python/build/py4j/tests/finalizer_test.py -%%DATADIR%%/python/build/py4j/tests/java_array_test.py -%%DATADIR%%/python/build/py4j/tests/java_callback_test.py -%%DATADIR%%/python/build/py4j/tests/java_gateway_test.py -%%DATADIR%%/python/build/py4j/tests/java_list_test.py -%%DATADIR%%/python/build/py4j/tests/java_map_test.py -%%DATADIR%%/python/build/py4j/tests/java_set_test.py -%%DATADIR%%/python/build/py4j/tests/multithreadtest.py -%%DATADIR%%/python/build/py4j/tests/py4j_callback_example.py -%%DATADIR%%/python/build/py4j/tests/py4j_callback_example2.py -%%DATADIR%%/python/build/py4j/tests/py4j_example.py -%%DATADIR%%/python/build/py4j/version.py %%DATADIR%%/python/docs/Makefile %%DATADIR%%/python/docs/conf.py %%DATADIR%%/python/docs/epytext.py %%DATADIR%%/python/docs/index.rst %%DATADIR%%/python/docs/make.bat %%DATADIR%%/python/docs/make2.bat +%%DATADIR%%/python/docs/pyspark.ml.rst %%DATADIR%%/python/docs/pyspark.mllib.rst %%DATADIR%%/python/docs/pyspark.rst %%DATADIR%%/python/docs/pyspark.sql.rst %%DATADIR%%/python/docs/pyspark.streaming.rst %%DATADIR%%/python/lib/PY4J_LICENSE.txt %%DATADIR%%/python/lib/py4j-0.8.2.1-src.zip +%%DATADIR%%/python/lib/pyspark.zip %%DATADIR%%/python/pyspark/__init__.py %%DATADIR%%/python/pyspark/accumulators.py %%DATADIR%%/python/pyspark/broadcast.py @@ -194,55 +222,119 @@ %%DATADIR%%/python/pyspark/heapq3.py %%DATADIR%%/python/pyspark/java_gateway.py %%DATADIR%%/python/pyspark/join.py +%%DATADIR%%/python/pyspark/ml/__init__.py +%%DATADIR%%/python/pyspark/ml/classification.py +%%DATADIR%%/python/pyspark/ml/clustering.py +%%DATADIR%%/python/pyspark/ml/evaluation.py +%%DATADIR%%/python/pyspark/ml/feature.py +%%DATADIR%%/python/pyspark/ml/param/__init__.py +%%DATADIR%%/python/pyspark/ml/param/_shared_params_code_gen.py +%%DATADIR%%/python/pyspark/ml/param/shared.py +%%DATADIR%%/python/pyspark/ml/pipeline.py +%%DATADIR%%/python/pyspark/ml/recommendation.py +%%DATADIR%%/python/pyspark/ml/regression.py +%%DATADIR%%/python/pyspark/ml/tests.py +%%DATADIR%%/python/pyspark/ml/tuning.py +%%DATADIR%%/python/pyspark/ml/util.py +%%DATADIR%%/python/pyspark/ml/wrapper.py %%DATADIR%%/python/pyspark/mllib/__init__.py %%DATADIR%%/python/pyspark/mllib/classification.py %%DATADIR%%/python/pyspark/mllib/clustering.py %%DATADIR%%/python/pyspark/mllib/common.py +%%DATADIR%%/python/pyspark/mllib/evaluation.py %%DATADIR%%/python/pyspark/mllib/feature.py -%%DATADIR%%/python/pyspark/mllib/linalg.py -%%DATADIR%%/python/pyspark/mllib/rand.py +%%DATADIR%%/python/pyspark/mllib/fpm.py +%%DATADIR%%/python/pyspark/mllib/linalg/__init__.py +%%DATADIR%%/python/pyspark/mllib/linalg/distributed.py +%%DATADIR%%/python/pyspark/mllib/random.py %%DATADIR%%/python/pyspark/mllib/recommendation.py %%DATADIR%%/python/pyspark/mllib/regression.py -%%DATADIR%%/python/pyspark/mllib/stat.py +%%DATADIR%%/python/pyspark/mllib/stat/KernelDensity.py +%%DATADIR%%/python/pyspark/mllib/stat/__init__.py +%%DATADIR%%/python/pyspark/mllib/stat/_statistics.py +%%DATADIR%%/python/pyspark/mllib/stat/distribution.py +%%DATADIR%%/python/pyspark/mllib/stat/test.py %%DATADIR%%/python/pyspark/mllib/tests.py %%DATADIR%%/python/pyspark/mllib/tree.py %%DATADIR%%/python/pyspark/mllib/util.py +%%DATADIR%%/python/pyspark/profiler.py %%DATADIR%%/python/pyspark/rdd.py %%DATADIR%%/python/pyspark/rddsampler.py %%DATADIR%%/python/pyspark/resultiterable.py %%DATADIR%%/python/pyspark/serializers.py %%DATADIR%%/python/pyspark/shell.py %%DATADIR%%/python/pyspark/shuffle.py -%%DATADIR%%/python/pyspark/sql.py +%%DATADIR%%/python/pyspark/sql/__init__.py +%%DATADIR%%/python/pyspark/sql/column.py +%%DATADIR%%/python/pyspark/sql/context.py +%%DATADIR%%/python/pyspark/sql/dataframe.py +%%DATADIR%%/python/pyspark/sql/functions.py +%%DATADIR%%/python/pyspark/sql/group.py +%%DATADIR%%/python/pyspark/sql/readwriter.py +%%DATADIR%%/python/pyspark/sql/tests.py +%%DATADIR%%/python/pyspark/sql/types.py +%%DATADIR%%/python/pyspark/sql/utils.py +%%DATADIR%%/python/pyspark/sql/window.py %%DATADIR%%/python/pyspark/statcounter.py +%%DATADIR%%/python/pyspark/status.py %%DATADIR%%/python/pyspark/storagelevel.py %%DATADIR%%/python/pyspark/streaming/__init__.py %%DATADIR%%/python/pyspark/streaming/context.py %%DATADIR%%/python/pyspark/streaming/dstream.py +%%DATADIR%%/python/pyspark/streaming/flume.py +%%DATADIR%%/python/pyspark/streaming/kafka.py +%%DATADIR%%/python/pyspark/streaming/kinesis.py +%%DATADIR%%/python/pyspark/streaming/mqtt.py %%DATADIR%%/python/pyspark/streaming/tests.py %%DATADIR%%/python/pyspark/streaming/util.py %%DATADIR%%/python/pyspark/tests.py %%DATADIR%%/python/pyspark/traceback_utils.py %%DATADIR%%/python/pyspark/worker.py %%DATADIR%%/python/run-tests +%%DATADIR%%/python/run-tests.py %%DATADIR%%/python/test_support/SimpleHTTPServer.py %%DATADIR%%/python/test_support/hello.txt -%%DATADIR%%/python/test_support/userlib-0.1-py%%PYTHON_VER%%.egg +%%DATADIR%%/python/test_support/sql/orc_partitioned/_SUCCESS +%%DATADIR%%/python/test_support/sql/orc_partitioned/b=0/c=0/.part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc.crc +%%DATADIR%%/python/test_support/sql/orc_partitioned/b=0/c=0/part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc +%%DATADIR%%/python/test_support/sql/orc_partitioned/b=1/c=1/.part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc.crc +%%DATADIR%%/python/test_support/sql/orc_partitioned/b=1/c=1/part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc +%%DATADIR%%/python/test_support/sql/parquet_partitioned/_SUCCESS +%%DATADIR%%/python/test_support/sql/parquet_partitioned/_common_metadata +%%DATADIR%%/python/test_support/sql/parquet_partitioned/_metadata +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2014/month=9/day=1/.part-r-00008.gz.parquet.crc +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2014/month=9/day=1/part-r-00008.gz.parquet +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2015/month=10/day=25/.part-r-00002.gz.parquet.crc +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2015/month=10/day=25/.part-r-00004.gz.parquet.crc +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2015/month=10/day=25/part-r-00002.gz.parquet +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2015/month=10/day=25/part-r-00004.gz.parquet +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2015/month=10/day=26/.part-r-00005.gz.parquet.crc +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2015/month=10/day=26/part-r-00005.gz.parquet +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2015/month=9/day=1/.part-r-00007.gz.parquet.crc +%%DATADIR%%/python/test_support/sql/parquet_partitioned/year=2015/month=9/day=1/part-r-00007.gz.parquet +%%DATADIR%%/python/test_support/sql/people.json +%%DATADIR%%/python/test_support/userlib-0.1.zip %%DATADIR%%/python/test_support/userlibrary.py %%DATADIR%%/sbin/slaves.sh %%DATADIR%%/sbin/spark-config.sh %%DATADIR%%/sbin/spark-daemon.sh %%DATADIR%%/sbin/spark-daemons.sh -%%DATADIR%%/sbin/spark-executor %%DATADIR%%/sbin/start-all.sh %%DATADIR%%/sbin/start-history-server.sh %%DATADIR%%/sbin/start-master.sh +%%DATADIR%%/sbin/start-mesos-dispatcher.sh +%%DATADIR%%/sbin/start-mesos-shuffle-service.sh +%%DATADIR%%/sbin/start-shuffle-service.sh %%DATADIR%%/sbin/start-slave.sh %%DATADIR%%/sbin/start-slaves.sh %%DATADIR%%/sbin/start-thriftserver.sh %%DATADIR%%/sbin/stop-all.sh %%DATADIR%%/sbin/stop-history-server.sh %%DATADIR%%/sbin/stop-master.sh +%%DATADIR%%/sbin/stop-mesos-dispatcher.sh +%%DATADIR%%/sbin/stop-mesos-shuffle-service.sh +%%DATADIR%%/sbin/stop-shuffle-service.sh +%%DATADIR%%/sbin/stop-slave.sh %%DATADIR%%/sbin/stop-slaves.sh %%DATADIR%%/sbin/stop-thriftserver.sh @dir(%%SPARK_USER%%,%%SPARK_GROUP%%,) /var/log/spark
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?201511061514.tA6FElqJ048659>