Commit 4848fe69 authored by titicaca's avatar titicaca
Browse files

upgrade spark to v2.3.x

parent 14b659e7
......@@ -6,7 +6,7 @@
<groupId>org.apache.spark.ml</groupId>
<artifactId>spark-iforest</artifactId>
<version>2.2.0</version>
<version>2.3.0</version>
<properties>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
......@@ -18,7 +18,7 @@
<log4j.version>1.2.17</log4j.version>
<skipTests>false</skipTests>
<maven.version>3.3.9</maven.version>
<spark.version>2.2.0</spark.version>
<spark.version>2.3.0</spark.version>
</properties>
<dependencies>
......
......@@ -89,7 +89,7 @@ class IForestModel (
Array(1 - $(contamination)), 0)
// set anomaly instance label 1
val predictUDF = udf { (anomalyScore: Double) =>
if (anomalyScore >= threshold(0)) 1.0 else 0.0
if (anomalyScore > threshold(0)) 1.0 else 0.0
}
scoreDataset.withColumn($(predictionCol), predictUDF(col($(anomalyScoreCol))))
}
......
log4j.rootCategory=WARN, console
log4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.err
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
# Settings to quiet third party logs that are too verbose
log4j.logger.org.eclipse.jetty=WARN
log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR
log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
......@@ -107,7 +107,6 @@ class IForestSuite extends SparkFunSuite with MLlibTestSparkContext with Default
val summary = model.summary
val anomalies = summary.anomalies.collect
assert(anomalies.length === 10)
// TODO In Spark 2.3.x, function approxQuantile seems to be changed, numAnomalies might be not accurate.
assert(summary.numAnomalies === 2)
val transformed = model.transform(dataset)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment