Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
Maria Karanasou
spark-iforest
Commits
4848fe69
Commit
4848fe69
authored
Dec 27, 2018
by
titicaca
Browse files
upgrade spark to v2.3.x
parent
14b659e7
Changes
4
Hide whitespace changes
Inline
Side-by-side
pom.xml
View file @
4848fe69
...
...
@@ -6,7 +6,7 @@
<groupId>
org.apache.spark.ml
</groupId>
<artifactId>
spark-iforest
</artifactId>
<version>
2.
2
.0
</version>
<version>
2.
3
.0
</version>
<properties>
<project.reporting.outputEncoding>
UTF-8
</project.reporting.outputEncoding>
...
...
@@ -18,7 +18,7 @@
<log4j.version>
1.2.17
</log4j.version>
<skipTests>
false
</skipTests>
<maven.version>
3.3.9
</maven.version>
<spark.version>
2.
2
.0
</spark.version>
<spark.version>
2.
3
.0
</spark.version>
</properties>
<dependencies>
...
...
src/main/scala/org/apache/spark/ml/iforest/IForest.scala
View file @
4848fe69
...
...
@@ -89,7 +89,7 @@ class IForestModel (
Array
(
1
-
$
(
contamination
)),
0
)
// set anomaly instance label 1
val
predictUDF
=
udf
{
(
anomalyScore
:
Double
)
=>
if
(
anomalyScore
>
=
threshold
(
0
))
1.0
else
0.0
if
(
anomalyScore
>
threshold
(
0
))
1.0
else
0.0
}
scoreDataset
.
withColumn
(
$
(
predictionCol
),
predictUDF
(
col
(
$
(
anomalyScoreCol
))))
}
...
...
src/test/resources/log4j.properties
0 → 100644
View file @
4848fe69
log4j.rootCategory
=
WARN, console
log4j.appender.console
=
org.apache.log4j.ConsoleAppender
log4j.appender.console.target
=
System.err
log4j.appender.console.layout
=
org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern
=
%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
# Settings to quiet third party logs that are too verbose
log4j.logger.org.eclipse.jetty
=
WARN
log4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle
=
ERROR
log4j.logger.org.apache.spark.repl.SparkIMain$
exprTyper
=
INFO
log4j.logger.org.apache.spark.repl.SparkILoop$
SparkILoopInterpreter
=
INFO
src/test/scala/org/apache/spark/ml/iforest/IForestSuite.scala
View file @
4848fe69
...
...
@@ -107,7 +107,6 @@ class IForestSuite extends SparkFunSuite with MLlibTestSparkContext with Default
val
summary
=
model
.
summary
val
anomalies
=
summary
.
anomalies
.
collect
assert
(
anomalies
.
length
===
10
)
// TODO In Spark 2.3.x, function approxQuantile seems to be changed, numAnomalies might be not accurate.
assert
(
summary
.
numAnomalies
===
2
)
val
transformed
=
model
.
transform
(
dataset
)
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment