Skip to content

GitLab

Commit 1cbf5abf authored by Christian Müller's avatar Christian Müller
Browse files

fix tests

parent 9d5cd1c4
Hide whitespace changes
Inline Side-by-side
Showing with 362 additions and 161 deletions
examples/omitting/conference.spec
View file @ 1cbf5abf
......@@ -7,17 +7,17 @@ forallmay x:A,p:P
forall x:A,p:P,r:R
(Assign(x,p) ∧ O(x,p,r)) → Review += (x,p,r)
loop {
forall xa:A,xb:A,p:P,r:R (Assign(xa,p) ∧ Review(xb,p,r)) → Read += (xa,p,r)
forall x:A,y:A,p:P,r:R (Assign(x,p) ∧ Review(y,p,r)) → Read += (x,r)
forallmay x:A,p:P,r:R (Assign(x,p)) → Review += (x,p,r)
}
Declassify
O(x:A,p:P,r:R): ¬ Conf(xat:A,p:P)
O(x:A,p:P,r:R): ¬ Conf(xt:A,p:P)
Target
Read(xat:A, pt:P, rt:R)
Read(xt:A, rt:R)
Causality
......
examples/omitting/conference_fixed.spec 0 → 100644
View file @ 1cbf5abf
Workflow
forallmay x:A,p:P
True → Conf += (x,p)
forallmay x:A,p:P
!Conf(x,p) → Assign += (x,p)
forall x:A,y:A,p:P,q:P
(Assign(x,p) ∧ Assign(y,p) ∧ Conf(x,q) ∧ !Conf(y,q)) → Assign -= (y,p)
forall x:A,p:P,r:R
(Assign(x,p) ∧ O(x,p,r)) → Review += (x,p,r)
loop {
forall x:A,y:A,p:P,r:R (Assign(x,p) ∧ Review(y,p,r)) → Read += (x,r)
forallmay x:A,p:P,r:R (Assign(x,p)) → Review += (x,p,r)
}
Declassify
O(x:A,p:P,r:R): ¬ Conf(xt:A,p:P)
Target
Read(xt:A, rt:R)
Causality
a:A
examples/omitting/conference_linear_safe.spec → examples/omitting/conference_linear_fixed.spec
View file @ 1cbf5abf
......@@ -5,7 +5,7 @@ forallmay x:X,p:P
forallmay x:X,p:P
!Conf(x,p) -> Assign += (x,p)
forall x:X,y:X,p:P,q:P
(Conf(x,p) ∧ ¬ Conf(y,p) ∧ Assign(x,p) ∧ Assign(y,q)) -> Assign -= (y,q)
(Conf(x,p) ∧ ¬ Conf(y,p) ∧ Assign(x,q) ∧ Assign(y,q)) -> Assign -= (y,q)
forall x:X,p:P,r:R
(Assign(x,p) ∧ O(x,p,r)) -> Read += (x,p,r)
forallmay y:X,x:X,p:P
......
src/main/scala/de/tum/workflows/Encoding.scala
View file @ 1cbf5abf
......@@ -197,7 +197,7 @@ object Encoding extends LazyLogging {
val MAXINVLENGTH = 1200
def toDot(g: WorkflowGraph)(labels:Map[WorkflowGraph#NodeT, String], edges:Set[g.EdgeT]) = {
def toDot(g: WorkflowGraph)(labels:Map[Encoding.WorkflowGraph#NodeT, String], edges:Set[g.EdgeT]) = {
val root = DotRootGraph(
directed = true,
id = Some("Invariant Labelling"))
......
src/main/scala/de/tum/workflows/toz3/InvariantChecker.scala
View file @ 1cbf5abf
......@@ -95,7 +95,7 @@ object InvariantChecker extends LazyLogging {
(checkOnZ3(f.simplify()), z3simpednewinv)
}
def checkInvariantFP(spec: Spec, inv: Formula, properties:InvProperties = InvProperties.DEFAULT) = {
def checkInvariantFPLabelling(spec: Spec, inv: Formula, properties:InvProperties) = {
val graph = Encoding.toGraph(spec.w)
type Node = graph.NodeT
type Edge = graph.EdgeT
......@@ -171,13 +171,19 @@ object InvariantChecker extends LazyLogging {
// create labelling - FIXME: maybe use inv only for last
val labels = (for (n <- graph.nodes) yield { n -> inv }).toMap
val (result, afterlabels, proven) = checkInvariantRec(labels, List(Set()))
val (time, (result, labellings, proven)) = Utils.time { checkInvariantRec(labels, List(Set())) }
val dot1 = afterlabels.zip(proven).reverse
val dot2 = dot1.map((tup) => Encoding.toDot(graph)(tup._1.map(t => (t._1, t._2.pretty())), tup._2))
val dot1 = labellings.zip(proven).reverse
val dot2 = dot1.map((tup) => Encoding.toDot(graph)(tup._1.map(t => (t._1, {
if (t._2.opsize() <= 5000) t._2.pretty() else "TOO BIG"
})), tup._2))
(result, dot2)
(result, graph, labellings.reverse, proven.reverse, dot2, time)
}
def checkInvariantFP(spec: Spec, inv: Formula, properties:InvProperties = InvProperties.DEFAULT) = {
val (result, graph, afterlabels, proven, dot, time) = checkInvariantFPLabelling(spec, inv, properties)
(result, dot)
}
def checkInvariantOnce(spec: Spec, inv: Formula, properties: InvProperties) = {
......
src/main/scala/de/tum/workflows/toz3/InvariantGenerator.scala
View file @ 1cbf5abf
......@@ -62,8 +62,9 @@ object InvariantGenerator {
// }
def invariantAllEqual(spec: Spec) = {
And.make(for (r <- spec.w.sig.preds.toList) yield {
Forall(r.params, genEq(r, r.params))
And.make(for (r <- spec.w.sig.preds.toList if r.params.head.typ == spec.target.params.head.typ) yield {
val params = r.params.map(v => Var(v.name + "t", v.typ))
Forall(params, genEq(r, params))
})
}
}
\ No newline at end of file
src/test/scala/de/tum/workflows/ltl/tests/InvariantEasychairTest.scala deleted 100644 → 0
View file @ 9d5cd1c4
package de.tum.workflows.ltl.tests
import org.scalatest.FlatSpec
import org.scalatest.Matchers._
import org.scalatest.Inspectors._
import de.tum.workflows.foltl.FOLTL._
import de.tum.workflows.Implicits._
import de.tum.workflows.blocks._
import de.tum.workflows.Preconditions
import de.tum.workflows.foltl.FormulaFunctions
import de.tum.workflows.ExampleWorkflows
import de.tum.workflows.toz3.InvariantChecker
import de.tum.workflows.Encoding._
import de.tum.workflows.Utils
import de.tum.workflows.Encoding
import de.tum.workflows.ExampleWorkflows
import de.tum.workflows.toz3.InvariantChecker
import de.tum.workflows.toz3.InvariantGenerator
import de.tum.workflows.tests.TestUtils._
class InvariantEasychairTest extends FlatSpec {
"InvariantChecker" should "prove nonomitting/conference safe" in {
val name = "nonomitting/conference"
val x = Var("xt", "A")
val p = Var("pt","P")
val inv = Forall(List(x, p), genEq("Conf", List(x, p)))
assert(checkSafeStubborn(name, inv))
}
it should "prove nonomitting/conference_linear with declass safe" in {
val name = "nonomitting/conference_linear"
val x = Var("xt", "X")
val y = Var("yt","X")
val p = Var("pt","P")
val inv = Forall(List(x, p), genEq("Comm", List(x, y, p)))
assert(checkSafeStubborn(name, inv))
}
it should "prove tests/conferenceNoOracle safe" in {
val name = "tests/conferenceNoOracle"
val xa = Var("xat","A")
val xb = Var("xbt","A")
val p = Var("pt","P")
val inv = Forall(List(xa, xb, p), genEq("Read", List(xa, xb, p)))
assert(checkSafeStubborn(name, inv))
}
it should "prove tests/omittingConferenceNoOracle safe" in {
val name = "tests/omittingConferenceNoOracle"
val xa = Var("xat","A")
val xb = Var("xbt","A")
val inv = Forall(List(xa, xb), genEq("Read", List(xa, xb)))
assert(checkSafeStubborn(name, inv))
}
it should "fail on oracly nonomitting/conference without declass" in {
val name = "nonomitting/conferenceNoDeclass"
val xa = Var("xat","A")
val xb = Var("xbt","A")
val p = Var("pt", "P")
val r = Var("rt", "R")
val inv = Forall(List(xa, xb, p, r), genEq("Read", List(xa, xb, p, r)))
assert(!checkSafeStubborn(name, inv))
}
it should "prove oracly nonomitting/conference with declass" in {
val name = "nonomitting/conference"
val xa = Var("xat","A")
val xb = Var("xbt","A")
val p = Var("pt", "P")
val r = Var("rt", "R")
val inv = Forall(List(xa, xb, p, r), genEq("Read", List(xa, xb, p, r)))
assert(checkSafeStubborn(name, inv))
}
}
\ No newline at end of file
src/test/scala/de/tum/workflows/ltl/tests/InvariantCausalFilesTest.scala → src/test/scala/de/tum/workflows/ltl/tests/InvariantSmallCausalFiltesTest.scala
View file @ 1cbf5abf
......@@ -18,7 +18,7 @@ import de.tum.workflows.Encoding
import de.tum.workflows.tests.TestUtils._
import de.tum.workflows.foltl.Properties
class InvariantCausalFilesTest extends FlatSpec {
class InvariantSmallCausalFilesTest extends FlatSpec {
"InvariantChecker" should "prove simple causal may things safe" in {
val name = "tests/simpleChoiceOmittingNoOracle"
......@@ -66,62 +66,4 @@ class InvariantCausalFilesTest extends FlatSpec {
val inv = Forall(List(xt, yt, pt, rt), genEq("Read", List(yt,pt,rt)))
assert(!checkSafeCausal(name, inv))
}
it should "fail to prove conference_linear" in {
val name = "nonomitting/conference_linear"
val xt = Var("xt","X")
val yt = Var("yt","X")
val pt = Var("pt","P")
val inv = Forall(List(xt, pt, yt), genEq("Comm", List(xt, yt, pt)))
assert(!checkSafeCausal(name, inv))
}
it should "fail to prove conference_linear with elim" in {
val name = "nonomitting/conference_linear"
val xt = Var("xt","X")
val yt = Var("yt","X")
val pt = Var("pt","P")
val inv = Forall(List(xt, pt, yt), genEq("Comm", List(xt, yt, pt)))
assert(!checkSafeCausalElim(name, inv))
}
it should "fail to prove nonomitting/conference" in {
val name = "nonomitting/conference"
val xt = Var("xat","X")
val yt = Var("xbt","X")
val pt = Var("pt","P")
val rt = Var("rt","R")
val inv = Forall(List(xt, pt, yt, rt), genEq("Read", List(xt, yt, pt, rt)))
assert(!checkSafeCausal(name, inv))
}
// HEAP SPACE
it should "fail to prove nonomitting/conference with elim" ignore {
val name = "nonomitting/conference"
val xt = Var("xat","X")
val yt = Var("xbt","X")
val pt = Var("pt","P")
val rt = Var("rt","R")
val inv = Forall(List(xt, pt, yt, rt), genEq("Read", List(xt, yt, pt, rt)))
assert(!checkSafeCausalElim(name, inv))
}
it should "fail to prove omitting/conference" in {
val name = "omitting/conference"
val xt = Var("xat","X")
val pt = Var("pt","P")
val rt = Var("rt","R")
val inv = Forall(List(xt, pt), genEq("Read", List(xt, pt, rt)))
assert(!checkSafeCausal(name, inv))
}
it should "fail to prove omitting/conference with elim" in {
val name = "omitting/conference"
val xt = Var("xat","X")
val pt = Var("pt","P")
val rt = Var("rt","R")
val inv = Forall(List(xt, pt), genEq("Read", List(xt, pt, rt)))
assert(!checkSafeCausalElim(name, inv))
}
}
\ No newline at end of file
src/test/scala/de/tum/workflows/ltl/tests/TestUtils.scala
View file @ 1cbf5abf
......@@ -8,28 +8,59 @@ import de.tum.workflows.toz3.InvProperties
object TestUtils {
def check(name:String, desc:String, inv:Formula, properties:InvProperties) = {
val model = if (properties.stubborn) "stubborn" else "causal"
val filenames = s"${name}_${model}_${desc}"
val spec = ExampleWorkflows.parseExample(name).get
// do not blow up the formula with auxilliary elimination
val (time, (res, dot)) = Utils.time {
InvariantChecker.checkInvariantFP(spec, inv, properties)
}
val (res, graph, labelling, provens, dot, time) =
InvariantChecker.checkInvariantFPLabelling(spec, inv, properties)
for ((s, i) <- dot.zipWithIndex) {
Utils.write(s"${name}_${desc}_${i}.dot", s)
Utils.write(s"${filenames}_${i}.dot", s)
}
val strengthenings = for (List(a, b) <- labelling.sliding(2) if (a != b)) yield {
true
}
Utils.write(s"${name}_${desc}.metrics", s"Took $time ms to terminate.\nDid ${dot.size} proof steps.")
val wfsize = graph.edges.size - 1
val invsizes = labelling.last.map(_._2.opsize())
val maxsize = invsizes.max
val avgsize = invsizes.sum / invsizes.size
Utils.write(s"${filenames}.metrics",
s"""Name: $name
Description: $desc
Invariant: $inv
Model: $model
Result: ${if (!res) "not " else ""}inductive
WF size: $wfsize
Time: $time ms
Proof steps: ${dot.size}
Strengthenings: ${strengthenings.size}
Largest Inv: ${maxsize}
Average Inv: ${avgsize}""")
res
}
def checkSafeStubborn(name: String, inv: Formula) = {
check(name, "stubborn", inv, InvProperties(true, true))
def checkSafeStubborn(name: String, inv: Formula):Boolean = {
checkSafeStubborn(name, "", inv)
}
def checkSafeStubbornNoElim(name: String, inv: Formula):Boolean = {
check(name, "noelim", inv, InvProperties(true, false))
}
def checkSafeStubborn(name: String, desc:String, inv: Formula) = {
check(name, desc, inv, InvProperties(true, true))
}
def checkSafeCausal(name: String, inv: Formula):Boolean = {
checkSafeCausal(name, "", inv)
}
def checkSafeCausal(name: String, inv: Formula) = {
check(name, "causal", inv, InvProperties(false, false))
def checkSafeCausal(name: String, desc:String, inv: Formula) = {
check(name, desc, inv, InvProperties(false, false))
}
def checkSafeCausalElim(name: String, inv: Formula) = {
check(name, "causal_elim", inv, InvProperties(false, true))
check(name, "w/ elim", inv, InvProperties(false, true))
}
def genEq(name: String, params: List[Var]) = {
......
src/test/scala/de/tum/workflows/ltl/tests/papertests/InvariantCausalFilesTest.scala 0 → 100644
View file @ 1cbf5abf
package de.tum.workflows.ltl.tests.papertests
import org.scalatest.FlatSpec
import org.scalatest.Matchers._
import org.scalatest.Inspectors._
import de.tum.workflows.foltl.FOLTL._
import de.tum.workflows.Implicits._
import de.tum.workflows.blocks._
import de.tum.workflows.Preconditions
import de.tum.workflows.foltl.FormulaFunctions
import de.tum.workflows.ExampleWorkflows
import de.tum.workflows.toz3.InvariantChecker
import de.tum.workflows.Encoding._
import de.tum.workflows.Utils
import de.tum.workflows.Encoding
import de.tum.workflows.tests.TestUtils._
import de.tum.workflows.foltl.Properties
import de.tum.workflows.toz3.InvariantGenerator
class InvariantCausalFilesTest extends FlatSpec {
// it should "fail to prove conference_linear" ignore {
// val name = "nonomitting/conference_linear"
// val xt = Var("xt","X")
// val yt = Var("yt","X")
// val pt = Var("pt","P")
// val inv = Forall(List(xt, pt, yt), genEq("Comm", List(xt, yt, pt)))
// assert(!checkSafeCausal(name, inv))
// }
//
// it should "fail to prove conference_linear alleq" in {
// val name = "nonomitting/conference_linear"
// val inv = InvariantGenerator.invariantAllEqual(ExampleWorkflows.parseExample(name).get)
// assert(!checkSafeCausal(name, "alleq", inv))
// }
//
// // HEAP SPACE
// it should "fail to prove conference_linear with elim" ignore {
// val name = "nonomitting/conference_linear"
// val xt = Var("xt","X")
// val yt = Var("yt","X")
// val pt = Var("pt","P")
// val inv = Forall(List(xt, pt, yt), genEq("Comm", List(xt, yt, pt)))
// assert(!checkSafeCausalElim(name, inv))
// }
//
// // takes about 30s
// it should "fail to prove nonomitting/conference" in {
// val name = "nonomitting/conference"
// val xt = Var("xat","X")
// val yt = Var("xbt","X")
// val pt = Var("pt","P")
// val rt = Var("rt","R")
// val inv = Forall(List(xt, yt, pt, rt), genEq("Read", List(xt, yt, pt, rt)))
// assert(!checkSafeCausal(name, inv))
// }
//
// // takes forever (up to 8s Z3 calls :(, simplification gets rid of 11mio chars), but passes
// it should "fail to prove nonomitting/conference alleq" ignore {
// val name = "nonomitting/conference"
// val inv = InvariantGenerator.invariantAllEqual(ExampleWorkflows.parseExample(name).get)
// assert(!checkSafeCausal(name, "alleq", inv))
// }
//
// // HEAP SPACE
// it should "fail to prove nonomitting/conference with elim" ignore {
// val name = "nonomitting/conference"
// val xt = Var("xat","X")
// val yt = Var("xbt","X")
// val pt = Var("pt","P")
// val rt = Var("rt","R")
// val inv = Forall(List(xt, pt, yt, rt), genEq("Read", List(xt, yt, pt, rt)))
// assert(!checkSafeCausalElim(name, inv))
// }
//
// // about 20 secs
// it should "fail to prove omitting/conference" in {
// val name = "omitting/conference"
// val xt = Var("xat","X")
// val pt = Var("pt","P")
// val rt = Var("rt","R")
// val inv = Forall(List(xt, pt), genEq("Read", List(xt, pt, rt)))
// assert(!checkSafeCausal(name, inv))
// }
//
// // about 60 seconds
// it should "fail to prove omitting/conference alleq" in {
// val name = "omitting/conference"
// val inv = InvariantGenerator.invariantAllEqual(ExampleWorkflows.parseExample(name).get)
// assert(!checkSafeCausal(name, "alleq", inv))
// }
// it should "prove omitting/conference_fixed" in {
// val name = "omitting/conference_fixed"
// val xt = Var("xt","A")
// val pt = Var("rt","R")
// val inv = Forall(List(xt, pt), genEq("Read", List(xt, pt)))
// assert(checkSafeCausal(name, inv))
// }
//
// it should "prove omitting/conference_fixed alleq" ignore {
// val name = "omitting/conference_fixed"
// val inv = InvariantGenerator.invariantAllEqual(ExampleWorkflows.parseExample(name).get)
// assert(checkSafeCausal(name, "alleq", inv))
// }
//
// it should "prove conference_linear_fixed" in {
// val name = "omitting/conference_linear_fixed"
// val xt = Var("xt","X")
// val yt = Var("yt","X")
// val pt = Var("pt","P")
// val inv = Forall(List(xt, pt, yt), genEq("Comm", List(xt, yt, pt)))
// assert(checkSafeCausal(name, inv))
// }
//
// // maybe errors?
// it should "prove conference_linear_fixed alleq" ignore {
// val name = "omitting/conference_linear_fixed"
// val inv = InvariantGenerator.invariantAllEqual(ExampleWorkflows.parseExample(name).get)
// assert(checkSafeCausal(name, "alleq", inv))
// }
//
// // HEAP SPACE
// it should "fail to prove omitting/conference with elim" ignore {
// val name = "omitting/conference"
// val xt = Var("xat","X")
// val pt = Var("pt","P")
// val rt = Var("rt","R")
// val inv = Forall(List(xt, pt), genEq("Read", List(xt, pt, rt)))
// assert(!checkSafeCausalElim(name, inv))
// }
}
\ No newline at end of file
src/test/scala/de/tum/workflows/ltl/tests/papertests/InvariantEasychairTest.scala 0 → 100644
View file @ 1cbf5abf
package de.tum.workflows.ltl.tests
import org.scalatest.FlatSpec
import org.scalatest.Matchers._
import org.scalatest.Inspectors._
import de.tum.workflows.foltl.FOLTL._
import de.tum.workflows.Implicits._
import de.tum.workflows.blocks._
import de.tum.workflows.Preconditions
import de.tum.workflows.foltl.FormulaFunctions
import de.tum.workflows.ExampleWorkflows
import de.tum.workflows.toz3.InvariantChecker
import de.tum.workflows.Encoding._
import de.tum.workflows.Utils
import de.tum.workflows.Encoding
import de.tum.workflows.ExampleWorkflows
import de.tum.workflows.toz3.InvariantChecker
import de.tum.workflows.toz3.InvariantGenerator
import de.tum.workflows.tests.TestUtils._
class InvariantEasychairTest extends FlatSpec {
// "InvariantChecker" should "prove trivial nonomitting/conference safe" in {
// val name = "nonomitting/conference"
// val x = Var("xt", "A")
// val p = Var("pt","P")
// val inv = Forall(List(x, p), genEq("Conf", List(x, p)))
// assert(checkSafeStubborn(name, "onlyconf", inv))
// }
//
// it should "prove nonomitting/conference_linear with declass safe" ignore {
// val name = "nonomitting/conference_linear"
// val x = Var("xt", "X")
// val y = Var("yt","X")
// val p = Var("pt","P")
// val inv = Forall(List(x, p), genEq("Comm", List(x, y, p)))
// assert(checkSafeStubborn(name, inv))
// }
//
// it should "prove stubborn conference_linear alleq" in {
// val name = "nonomitting/conference_linear"
// val inv = InvariantGenerator.invariantAllEqual(ExampleWorkflows.parseExample(name).get)
// assert(checkSafeStubborn(name, "alleq", inv))
// }
it should "prove omitting/conference_fixed" in {
val name = "omitting/conference_fixed"
val xt = Var("xt", "A")
val pt = Var("rt", "R")
val inv = Forall(List(xt, pt), genEq("Read", List(xt, pt)))
assert(checkSafeStubbornNoElim(name, inv))
}
it should "prove omitting/conference_fixed alleq" ignore {
val name = "omitting/conference_fixed"
val inv = InvariantGenerator.invariantAllEqual(ExampleWorkflows.parseExample(name).get)
assert(checkSafeStubborn(name, "alleq", inv))
}
it should "prove stubborn conference" in {
val name = "omitting/conference"
val x = Var("xt", "A")
val r = Var("rt", "R")
val inv = Forall(List(x, r), genEq("Read", List(x, r)))
assert(checkSafeStubbornNoElim(name, inv))
}
it should "prove stubborn conference alleq" ignore {
val name = "omitting/conference"
val inv = InvariantGenerator.invariantAllEqual(ExampleWorkflows.parseExample(name).get)
assert(checkSafeStubborn(name, "alleq", inv))
}
it should "prove stubborn conference_linear_fixed" in {
val name = "omitting/conference_linear_fixed"
val x = Var("xt", "X")
val y = Var("yt", "X")
val p = Var("pt", "P")
val inv = Forall(List(x, y, p), genEq("Comm", List(x, y, p)))
assert(checkSafeStubbornNoElim(name, inv))
}
// it should "prove stubborn conference_linear_fixed alleq" ignore {
// val name = "omitting/conference_linear_fixed"
// val inv = InvariantGenerator.invariantAllEqual(ExampleWorkflows.parseExample(name).get)
// assert(checkSafeStubborn(name, "alleq", inv))
// }
//
// it should "prove tests/conferenceNoOracle safe" ignore {
// val name = "tests/conferenceNoOracle"
// val xa = Var("xat","A")
// val xb = Var("xbt","A")
// val p = Var("pt","P")
// val inv = Forall(List(xa, xb, p), genEq("Read", List(xa, xb, p)))
// assert(checkSafeStubborn(name, inv))
// }
//