Commit e330c5ce authored by Drew's avatar Drew

Add CoreTest testing suite

Add notEqual definition to functions
parent ef6b168f
Pipeline #1992 passed with stages
......@@ -15,6 +15,8 @@ osx:
osxcorepackage:
stage: package
script:
- atbuild coretest-gen
- atbuild coretest --toolchain xcode
- atbuild core-framework --toolchain xcode
tags:
......
......@@ -37,18 +37,37 @@ public class CarolineCoreEngine : CarolineEngine {
print("test \(test) started.")
test.test()
print("test \(test) ended.")
var state = CarolineState(test: test)
if state.outcome == .Initial { state.outcome = .Passed; state.commit() }
}
//count failures
print("Test report:")
print("-----------------------")
var failures = 0
var skipped = 0
for test in tests {
let state = CarolineState(test: test)
if state.outcome == .Failed { failures += 1}
if state.outcome == .Failed && !test.expectFailure {
failures += 1
print("\(test) failed")
}
if test.expectFailure && state.outcome != .XFailed {
failures += 1
print("\(test) didn't fail as expected – \(state.outcome)")
}
if state.outcome == .Skipped { skipped += 1}
}
print("-----------------------")
srand(UInt32(time(nil)))
if failures == 0 {
print(passSnark[Int(rand()) % passSnark.count])
}
else {
print(failSnark[Int(rand()) % failSnark.count])
}
print("Test execution finished with \(tests.count) tests and \(failures) failures.")
print ("Skipped \(skipped) tests.")
print("Test execution finished with \(tests.count) tests, \(failures) failures, \(skipped) skipped.")
if failures > 0 { return false }
return true
}
......
......@@ -16,9 +16,13 @@
public protocol CarolineTest: class {
///Implement the test behavior here
func test()
///The test should be skipped
var skip: Bool { get }
///Fail on pass, pass on failure
var expectFailure: Bool { get }
}
public extension CarolineTest {
var skip: Bool { return false }
var expectFailure: Bool { return false }
}
......@@ -13,23 +13,57 @@
// limitations under the License.
//MARK: Functions
#if swift(>=3.0)
private func compare<T: Sequence where T.Iterator.Element: Equatable>(_ a: T, _ b: T) -> Bool {
var iteratorB = b.makeIterator()
for itemA in a {
let itemB = iteratorB.next()
if itemB != itemA {
return false
}
}
//We've concluded iterating A. Did B have more elements?
if let _ = iteratorB.next() { return false }
return true
}
#else
private func compare<T: SequenceType where T.Generator.Element: Equatable>(_ a: T, _ b: T) -> Bool {
var iteratorB = b.generate()
for itemA in a {
let itemB = iteratorB.next()
if itemB != itemA {
return false
}
}
//We've concluded iterating A. Did B have more elements?
if let _ = iteratorB.next() { return false }
return true
}
#endif
extension CarolineTest {
public final func fail(_ message: String = "Test failed", file: String = #file, line: Int = #line) {
var state = CarolineState(test: self)
if self.expectFailure {
state.outcome = .XFailed
}
else {
state.outcome = .Failed
print("Test failed at \(file):\(line) - \(message)")
}
state.commit()
}
#if swift(>=3.0)
public final func assert(_ condition: @autoclosure () -> Bool, _ message: @autoclosure () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
if !condition() {
var state = CarolineState(test: self)
state.outcome = .Failed
state.commit()
print("Test failed at \(file):\(line) - \(message())")
self.fail(message(), file: file, line: line)
}
}
#else
public final func assert(@autoclosure _ condition: () -> Bool, @autoclosure _ message: () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
if !condition() {
var state = CarolineState(test: self)
state.outcome = .Failed
state.commit()
print("Test failed at \(file):\(line) - \(message())")
self.fail(message(), file: file, line: line)
}
}
#endif
......@@ -39,45 +73,43 @@ extension CarolineTest {
#if swift(>=3.0)
public final func assert<T: Equatable>(_ a: T, equals b: T, _ message: @autoclosure () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
if a != b {
var state = CarolineState(test: self)
state.outcome = .Failed
state.commit()
print("Test failed at \(file):\(line) - \(message()); \(a) != \(b)")
self.fail("\(message()) - \(a) != \(b)")
}
}
public final func assert<T: Equatable>(_ a: T, notEqual b: T, _ message: @autoclosure () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
if a == b {
self.fail("\(message()) - \(a) == \(b)")
}
}
public final func assert<T:Sequence where T.Iterator.Element: Equatable>(_ a: T, equals b: T, _ message: @autoclosure () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
var iteratorB = b.makeIterator()
for itemA in a {
let itemB = iteratorB.next()
if itemB != itemA {
var state = CarolineState(test: self)
state.outcome = .Failed
state.commit()
print("Test failed at \(file):\(line) - \(message()); \(a) != \(b)")
return
}
if !compare(a,b) {
self.fail("\(message()) - \(a) != \(b)")
}
}
public final func assert<T:Sequence where T.Iterator.Element: Equatable>(_ a: T, notEqual b: T, _ message: @autoclosure () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
if compare(a,b) {
self.fail("\(message()) - \(a) == \(b)")
}
}
#else
public final func assert<T: Equatable>(_ a: T, equals b: T, @autoclosure _ message: () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
if a != b {
var state = CarolineState(test: self)
state.outcome = .Failed
state.commit()
print("Test failed at \(file):\(line) - \(message()); \(a) != \(b)")
self.fail("\(message()) - \(a) != \(b)")
}
}
public final func assert<T: Equatable>(_ a: T, notEqual b: T, @autoclosure _ message: () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
if a == b {
self.fail("\(message()) - \(a) == \(b)")
}
}
public final func assert<T:SequenceType where T.Generator.Element: Equatable>(_ a: T, equals b: T, @autoclosure _ message: () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
var iteratorB = b.generate()
for itemA in a {
let itemB = iteratorB.next()
if itemB != itemA {
var state = CarolineState(test: self)
state.outcome = .Failed
state.commit()
print("Test failed at \(file):\(line) - \(message()); \(a) != \(b)")
return
}
if !compare(a,b) {
self.fail("\(message()) - \(a) != \(b)")
}
}
public final func assert<T:SequenceType where T.Generator.Element: Equatable>(_ a: T, notEqual b: T, @autoclosure _ message: () -> String = "Assertion failed", file: String = #file, line: Int = #line) {
if compare(a,b) {
self.fail("\(message()) - \(a) == \(b)")
}
}
#endif
......
......@@ -19,6 +19,8 @@ enum Outcome {
case Passed
///The test failed
case Failed
///The test XFailed (failed, but we expected failure)
case XFailed
///The test was skipped
case Skipped
}
\ No newline at end of file
// Copyright (c) 2016 Drew Crawford.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
let passSnark = [
"With a test suite this small, I'm surprised it took you so long.",
"Remember, just because the tests passed doesn't mean you didn't create a horrible, horrible bug.",
"Congratulations on passing an entirely meaningless test suite.",
"The only thing wrong with this code is all the things we didn't test.",
"I'd recommend shutting your laptop. You've used up all your good luck for the day.",
"The good news is, none of the branches we hit ran `rm -rf /`. The bad news is, your code coverage.",
"Congratulations! None of the bugs you thought of are in this code. I wonder how many your users will think of...",
"Look at how much time we spent not finding any new bugs.",
"This test suite is so small I'm surprised you bother to run it at all.",
"These tests are so trivial I don't even know why you're running them."
]
let failSnark = [
"Congratulations. Not on the test.",
"Don't worry, you'll get it next time. Or not. Probably not.",
"Why don't you just revert back to the previous commit?",
"You know, we never had these problems until you started making changes.",
"Insanity is doing the same thing and expecting different results. Try running the tests again and see what happens.",
"You know what they say. Fail early, fail often. Good to see you got the memo.",
"I'm pretty sure in TDD, the tests have to actually *pass* at some point.",
"If I plugged your commit message into StackOverflow, the top result would be better than this code.",
"Look, just forget about this feature. It's clearly beyond you.",
"I would say you'll get it next time. But I know better.",
"Go ahead, push it to master. Break the build. I dare you."
]
\ No newline at end of file
......@@ -41,6 +41,7 @@
:dependencies ["core-package"]
}
;;simpletest is a basic sanity check of the gen/test/runner infrastructure
:simpletest-gen {
:tool "shell"
:script "bin/caroline_static_tool --core tests/SimpleTest/foo.swift > tests/SimpleTest/main.swift"
......@@ -60,8 +61,46 @@
:tool "shell"
:script "bin/SimpleTest"
}
;;coretest tests the full core API
:coretest-gen {
:tool "shell"
:script "bin/caroline_static_tool --core tests/CoreTests/assert.swift tests/CoreTests/fail.swift tests/CoreTests/SimpleEqual.swift tests/CoreTests/SequenceEqual.swift > tests/CoreTests/main.swift"
:dependencies ["statictool"]
}
:coretest-build {
:dependencies ["core" "coretest-gen"]
:tool "atllbuild"
:output-type "executable"
:name "CoreTest"
:sources ["tests/CoreTests/**.swift"]
:link-with ["CarolineCore.a"]
:publish-product true
}
:coretest {
:dependencies ["coretest-build"]
:tool "shell"
:script "bin/CoreTest"
}
;;this alternative coretest tasks should be used with --toolchain xcode
;; you must also manually run coretest-gen prior to these working, as codetest-gen isn't supported on xcode toolchain
:coretest-build-xcode {
:dependencies ["core"]
:tool "atllbuild"
:output-type "executable"
:name "CoreTest"
:sources ["tests/CoreTests/**.swift"]
:link-with ["CarolineCore.a"]
:publish-product true
}
:coretest {
:dependencies ["coretest-build-xcode"]
:tool "shell"
:script "bin/CoreTest"
}
:check {
:dependencies ["simpletest"]
:dependencies ["simpletest" "coretest"]
:tool "nop"
}
}
......
......@@ -44,4 +44,6 @@ print(allTestsDeclaration(tests: findTests(fileNames), indentation:0))
print("let engine = CarolineCoreEngine()")
print("engine.testAll(allTests)")
\ No newline at end of file
print("if !engine.testAll(allTests) {")
print(" fatalError(\"Caroline tests failed\")")
print("}")
\ No newline at end of file
// Copyright (c) 2016 Drew Crawford.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import CarolineCore
class SequenceEqual: CarolineTest {
func test() {
self.assert([1], equals: [1])
}
}
class SequenceEqualInverse: CarolineTest {
func test() {
self.assert([1], equals: [2])
}
var expectFailure: Bool { return true }
}
//there are 4 length cases
//A - first list too long
//B - second list too long
//C - first list too short
//D - second list too short
class SequenceEqualLengthA: CarolineTest {
func test() {
self.assert([1,2], equals: [1])
}
var expectFailure: Bool { return true }
}
class SequenceEqualLengthB: CarolineTest {
func test() {
self.assert([1], equals: [1,2])
}
var expectFailure: Bool { return true }
}
class SequenceEqualLengthC: CarolineTest {
func test() {
self.assert([], equals: [1])
}
var expectFailure: Bool { return true }
}
class SequenceEqualLengthD: CarolineTest {
func test() {
self.assert([1], equals: [])
}
var expectFailure: Bool { return true }
}
class SequenceNotEqual: CarolineTest {
func test() {
self.assert([1], notEqual: [2])
}
}
class SequenceNotEqualInverse: CarolineTest {
func test() {
self.assert([1], notEqual: [1])
}
var expectFailure: Bool { return true }
}
class SequenceNotEqualLengthA: CarolineTest {
func test() {
self.assert([1,2], notEqual: [1])
}
}
class SequenceNotEqualLengthB: CarolineTest {
func test() {
self.assert([1], notEqual: [1,2])
}
}
class SequenceNotEqualLengthC: CarolineTest {
func test() {
self.assert([], notEqual: [1])
}
}
class SequenceNotEqualLengthD: CarolineTest {
func test() {
self.assert([1], notEqual: [])
}
}
\ No newline at end of file
// Copyright (c) 2016 Drew Crawford.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import CarolineCore
class SimpleEqual: CarolineTest {
func test() {
self.assert(5, equals: 5)
}
}
class SimpleEqualInverse: CarolineTest {
func test() {
self.assert(5, equals: 6)
}
var expectFailure: Bool { return true }
}
class SimpleNotEqual: CarolineTest {
func test() {
self.assert(5, notEqual: 6)
}
}
class SimpleNotEqualInverse: CarolineTest {
func test() {
self.assert(5, notEqual: 5)
}
var expectFailure: Bool { return true }
}
\ No newline at end of file
// Copyright (c) 2016 Drew Crawford.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import CarolineCore
class AssertTrue: CarolineTest {
func test() {
self.assert(true)
}
}
class AssertFalse: CarolineTest {
func test() {
self.assert(false, "I am a bad programmer")
}
var expectFailure: Bool { return true }
}
\ No newline at end of file
// Copyright (c) 2016 Drew Crawford.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
import CarolineCore
class Fail: CarolineTest {
func test() {
self.fail()
}
var expectFailure: Bool { return true }
}
\ No newline at end of file
// Copyright (c) 2016 Drew Crawford.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is automatically generated by Caroline and should not be edited by hand.
import CarolineCore
let allTests: [CarolineTest] = [
AssertTrue(),
AssertFalse(),
Fail(),
SimpleEqual(),
SimpleEqualInverse(),
SimpleNotEqual(),
SimpleNotEqualInverse(),
SequenceEqual(),
SequenceEqualInverse(),
SequenceEqualLengthA(),
SequenceEqualLengthB(),
SequenceEqualLengthC(),
SequenceEqualLengthD(),
SequenceNotEqual(),
SequenceNotEqualInverse(),
SequenceNotEqualLengthA(),
SequenceNotEqualLengthB(),
SequenceNotEqualLengthC(),
SequenceNotEqualLengthD()
]
let engine = CarolineCoreEngine()
if !engine.testAll(allTests) {
fatalError("Caroline tests failed")
}
......@@ -17,8 +17,8 @@
// This file is automatically generated by Caroline and should not be edited by hand.
import CarolineCore
let allTests: [CarolineTest] = [
Simple(),
Additional()
AssertTrue(),
AssertFalse()
]
let engine = CarolineCoreEngine()
engine.testAll(allTests)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment