fix: tests with validation errors were skipped (#2496)

* fix: tests with validation errors were skipped

* fixup

* [no ci] fix one test

* fix some tests

* fix last test

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

pkg/runner/runner_test.go

@@ -196,11 +196,9 @@ func (j *TestJobFileInfo) runTest(ctx context.Context, t *testing.T, cfg *Config
 	assert.Nil(t, err, j.workflowPath)
 
 	planner, err := model.NewWorkflowPlanner(fullWorkflowPath, true)
-	if err != nil {
+	if j.errorMessage != "" && err != nil {
 		assert.Error(t, err, j.errorMessage)
-	} else {
+	} else if assert.Nil(t, err, fullWorkflowPath) {
-		assert.Nil(t, err, fullWorkflowPath)
-
 		plan, err := planner.PlanEvent(j.eventName)
 		assert.True(t, (err == nil) != (plan == nil), "PlanEvent should return either a plan or an error")
 		if err == nil && plan != nil {

pkg/runner/testdata/actions-environment-and-context-tests/push.yml

@@ -1,5 +1,5 @@
 name: actions-with-environment-and-context-tests
-description: "Actions with environment (env vars) and context (expression) tests"
+run-name: "Actions with environment (env vars) and context (expression) tests"
 on: push
 
 jobs:

pkg/runner/testdata/inputs-via-env-context/push.yml

@@ -4,7 +4,7 @@ jobs:
     runs-on: self-hosted
     steps:
     - uses: actions/checkout@v3
-    - uses: ./inputs-via-env-context
+    - uses: ./inputs-via-env-context/action
       with:
         test-env-input: ${{ env.test-env-input }}
       env:

pkg/runner/testdata/path-handling/push.yml

@@ -18,7 +18,7 @@ jobs:
           fi
 
       - id: action-with-output
-        uses: ./path-handling/
+        uses: ./path-handling/action/
 
       - name: test path (after local action)
         run: |

pkg/runner/testdata/shells/bash/push.yml

@@ -5,7 +5,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: bash
         run: |
           if [[ -n "$BASH" ]]; then
             echo "I'm $BASH!"
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     container: node:16-buster-slim
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: bash
         run: |
           if [[ -n "$BASH" ]]; then
             echo "I'm $BASH!"

pkg/runner/testdata/shells/pwsh/push.yml

@@ -5,14 +5,14 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: pwsh
         run: |
           $PSVersionTable
   check-container:
     runs-on: ubuntu-latest
     container: catthehacker/ubuntu:pwsh-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: pwsh
         run: |
           $PSVersionTable
   check-job-default:

pkg/runner/testdata/shells/python/push.yml

@@ -5,7 +5,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: python
         run: |
           import platform
           print(platform.python_version())
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     container: node:16-buster
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: python
         run: |
           import platform
           print(platform.python_version())

pkg/runner/testdata/shells/sh/push.yml

@@ -5,7 +5,7 @@ jobs:
   check:
     runs-on: ubuntu-latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: sh
         run: |
           if [ -z ${BASH+x} ]; then
             echo "I'm sh!"
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     container: alpine:latest
     steps:
-      - shell: ${{ env.MY_SHELL }}
+      - shell: sh
         run: |
           if [ -z ${BASH+x} ]; then
             echo "I'm sh!"

pkg/runner/testdata/windows-add-env/push.yml

@@ -35,7 +35,7 @@ jobs:
       env:
         KeY: 'n/a'
     - uses: actions/checkout@v3
-    - uses: ./windows-add-env
+    - uses: ./windows-add-env/action
     - name: Assert composite env is merged case insensitive
       run: exit 1
       if: env.KEY != 'n/a' || env.Key != 'n/a' || env.key != 'n/a'