parse_env_file discard utf8 bom (#2638)

* parse_env_file discard utf8 bom

* powershell 5 may add the BOM even when explicitly using utf8

* add test + apply to GITHUB_PATH as well

* fix it

* fix powershel 5 syntax

* misc

* fixup

* fix wrong subaction

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

pkg/container/parse_env_file.go

@@ -25,8 +25,16 @@ func parseEnvFile(e Container, srcPath string, env *map[string]string) common.Ex
 			return err
 		}
 		s := bufio.NewScanner(reader)
+		firstLine := true
 		for s.Scan() {
 			line := s.Text()
+			if firstLine {
+				firstLine = false
+				// skip utf8 bom, powershell 5 legacy uses it for utf8
+				if len(line) >= 3 && line[0] == 239 && line[1] == 187 && line[2] == 191 {
+					line = line[3:]
+				}
+			}
 			singleLineEnv := strings.Index(line, "=")
 			multiLineEnv := strings.Index(line, "<<")
 			if singleLineEnv != -1 && (multiLineEnv == -1 || singleLineEnv < multiLineEnv) {

pkg/runner/run_context.go

@@ -520,8 +520,16 @@ func (rc *RunContext) UpdateExtraPath(ctx context.Context, githubEnvPath string)
 		return err
 	}
 	s := bufio.NewScanner(reader)
+	firstLine := true
 	for s.Scan() {
 		line := s.Text()
+		if firstLine {
+			firstLine = false
+			// skip utf8 bom, powershell 5 legacy uses it for utf8
+			if len(line) >= 3 && line[0] == 239 && line[1] == 187 && line[2] == 191 {
+				line = line[3:]
+			}
+		}
 		if len(line) > 0 {
 			rc.addPath(ctx, line)
 		}

pkg/runner/runner_test.go

@@ -429,6 +429,8 @@ func TestRunEventHostEnvironment(t *testing.T) {
 		tables = append(tables, []TestJobFileInfo{
 			{workdir, "windows-prepend-path", "push", "", platforms, secrets},
 			{workdir, "windows-add-env", "push", "", platforms, secrets},
+			{workdir, "windows-prepend-path-powershell-5", "push", "", platforms, secrets},
+			{workdir, "windows-add-env-powershell-5", "push", "", platforms, secrets},
 			{workdir, "windows-shell-cmd", "push", "", platforms, secrets},
 		}...)
 	} else {

pkg/runner/testdata/windows-add-env-powershell-5/action/action.yml

@@ -0,0 +1,7 @@
+runs:
+  using: composite
+  steps:
+  - run: |
+      echo $env:GITHUB_ENV
+      echo "kEy=n/a" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+    shell: powershell

pkg/runner/testdata/windows-add-env-powershell-5/push.yml

@@ -0,0 +1,48 @@
+on:
+  push:
+jobs:
+  test:
+    runs-on: windows-latest
+    steps:
+    - run: |
+        echo $env:GITHUB_ENV
+        echo "key=val" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        echo "key2<<EOF" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        echo "line1" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        echo "line2" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        echo "EOF" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
+        cat $env:GITHUB_ENV
+      shell: powershell
+    - run: |
+        ls env:
+        if($env:key -ne 'val') {
+          echo "Unexpected value for `$env:key: $env:key"
+          exit 1
+        }
+        if($env:key2 -ne "line1`nline2") {
+          echo "Unexpected value for `$env:key2: $env:key2"
+          exit 1
+        }
+      shell: powershell
+    - run: |
+        echo $env:GITHUB_ENV
+        echo "KEY=test" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8
+        # Defect missing append, test is broken!!!
+        echo "Key=expected" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8
+      shell: powershell
+    - name: Assert GITHUB_ENV is merged case insensitive
+      run: exit 1
+      if: env.KEY != 'expected' || env.Key != 'expected' || env.key != 'expected'
+      shell: powershell
+    - name: Assert step env is merged case insensitive
+      run: exit 1
+      if: env.KEY != 'n/a' || env.Key != 'n/a' || env.key != 'n/a'
+      env:
+        KeY: 'n/a'
+      shell: powershell
+    - uses: actions/checkout@v3
+    - uses: ./windows-add-env-powershell-5/action
+    - name: Assert composite env is merged case insensitive
+      run: exit 1
+      if: env.KEY != 'n/a' || env.Key != 'n/a' || env.key != 'n/a'
+      shell: powershell

pkg/runner/testdata/windows-add-env/push.yml

@@ -6,7 +6,7 @@ jobs:
     steps:
     - run: |
         echo $env:GITHUB_ENV
-        echo "key=val" > $env:GITHUB_ENV
+        echo "key=val" >> $env:GITHUB_ENV
         echo "key2<<EOF" >> $env:GITHUB_ENV
         echo "line1" >> $env:GITHUB_ENV
         echo "line2" >> $env:GITHUB_ENV
@@ -24,7 +24,9 @@ jobs:
         }
     - run: |
         echo $env:GITHUB_ENV
+        # Defect missing append
         echo "KEY=test" > $env:GITHUB_ENV
+        # Defect missing append, test is broken!!!
         echo "Key=expected" > $env:GITHUB_ENV
     - name: Assert GITHUB_ENV is merged case insensitive
       run: exit 1

pkg/runner/testdata/windows-prepend-path-powershell-5/push.yml

@@ -0,0 +1,34 @@
+on:
+  push:
+defaults:
+  run:
+    shell: powershell
+jobs:
+  test:
+    runs-on: windows-latest
+    steps:
+    - run: |
+        mkdir build
+        echo '@echo off' | Out-File -FilePath build/test.cmd -Encoding utf8
+        echo 'echo Hi' | Out-File -FilePath build/test.cmd -Encoding utf8 -Append
+        mkdir build2
+        echo '@echo off' | Out-File -FilePath build/test2.cmd -Encoding utf8
+        echo 'echo test2' | Out-File -FilePath build/test2.cmd -Encoding utf8 -Append
+    - run: |
+        echo '${{ tojson(runner) }}'
+        ls
+        echo '${{ github.workspace }}'
+      working-directory: ${{ github.workspace }}\build
+    - run: |
+        echo $env:GITHUB_PATH
+        echo '${{ github.workspace }}\build' | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+        cat $env:GITHUB_PATH
+    - run: |
+        echo $env:PATH
+        test
+    - run: |
+        echo "PATH=$env:PATH;${{ github.workspace }}\build2" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+    - run: |
+        echo $env:PATH
+        test
+        test2