add common helpers

azure-pipelines/templates/test-job.yml

@@ -10,12 +10,9 @@ jobs:
   - task: PowerShell@2
     displayName: Fully cleanup the toolcache directory before testing
     inputs:
-      TargetType: inline
+      targetType: filePath
-      script: |
+      filePath: helpers/clean-toolcache.ps1
-        $NodeToolcachePath = Join-Path -Path $env:AGENT_TOOLSDIRECTORY -ChildPath "node"
+      arguments: -ToolName "node"
-        if (Test-Path $NodeToolcachePath) {
-          Remove-Item -Path $NodeToolcachePath -Recurse -Force
-        }
 
   - task: DownloadPipelineArtifact@2
     inputs:

config/node-manifest-config.json

@@ -0,0 +1,7 @@
+{
+    "regex": "node-\\d+\\.\\d+\\.\\d+-(\\w+)-(x\\d+)",
+    "groups": {
+        "arch": 2,
+        "platform": 1
+    }
+}

helpers/CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to make participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all project spaces, and it also applies when
+an individual is representing the project or its community in public spaces.
+Examples of representing a project or community include using an official
+project e-mail address, posting via an official social media account, or acting
+as an appointed representative at an online or offline event. Representation of
+a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at opensource@github.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

helpers/CONTRIBUTING.md

@@ -0,0 +1,31 @@
+## Contributing
+
+[fork]: https://github.com/actions/versions-package-tools/fork  
+[pr]: https://github.com/actions/versions-package-tools/compare  
+[code-of-conduct]: CODE_OF_CONDUCT.md  
+
+Hi there! We're thrilled that you'd like to contribute to this project. Your help is essential for keeping it great.
+
+Contributions to this project are [released](https://help.github.com/articles/github-terms-of-service/#6-contributions-under-repository-license) to the public under the [MIT](LICENSE.md).
+
+Please note that this project is released with a [Contributor Code of Conduct][code-of-conduct]. By participating in this project you agree to abide by its terms.
+
+## Submitting a pull request
+
+1. [Fork][fork] and clone the repository
+1. Create a new branch: `git checkout -b my-branch-name`
+1. Make your changes
+1. Push to your fork and [submit a pull request][pr]
+1. Make sure that checks in your pull request are green
+
+Here are a few things you can do that will increase the likelihood of your pull request being accepted:
+
+- Please include a summary of the change and which issue is fixed. Also include relevant motivation and context.
+- Follow the style guide for [PowerShell](https://github.com/PoshCode/PowerShellPracticeAndStyle).
+- Write [good commit messages](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html).
+
+## Resources
+
+- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/)
+- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/)
+- [GitHub Help](https://help.github.com)

helpers/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 GitHub
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

helpers/README.md

@@ -0,0 +1,5 @@
+# Common tools for generation of packages in the actions/*-versions repositories
+This repository contains PowerShell modules that are used to generate packages for Actions. The packages are consumed by the images generated through [actions/virtual-environments](https://github.com/actions/virtual-environments) and some of the setup-* Actions
+
+## Contribution
+Contributions are welcome! See [Contributor's Guide](./CONTRIBUTING.md) for more details about contribution process and code structure

helpers/SECURITY.md

@@ -0,0 +1,3 @@
+If you discover a security issue in this repo, please submit it through the [GitHub Security Bug Bounty](https://hackerone.com/github)
+
+Thanks for helping make GitHub Actions safe for everyone.

helpers/clean-toolcache.ps1

@@ -0,0 +1,13 @@
+param (
+    [string] $ToolName
+)
+
+$targetPath = $env:AGENT_TOOLSDIRECTORY
+if ($ToolName) {
+    $targetPath = Join-Path $targetPath $ToolName
+}
+
+if (Test-Path $targetPath) {
+    Get-ChildItem -Path $targetPath -Recurse | Where-Object { $_.LinkType -eq "SymbolicLink" } | ForEach-Object { $_.Delete() }
+    Remove-Item -Path $targetPath -Recurse -Force
+}

helpers/common-helpers.psm1

@@ -0,0 +1,85 @@
+<#
+.SYNOPSIS
+The execute command and print all output to the logs
+#>
+function Execute-Command {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory=$true)][string] $Command
+    )
+
+    Write-Debug "Execute $Command"
+
+    try {
+        Invoke-Expression $Command | ForEach-Object { Write-Host $_ }
+        if ($LASTEXITCODE -ne 0) { throw "Exit code: $LASTEXITCODE"}
+    }
+    catch {
+        $errorMessage = "Error happened during command execution: $Command `n $_"
+        Write-Host $errorMessage
+        if ($ErrorActionPreference -ne "Continue") {
+            # avoid logging Azure DevOps issues in case of $ErrorActionPreference -eq Continue
+            Write-Host "##vso[task.logissue type=error;] $errorMessage"
+        }
+    }
+}
+
+<#
+.SYNOPSIS
+Download file from url and return local path to file
+#>
+function Download-File {
+    param(
+        [Parameter(Mandatory=$true)]
+        [Uri]$Uri,
+        [Parameter(Mandatory=$true)]
+        [String]$OutputFolder
+    )
+
+    $targetFilename = [IO.Path]::GetFileName($Uri)
+    $targetFilepath = Join-Path $OutputFolder $targetFilename
+
+    Write-Debug "Download source from $Uri to $OutFile"
+    try {
+        (New-Object System.Net.WebClient).DownloadFile($Uri, $targetFilepath)
+        return $targetFilepath
+    } catch {
+        Write-Host "Error during downloading file from '$Uri'"
+        "$_"
+        exit 1
+    }    
+}
+
+<#
+.SYNOPSIS
+Generate file that contains the list of all files in particular directory
+#>
+function New-ToolStructureDump {
+    param(
+        [Parameter(Mandatory=$true)]
+        [String]$ToolPath,
+        [Parameter(Mandatory=$true)]
+        [String]$OutputFolder
+    )
+
+    $outputFile = Join-Path $OutputFolder "tools_structure.txt"
+
+    $folderContent = Get-ChildItem -Path $ToolPath -Recurse | Sort-Object | Select-Object -Property FullName, Length
+    $folderContent | ForEach-Object {
+        $relativePath = $_.FullName.Replace($ToolPath, "");
+        return "${relativePath}"
+    } | Out-File -FilePath $outputFile
+}
+
+<#
+.SYNOPSIS
+Check if it is macOS / Ubuntu platform
+#>
+function IsNixPlatform {
+    param(
+        [Parameter(Mandatory=$true)] [ValidateNotNullOrEmpty()]
+        [String]$Platform
+    )
+
+    return ($Platform -match "macos") -or ($Platform -match "ubuntu") -or ($Platform -match "linux")
+}

helpers/nix-helpers.psm1

@@ -20,13 +20,25 @@ function Create-TarArchive {
         [String]$SourceFolder,
         [Parameter(Mandatory=$true)]
         [String]$ArchivePath,
-        [string]$CompressionType = "gz"
+        [string]$CompressionType = "gz",
+        [switch]$DereferenceSymlinks
     )
 
-    $CompressionTypeArgument = If ([string]::IsNullOrWhiteSpace($CompressionType)) { "" } else { "--${CompressionType}" }
+    $arguments = @(
+        "-c",
+        "-f", $ArchivePath,
+        "."
+    )
+    If ($CompressionType) {
+        $arguments += "--${CompressionType}"
+    }
+
+    if ($DereferenceSymlinks) {
+        $arguments += "-h"
+    }
 
     Push-Location $SourceFolder
-    Write-Debug "tar -c $CompressionTypeArgument -f $ArchivePath ."
+    Write-Debug "tar $arguments"
-    tar -c $CompressionTypeArgument -f $ArchivePath .
+    tar @arguments
     Pop-Location
 }

helpers/packages-generation/generate-versions-manifest.ps1

@@ -1,158 +0,0 @@
-<#
-.SYNOPSIS
-Generate versions manifest based on repository releases
-
-.DESCRIPTION
-Versions manifest is needed to find the latest assets for particular version of tool
-.PARAMETER GitHubRepositoryOwner
-Required parameter. The organization which tool repository belongs
-.PARAMETER GitHubRepositoryName
-Optional parameter. The name of tool repository
-.PARAMETER GitHubAccessToken
-Required parameter. PAT Token to overcome GitHub API Rate limit
-.PARAMETER OutputFile
-Required parameter. File "*.json" where generated results will be saved
-.PARAMETER PlatformMapFile
-Optional parameter. Path to the json file with platform map
-Structure example:
-{
-    "macos-1014": [
-        {
-            "platform": "darwin",
-            "platform_version": "10.14"
-        }, ...
-    ], ...
-}
-#>
-
-param (
-    [Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
-    [Parameter(Mandatory)] [string] $GitHubRepositoryName,
-    [Parameter(Mandatory)] [string] $GitHubAccessToken,
-    [Parameter(Mandatory)] [string] $OutputFile,
-    [string] $PlatformMapFile
-)
-
-Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1")
-
-if ($PlatformMapFile -and (Test-Path $PlatformMapFile)) {
-    $PlatformMap = Get-Content $PlatformMapFile -Raw | ConvertFrom-Json -AsHashtable
-} else {
-    $PlatformMap = @{}
-}
-
-function Get-FileNameWithoutExtension {
-    param (
-        [Parameter(Mandatory)][string]$Filename
-    )
-
-    if ($Filename.EndsWith(".tar.gz")) {
-        $Filename = [IO.path]::GetFileNameWithoutExtension($Filename)
-    }
-
-    return [IO.path]::GetFileNameWithoutExtension($Filename)
-}
-
-function New-AssetItem {
-    param (
-        [Parameter(Mandatory)][string]$Filename,
-        [Parameter(Mandatory)][string]$DownloadUrl,
-        [Parameter(Mandatory)][string]$Arch,
-        [Parameter(Mandatory)][string]$Platform,
-        [string]$PlatformVersion
-    )
-    $asset = New-Object PSObject
-
-    $asset | Add-Member -Name "filename" -Value $Filename -MemberType NoteProperty
-    $asset | Add-Member -Name "arch" -Value $Arch -MemberType NoteProperty
-    $asset | Add-Member -Name "platform" -Value $Platform -MemberType NoteProperty
-    if ($PlatformVersion) { $asset | Add-Member -Name "platform_version" -Value $PlatformVersion -MemberType NoteProperty }
-    $asset | Add-Member -Name "download_url" -Value $DownloadUrl -MemberType NoteProperty
-
-    return $asset
-}
-
-function Build-AssetsList {
-    param (
-        [AllowEmptyCollection()]
-        [Parameter(Mandatory)][array]$ReleaseAssets
-    )
-
-    
-    $assets = @()
-    foreach($releaseAsset in $ReleaseAssets) {
-        $filename = Get-FileNameWithoutExtension -Filename $releaseAsset.name
-        $parts = $filename.Split("-")
-        $arch = $parts[-1]
-        $buildPlatform = [string]::Join("-", $parts[2..($parts.Length-2)])
-
-        if ($PlatformMap[$buildPlatform]) {
-            $PlatformMap[$buildPlatform] | ForEach-Object {
-                $assets += New-AssetItem -Filename $releaseAsset.name `
-                                         -DownloadUrl $releaseAsset.browser_download_url `
-                                         -Arch $arch `
-                                         -Platform $_.platform `
-                                         -PlatformVersion $_.platform_version
-            }
-
-        } else {
-            $assets += New-AssetItem -Filename $releaseAsset.name `
-                                     -DownloadUrl $releaseAsset.browser_download_url `
-                                     -Arch $arch `
-                                     -Platform $buildPlatform
-        }
-    }
-
-    return $assets
-}
-
-function Get-VersionFromRelease {
-    param (
-        [Parameter(Mandatory)][object]$Release
-    )
-    # Release name can contain additional information after ':' so filter it
-    [string]$releaseName = $Release.name.Split(':')[0]
-    [Version]$version = $null
-    if (![Version]::TryParse($releaseName, [ref]$version)) {
-        throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
-    }
-
-    return $version
-}
-
-function Build-VersionsManifest {
-    param (
-        [Parameter(Mandatory)][array]$Releases
-    )
-
-    $Releases = $Releases | Sort-Object -Property "published_at" -Descending
-
-    $versionsHash = @{}
-    foreach ($release in $Releases) {
-        if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
-            continue
-        }
-
-        [Version]$version = Get-VersionFromRelease $release
-        $versionKey = $version.ToString()
-
-        if ($versionsHash.ContainsKey($versionKey)) {
-            continue
-        }
-
-        $versionsHash.Add($versionKey, [PSCustomObject]@{
-            version = $versionKey
-            stable = $true
-            release_url = $release.html_url
-            files = Build-AssetsList $release.assets
-        })
-    }
-
-    # Sort versions by descending
-    return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
-}
-
-$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
-$releases = $gitHubApi.GetReleases()
-$versionIndex = Build-VersionsManifest $releases
-$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force

helpers/packages-generation/manifest-generator.ps1

@@ -0,0 +1,35 @@
+<#
+.SYNOPSIS
+Generate versions manifest based on repository releases
+
+.DESCRIPTION
+Versions manifest is needed to find the latest assets for particular version of tool
+.PARAMETER GitHubRepositoryOwner
+Required parameter. The organization which tool repository belongs
+.PARAMETER GitHubRepositoryName
+Optional parameter. The name of tool repository
+.PARAMETER GitHubAccessToken
+Required parameter. PAT Token to overcome GitHub API Rate limit
+.PARAMETER OutputFile
+Required parameter. File "*.json" where generated results will be saved
+.PARAMETER ConfigurationFile
+Path to the json file with parsing configuration
+#>
+
+param (
+    [Parameter(Mandatory)] [string] $GitHubRepositoryOwner,
+    [Parameter(Mandatory)] [string] $GitHubRepositoryName,
+    [Parameter(Mandatory)] [string] $GitHubAccessToken,
+    [Parameter(Mandatory)] [string] $OutputFile,
+    [Parameter(Mandatory)] [string] $ConfigurationFile
+)
+
+Import-Module (Join-Path $PSScriptRoot "../github/github-api.psm1")
+Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force
+
+$configuration = Read-ConfigurationFile -Filepath $ConfigurationFile
+
+$gitHubApi = Get-GitHubApi -AccountName $GitHubRepositoryOwner -ProjectName $GitHubRepositoryName -AccessToken $GitHubAccessToken
+$releases = $gitHubApi.GetReleases()
+$versionIndex = Build-VersionsManifest -Releases $releases -Configuration $configuration
+$versionIndex | ConvertTo-Json -Depth 5 | Out-File $OutputFile -Encoding UTF8NoBOM -Force

helpers/packages-generation/manifest-utils.Tests.ps1

@@ -0,0 +1,116 @@
+Import-Module (Join-Path $PSScriptRoot "manifest-utils.psm1") -Force
+
+Describe "New-AssetItem" {
+    It "use regex to parse all values in correct order" {
+        $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; }
+        $configuration = @{
+            regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+            groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; };
+        }
+        $expectedOutput = [PSCustomObject]@{
+            filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04";
+            arch = "x64"; download_url = "long_url";
+        }
+
+        $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
+        Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
+    }
+
+    It "support constant values in groups" {
+        $githubAsset = @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "long_url"; }
+        $configuration = @{
+            regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+            groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; }
+        }
+        $expectedOutput = [PSCustomObject]@{
+            filename = "python-3.8.3-linux-16.04-x64.tar.gz"; platform = "linux"; platform_version = "16.04";
+            arch = "x64"; download_url = "long_url";
+        }
+
+        $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
+        Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
+    }
+
+    It "Skip empty groups" {
+        $githubAsset = @{ name = "python-3.8.3-win32-x64.zip"; browser_download_url = "long_url"; }
+        $configuration = @{
+            regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+            groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = 3; }
+        }
+        $expectedOutput = [PSCustomObject]@{
+            filename = "python-3.8.3-win32-x64.zip"; platform = "win32";
+            arch = "x64"; download_url = "long_url";
+        }
+
+        $actualOutput = New-AssetItem -ReleaseAsset $githubAsset -Configuration $configuration
+        Assert-Equivalent -Actual $actualOutput -Expected $expectedOutput
+    }
+}
+
+Describe "Get-VersionFromRelease" {
+    It "clear version" {
+        $release = @{ name = "3.8.3" }
+        Get-VersionFromRelease -Release $release | Should -Be "3.8.3"
+    }
+
+    It "version with title" {
+        $release = @{ name = "3.8.3: Release title" }
+        Get-VersionFromRelease -Release $release | Should -Be "3.8.3"
+    }
+}
+
+Describe "Build-VersionsManifest" {
+    $assets = @(
+        @{ name = "python-3.8.3-linux-16.04-x64.tar.gz"; browser_download_url = "fake_url"; }
+        @{ name = "python-3.8.3-linux-18.04-x64.tar.gz"; browser_download_url = "fake_url"; }
+    )
+    $configuration = @{
+        regex = "python-\d+\.\d+\.\d+-(\w+)-([\w\.]+)?-?(x\d+)";
+        groups = [PSCustomObject]@{ platform = 1; platform_version = 2; arch = "x64"; }
+    }
+    $expectedManifestFiles = @(
+        [PSCustomObject]@{ filename = "python-3.8.3-linux-16.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "16.04"; download_url = "fake_url" },
+        [PSCustomObject]@{ filename = "python-3.8.3-linux-18.04-x64.tar.gz"; arch = "x64"; platform = "linux"; platform_version = "18.04"; download_url = "fake_url" }
+    )
+
+    It "build manifest with correct version order" {
+        $releases = @(
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
+            @{ name = "3.5.2: Hello"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
+            @{ name = "3.8.3: Release title"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles },
+            [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles },
+            [PSCustomObject]@{ version = "3.5.2"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }
+        )
+        $actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
+    }
+
+    It "Skip draft and prerelease" {
+        $releases = @(
+            @{ name = "3.8.1"; draft = $true; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
+            @{ name = "3.5.2"; draft = $false; prerelease = $true; html_url = "fake_html_url"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
+            @{ name = "3.8.3"; draft = $false; prerelease = $false; html_url = "fake_html_url"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.3"; stable = $true; release_url = "fake_html_url"; files = $expectedManifestFiles }
+        )
+        [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest 
+    }
+
+    It "take latest published release for each version" {
+        $releases = @(
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url1"; published_at = "2020-05-06T11:45:36Z"; assets = $assets },
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url2"; published_at = "2020-05-14T09:54:06Z"; assets = $assets },
+            @{ name = "3.8.1"; draft = $false; prerelease = $false; html_url = "fake_html_url3"; published_at = "2020-05-06T11:43:38Z"; assets = $assets }
+        )
+        $expectedManifest = @(
+            [PSCustomObject]@{ version = "3.8.1"; stable = $true; release_url = "fake_html_url2"; files = $expectedManifestFiles }
+        )
+        [array]$actualManifest = Build-VersionsManifest -Releases $releases -Configuration $configuration
+        Assert-Equivalent -Actual $actualManifest -Expected $expectedManifest
+    }
+}

helpers/packages-generation/manifest-utils.psm1

@@ -0,0 +1,77 @@
+function Read-ConfigurationFile {
+    param ([Parameter(Mandatory)][string]$Filepath)
+    return Get-Content $Filepath -Raw | ConvertFrom-Json
+}
+
+function New-AssetItem {
+    param (
+        [Parameter(Mandatory)][object]$ReleaseAsset,
+        [Parameter(Mandatory)][object]$Configuration
+    )
+    $regexResult = [regex]::Match($ReleaseAsset.name, $Configuration.regex)
+    if (-not $regexResult.Success) { throw "Can't match asset filename '$($_.name)' to regex" }
+
+    $result = New-Object PSObject
+    $result | Add-Member -Name "filename" -Value $ReleaseAsset.name -MemberType NoteProperty
+    $Configuration.groups.PSObject.Properties | ForEach-Object {
+        if (($_.Value).GetType().Name.StartsWith("Int")) {
+            $value = $regexResult.Groups[$_.Value].Value
+        } else {
+            $value = $_.Value
+        }
+
+        if (-not ([string]::IsNullOrEmpty($value))) {
+            $result | Add-Member -Name $_.Name -Value $value -MemberType NoteProperty
+        }
+    }
+
+    $result | Add-Member -Name "download_url" -Value $ReleaseAsset.browser_download_url -MemberType NoteProperty
+    return $result
+}
+
+function Get-VersionFromRelease {
+    param (
+        [Parameter(Mandatory)][object]$Release
+    )
+    # Release name can contain additional information after ':' so filter it
+    [string]$releaseName = $Release.name.Split(':')[0]
+    [Version]$version = $null
+    if (![Version]::TryParse($releaseName, [ref]$version)) {
+        throw "Release '$($Release.id)' has invalid title '$($Release.name)'. It can't be parsed as version. ( $($Release.html_url) )"
+    }
+
+    return $version
+}
+
+function Build-VersionsManifest {
+    param (
+        [Parameter(Mandatory)][array]$Releases,
+        [Parameter(Mandatory)][object]$Configuration
+    )
+
+    $Releases = $Releases | Sort-Object -Property "published_at" -Descending
+
+    $versionsHash = @{}
+    foreach ($release in $Releases) {
+        if (($release.draft -eq $true) -or ($release.prerelease -eq $true)) {
+            continue
+        }
+
+        [Version]$version = Get-VersionFromRelease $release
+        $versionKey = $version.ToString()
+
+        if ($versionsHash.ContainsKey($versionKey)) {
+            continue
+        }
+
+        $versionsHash.Add($versionKey, [PSCustomObject]@{
+            version = $versionKey
+            stable = $true
+            release_url = $release.html_url
+            files = $release.assets | ForEach-Object { New-AssetItem -ReleaseAsset $_ -Configuration $Configuration }
+        })
+    }
+
+    # Sort versions by descending
+    return $versionsHash.Values | Sort-Object -Property @{ Expression = { [Version]$_.version }; Descending = $true }
+}

helpers/win-helpers.psm1

@@ -21,14 +21,19 @@ function Create-SevenZipArchive {
         [Parameter(Mandatory=$true)]
         [String]$ArchivePath,
         [String]$ArchiveType = "zip",
-        [String]$CompressionLevel = 5
+        [String]$CompressionLevel = 5,
+        [switch]$IncludeSymlinks
     )
 
-    $ArchiveTypeArgument = "-t${ArchiveType}"
+    $ArchiveTypeArguments = @(
-    $CompressionLevelArgument = "-mx=${CompressionLevel}"
+        "-t${ArchiveType}",
-    
+        "-mx=${CompressionLevel}"
+    )
+    if ($IncludeSymlinks) {
+        $ArchiveTypeArguments += "-snl"
+    }
     Push-Location $SourceFolder
-    Write-Debug "7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath @$SourceFolder"
+    Write-Debug "7z a $ArchiveTypeArgument $ArchivePath @$SourceFolder"
-    7z a $ArchiveTypeArgument $CompressionLevelArgument $ArchivePath $SourceFolder\*
+    7z a @ArchiveTypeArguments $ArchivePath $SourceFolder\*
     Pop-Location
 }

helpers/win-vs-env.psm1

@@ -0,0 +1,48 @@
+###
+# Visual Studio helper functions
+###
+
+function Get-VSWhere {
+    $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe";
+
+    if (-not (Test-Path $vswhere )) {
+        [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12
+        $vswhere = ".\vswhere.exe"
+        $vswhereApiUri = "https://api.github.com/repos/Microsoft/vswhere/releases/latest"
+        $tag = (Invoke-RestMethod -Uri $vswhereApiUri)[0].tag_name
+        $vswhereUri = "https://github.com/Microsoft/vswhere/releases/download/$tag/vswhere.exe"
+        Invoke-WebRequest -Uri $vswhereUri -OutFile $vswhere | Out-Null
+    }
+
+    return $vswhere
+}
+
+function Invoke-Environment
+{
+    Param
+    (
+        [Parameter(Mandatory)]
+        [string]
+        $Command
+    )
+
+    & "${env:COMSPEC}" /s /c "`"$Command`" -no_logo && set" | Foreach-Object {
+        if ($_ -match '^([^=]+)=(.*)') {
+            [System.Environment]::SetEnvironmentVariable($matches[1], $matches[2])
+        }
+    }
+}
+
+function Get-VSInstallationPath {
+    $vswhere = Get-VSWhere
+    $installationPath = & $vswhere -prerelease -legacy -latest -property installationPath
+
+    return $installationPath
+}
+
+function Invoke-VSDevEnvironment {
+    Write-Host "Invoke-VSDevEnvironment had been invoked"
+    $installationPath = Get-VSInstallationPath
+    $envFilepath = Join-Path $installationPath "Common7\Tools\vsdevcmd.bat"
+    Invoke-Environment -Command $envFilepath
+}

tests/Node.Tests.ps1

@@ -3,7 +3,7 @@ param (
     $Version
 )
 
-Import-Module (Join-Path $PSScriptRoot "../helpers/packages-generation/pester-extensions.psm1")
+Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
 
 function Get-UseNodeLogs {
     $logsFolderPath = Join-Path -Path $env:AGENT_HOMEDIRECTORY -ChildPath "_diag" | Join-Path -ChildPath "pages"