From 739154f76b6d2caca93e1942ee1258e0862bf510 Mon Sep 17 00:00:00 2001 From: Dmitry Shibanov Date: Mon, 14 Dec 2020 16:59:06 +0300 Subject: [PATCH] add support to install pypy --- dist/index.js | 340 ++++++++++++++++++++++++++++++++++++++++-- src/find-pypy.ts | 116 ++++++++++++++ src/find-python.ts | 4 +- src/install-pypy.ts | 216 +++++++++++++++++++++++++++ src/install-python.ts | 5 +- src/setup-python.ts | 18 ++- src/utils.ts | 42 ++++++ 7 files changed, 718 insertions(+), 23 deletions(-) create mode 100644 src/find-pypy.ts create mode 100644 src/install-pypy.ts create mode 100644 src/utils.ts diff --git a/dist/index.js b/dist/index.js index 41cfa78b..c6684881 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1072,6 +1072,108 @@ function _readLinuxVersionFile() { exports._readLinuxVersionFile = _readLinuxVersionFile; //# sourceMappingURL=manifest.js.map +/***/ }), + +/***/ 50: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; + result["default"] = mod; + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const path = __importStar(__webpack_require__(622)); +const pypyInstall = __importStar(__webpack_require__(369)); +const utils_1 = __webpack_require__(163); +const semver = __importStar(__webpack_require__(876)); +const core = __importStar(__webpack_require__(470)); +const tc = __importStar(__webpack_require__(533)); +function findPyPyVersion(versionSpec, architecture) { + return __awaiter(this, void 0, void 0, function* () { + let resolvedPyPyVersion = ''; + let resolvedPythonVersion = ''; + let installDir; + const pypyVersionSpec = parsePyPyVersion(versionSpec); + // PyPy only precompiles binaries for x86, but the architecture parameter defaults to x64. + if (utils_1.IS_WINDOWS && architecture === 'x64') { + architecture = 'x86'; + } + ({ installDir, resolvedPythonVersion, resolvedPyPyVersion } = findPyPyToolCache(pypyVersionSpec.pythonVersion, pypyVersionSpec.pypyVersion, architecture)); + if (!installDir) { + ({ + installDir, + resolvedPythonVersion, + resolvedPyPyVersion + } = yield pypyInstall.installPyPy(pypyVersionSpec.pypyVersion, pypyVersionSpec.pythonVersion, architecture)); + } + const pipDir = utils_1.IS_WINDOWS ? 'Scripts' : 'bin'; + const _binDir = path.join(installDir, pipDir); + const pythonLocation = pypyInstall.getPyPyBinaryPath(installDir); + core.exportVariable('pythonLocation', pythonLocation); + core.addPath(pythonLocation); + core.addPath(_binDir); + return { resolvedPyPyVersion, resolvedPythonVersion }; + }); +} +exports.findPyPyVersion = findPyPyVersion; +function findPyPyToolCache(pythonVersion, pypyVersion, architecture) { + let resolvedPyPyVersion = ''; + let resolvedPythonVersion = ''; + let installDir = tc.find('PyPy', pythonVersion, architecture); + if (installDir) { + // 'tc.find' finds tool based on Python version but we also need to check + // whether PyPy version satisfies requested version. + resolvedPythonVersion = getPyPyVersionFromPath(installDir); + resolvedPyPyVersion = pypyInstall.readExactPyPyVersion(installDir); + const isPyPyVersionSatisfies = semver.satisfies(resolvedPyPyVersion, pypyVersion); + if (!isPyPyVersionSatisfies) { + installDir = null; + resolvedPyPyVersion = ''; + resolvedPythonVersion = ''; + } + } + if (!installDir) { + core.info(`PyPy version ${pythonVersion} (${pypyVersion}) was not found in the local cache`); + } + return { installDir, resolvedPythonVersion, resolvedPyPyVersion }; +} +function parsePyPyVersion(versionSpec) { + const versions = versionSpec.split('-').filter(item => !!item); + if (versions.length < 2) { + throw new Error("Invalid 'version' property for PyPy. PyPy version should be specified as 'pypy-'. See readme for more examples."); + } + const pythonVersion = versions[1]; + let pypyVersion; + if (versions.length > 2) { + pypyVersion = pypyInstall.pypyVersionToSemantic(versions[2]); + } + else { + pypyVersion = 'x'; + } + return { + pypyVersion: pypyVersion, + pythonVersion: pythonVersion + }; +} +function getPyPyVersionFromPath(installDir) { + return path.basename(path.dirname(installDir)); +} + + /***/ }), /***/ 65: @@ -2197,6 +2299,43 @@ if (process.env.NODE_DEBUG && /\btunnel\b/.test(process.env.NODE_DEBUG)) { exports.debug = debug; // for test +/***/ }), + +/***/ 163: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; + result["default"] = mod; + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const fs = __importStar(__webpack_require__(747)); +const path = __importStar(__webpack_require__(622)); +exports.IS_WINDOWS = process.platform === 'win32'; +exports.IS_LINUX = process.platform === 'linux'; +/** create Symlinks for downloaded PyPy + * It should be executed only for downloaded versions in runtime, because + * toolcache versions have this setup. + */ +function createSymlinkInFolder(folderPath, sourceName, targetName, setExecutable = false) { + const sourcePath = path.join(folderPath, sourceName); + const targetPath = path.join(folderPath, targetName); + if (fs.existsSync(targetPath)) { + return; + } + fs.symlinkSync(sourcePath, targetPath); + if (!exports.IS_WINDOWS && setExecutable) { + fs.chmodSync(targetPath, '755'); + } +} +exports.createSymlinkInFolder = createSymlinkInFolder; + + /***/ }), /***/ 164: @@ -2443,16 +2582,26 @@ var __importStar = (this && this.__importStar) || function (mod) { Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(__webpack_require__(470)); const finder = __importStar(__webpack_require__(927)); +const finderPyPy = __importStar(__webpack_require__(50)); const path = __importStar(__webpack_require__(622)); const os = __importStar(__webpack_require__(87)); +function isPyPyVersion(versionSpec) { + return versionSpec.startsWith('pypy-'); +} function run() { return __awaiter(this, void 0, void 0, function* () { try { let version = core.getInput('python-version'); if (version) { const arch = core.getInput('architecture') || os.arch(); - const installed = yield finder.findPythonVersion(version, arch); - core.info(`Successfully setup ${installed.impl} (${installed.version})`); + if (isPyPyVersion(version)) { + const installed = yield finderPyPy.findPyPyVersion(version, arch); + core.info(`Successfully setup PyPy ${installed.resolvedPyPyVersion} with Python (${installed.resolvedPythonVersion})`); + } + else { + const installed = yield finder.findPythonVersion(version, arch); + core.info(`Successfully setup ${installed.impl} (${installed.version})`); + } } const matchersPath = path.join(__dirname, '..', '.github'); core.info(`##[add-matcher]${path.join(matchersPath, 'python.json')}`); @@ -2580,6 +2729,171 @@ module.exports = ltr module.exports = require("assert"); +/***/ }), + +/***/ 369: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; + result["default"] = mod; + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const path = __importStar(__webpack_require__(622)); +const core = __importStar(__webpack_require__(470)); +const tc = __importStar(__webpack_require__(533)); +const semver = __importStar(__webpack_require__(876)); +const httpm = __importStar(__webpack_require__(539)); +const exec = __importStar(__webpack_require__(986)); +const fs = __importStar(__webpack_require__(747)); +const utils_1 = __webpack_require__(163); +const PYPY_VERSION_FILE = 'PYPY_VERSION'; +function installPyPy(pypyVersion, pythonVersion, architecture) { + return __awaiter(this, void 0, void 0, function* () { + let downloadDir; + const releases = yield getAvailablePyPyVersions(); + const releaseData = findRelease(releases, pythonVersion, pypyVersion, architecture); + if (!releaseData || !releaseData.foundAsset) { + throw new Error(`PyPy version ${pythonVersion} (${pypyVersion}) with arch ${architecture} not found`); + } + const { foundAsset, resolvedPythonVersion, resolvedPyPyVersion } = releaseData; + let downloadUrl = `${foundAsset.download_url}`; + core.info(`Downloading PyPy from "${downloadUrl}" ...`); + const pypyPath = yield tc.downloadTool(downloadUrl); + core.info('Extracting downloaded archive...'); + if (utils_1.IS_WINDOWS) { + downloadDir = yield tc.extractZip(pypyPath); + } + else { + downloadDir = yield tc.extractTar(pypyPath, undefined, 'x'); + } + // root folder in archive can have unpredictable name so just take the first folder + // downloadDir is unique folder under TEMP and can't contain any other folders + const archiveName = fs.readdirSync(downloadDir)[0]; + const toolDir = path.join(downloadDir, archiveName); + let installDir = toolDir; + if (!isNightlyKeyword(resolvedPyPyVersion)) { + installDir = yield tc.cacheDir(toolDir, 'PyPy', resolvedPythonVersion, architecture); + } + writeExactPyPyVersionFile(installDir, resolvedPyPyVersion); + const binaryPath = getPyPyBinaryPath(installDir); + yield createPyPySymlink(binaryPath, resolvedPythonVersion); + yield installPip(binaryPath); + return { installDir, resolvedPythonVersion, resolvedPyPyVersion }; + }); +} +exports.installPyPy = installPyPy; +function getAvailablePyPyVersions() { + return __awaiter(this, void 0, void 0, function* () { + const url = 'https://downloads.python.org/pypy/versions.json'; + const http = new httpm.HttpClient('tool-cache'); + const response = yield http.getJson(url); + if (!response.result) { + throw new Error(`Unable to retrieve the list of available PyPy versions from '${url}'`); + } + return response.result; + }); +} +function createPyPySymlink(pypyBinaryPath, pythonVersion) { + return __awaiter(this, void 0, void 0, function* () { + const version = semver.coerce(pythonVersion); + const pythonBinaryPostfix = semver.major(version); + const pypyBinaryPostfix = pythonBinaryPostfix === 2 ? '' : '3'; + let binaryExtension = utils_1.IS_WINDOWS ? '.exe' : ''; + core.info('Creating symlinks...'); + utils_1.createSymlinkInFolder(pypyBinaryPath, `pypy${pypyBinaryPostfix}${binaryExtension}`, `python${pythonBinaryPostfix}${binaryExtension}`, true); + utils_1.createSymlinkInFolder(pypyBinaryPath, `pypy${pypyBinaryPostfix}${binaryExtension}`, `python${binaryExtension}`, true); + }); +} +function installPip(pythonLocation) { + return __awaiter(this, void 0, void 0, function* () { + core.info('Installing and updating pip'); + const pythonBinary = path.join(pythonLocation, 'python'); + yield exec.exec(`${pythonBinary} -m ensurepip`); + // TO-DO should we skip updating of pip ? + yield exec.exec(`${pythonLocation}/python -m pip install --ignore-installed pip`); + }); +} +function findRelease(releases, pythonVersion, pypyVersion, architecture) { + const filterReleases = releases.filter(item => { + const isPythonVersionSatisfies = semver.satisfies(semver.coerce(item.python_version), pythonVersion); + const isPyPyNightly = isNightlyKeyword(pypyVersion) && isNightlyKeyword(item.pypy_version); + const isPyPyVersionSatisfies = isPyPyNightly || + semver.satisfies(pypyVersionToSemantic(item.pypy_version), pypyVersion); + const isArchExists = item.files.some(file => file.arch === architecture && file.platform === process.platform); + return isPythonVersionSatisfies && isPyPyVersionSatisfies && isArchExists; + }); + if (filterReleases.length === 0) { + return null; + } + const sortedReleases = filterReleases.sort((previous, current) => { + return (semver.compare(semver.coerce(pypyVersionToSemantic(current.pypy_version)), semver.coerce(pypyVersionToSemantic(previous.pypy_version))) || + semver.compare(semver.coerce(current.python_version), semver.coerce(previous.python_version))); + }); + const foundRelease = sortedReleases[0]; + const foundAsset = foundRelease.files.find(item => item.arch === architecture && item.platform === process.platform); + return { + foundAsset, + resolvedPythonVersion: foundRelease.python_version, + resolvedPyPyVersion: foundRelease.pypy_version + }; +} +// helper functions +/** + * In tool-cache, we put PyPy to '/PyPy//x64' + * There is no easy way to determine what PyPy version is located in specific folder + * 'pypy --version' is not reliable enough since it is not set properly for preview versions + * "7.3.3rc1" is marked as '7.3.3' in 'pypy --version' + * so we put PYPY_VERSION file to PyPy directory when install it to VM and read it when we need to know version + * PYPY_VERSION contains exact version from 'versions.json' + */ +function readExactPyPyVersion(installDir) { + let pypyVersion = ''; + let fileVersion = path.join(installDir, PYPY_VERSION_FILE); + if (fs.existsSync(fileVersion)) { + pypyVersion = fs.readFileSync(fileVersion).toString(); + core.debug(`Version from ${PYPY_VERSION_FILE} file is ${pypyVersion}`); + } + return pypyVersion; +} +exports.readExactPyPyVersion = readExactPyPyVersion; +function writeExactPyPyVersionFile(installDir, resolvedPyPyVersion) { + const pypyFilePath = path.join(installDir, PYPY_VERSION_FILE); + fs.writeFileSync(pypyFilePath, resolvedPyPyVersion); +} +/** Get PyPy binary location from the tool of installation directory + * - On Linux and macOS, the Python interpreter is in 'bin'. + * - On Windows, it is in the installation root. + */ +function getPyPyBinaryPath(installDir) { + const _binDir = path.join(installDir, 'bin'); + return utils_1.IS_WINDOWS ? installDir : _binDir; +} +exports.getPyPyBinaryPath = getPyPyBinaryPath; +function isNightlyKeyword(pypyVersion) { + return pypyVersion === 'nightly'; +} +function pypyVersionToSemantic(versionSpec) { + const prereleaseVersion = /(\d+\.\d+\.\d+)((?:a|b|rc))(\d*)/g; + return versionSpec.replace(prereleaseVersion, '$1-$2.$3'); +} +exports.pypyVersionToSemantic = pypyVersionToSemantic; + + /***/ }), /***/ 413: @@ -6426,14 +6740,13 @@ const path = __importStar(__webpack_require__(622)); const core = __importStar(__webpack_require__(470)); const tc = __importStar(__webpack_require__(533)); const exec = __importStar(__webpack_require__(986)); +const utils_1 = __webpack_require__(163); const TOKEN = core.getInput('token'); const AUTH = !TOKEN || isGhes() ? undefined : `token ${TOKEN}`; const MANIFEST_REPO_OWNER = 'actions'; const MANIFEST_REPO_NAME = 'python-versions'; const MANIFEST_REPO_BRANCH = 'main'; exports.MANIFEST_URL = `https://raw.githubusercontent.com/${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}/${MANIFEST_REPO_BRANCH}/versions-manifest.json`; -const IS_WINDOWS = process.platform === 'win32'; -const IS_LINUX = process.platform === 'linux'; function findReleaseFromManifest(semanticVersionSpec, architecture) { return __awaiter(this, void 0, void 0, function* () { const manifest = yield tc.getManifestFromRepo(MANIFEST_REPO_OWNER, MANIFEST_REPO_NAME, AUTH, MANIFEST_REPO_BRANCH); @@ -6445,7 +6758,7 @@ function installPython(workingDirectory) { return __awaiter(this, void 0, void 0, function* () { const options = { cwd: workingDirectory, - env: Object.assign(Object.assign({}, process.env), (IS_LINUX && { LD_LIBRARY_PATH: path.join(workingDirectory, 'lib') })), + env: Object.assign(Object.assign({}, process.env), (utils_1.IS_LINUX && { LD_LIBRARY_PATH: path.join(workingDirectory, 'lib') })), silent: true, listeners: { stdout: (data) => { @@ -6456,7 +6769,7 @@ function installPython(workingDirectory) { } } }; - if (IS_WINDOWS) { + if (utils_1.IS_WINDOWS) { yield exec.exec('powershell', ['./setup.ps1'], options); } else { @@ -6471,7 +6784,7 @@ function installCpythonFromRelease(release) { const pythonPath = yield tc.downloadTool(downloadUrl, undefined, AUTH); core.info('Extract downloaded archive'); let pythonExtractedFolder; - if (IS_WINDOWS) { + if (utils_1.IS_WINDOWS) { pythonExtractedFolder = yield tc.extractZip(pythonPath); } else { @@ -6686,12 +6999,11 @@ var __importStar = (this && this.__importStar) || function (mod) { Object.defineProperty(exports, "__esModule", { value: true }); const os = __importStar(__webpack_require__(87)); const path = __importStar(__webpack_require__(622)); +const utils_1 = __webpack_require__(163); const semver = __importStar(__webpack_require__(876)); const installer = __importStar(__webpack_require__(824)); const core = __importStar(__webpack_require__(470)); const tc = __importStar(__webpack_require__(533)); -const IS_WINDOWS = process.platform === 'win32'; -const IS_LINUX = process.platform === 'linux'; // Python has "scripts" or "bin" directories where command-line tools that come with packages are installed. // This is where pip is, along with anything that pip installs. // There is a seperate directory for `pip install --user`. @@ -6705,7 +7017,7 @@ const IS_LINUX = process.platform === 'linux'; // (--user) %APPDATA%\Python\PythonXY\Scripts // See https://docs.python.org/3/library/sysconfig.html function binDir(installDir) { - if (IS_WINDOWS) { + if (utils_1.IS_WINDOWS) { return path.join(installDir, 'Scripts'); } else { @@ -6720,7 +7032,7 @@ function binDir(installDir) { function usePyPy(majorVersion, architecture) { const findPyPy = tc.find.bind(undefined, 'PyPy', majorVersion); let installDir = findPyPy(architecture); - if (!installDir && IS_WINDOWS) { + if (!installDir && utils_1.IS_WINDOWS) { // PyPy only precompiles binaries for x86, but the architecture parameter defaults to x64. // On our Windows virtual environments, we only install an x86 version. // Fall back to x86. @@ -6734,7 +7046,7 @@ function usePyPy(majorVersion, architecture) { const _binDir = path.join(installDir, 'bin'); // On Linux and macOS, the Python interpreter is in 'bin'. // On Windows, it is in the installation root. - const pythonLocation = IS_WINDOWS ? installDir : _binDir; + const pythonLocation = utils_1.IS_WINDOWS ? installDir : _binDir; core.exportVariable('pythonLocation', pythonLocation); core.addPath(installDir); core.addPath(_binDir); @@ -6764,7 +7076,7 @@ function useCpythonVersion(version, architecture) { ].join(os.EOL)); } core.exportVariable('pythonLocation', installDir); - if (IS_LINUX) { + if (utils_1.IS_LINUX) { const libPath = process.env.LD_LIBRARY_PATH ? `:${process.env.LD_LIBRARY_PATH}` : ''; @@ -6775,7 +7087,7 @@ function useCpythonVersion(version, architecture) { } core.addPath(installDir); core.addPath(binDir(installDir)); - if (IS_WINDOWS) { + if (utils_1.IS_WINDOWS) { // Add --user directory // `installDir` from tool cache should look like $RUNNER_TOOL_CACHE/Python//x64/ // So if `findLocalTool` succeeded above, we must have a conformant `installDir` diff --git a/src/find-pypy.ts b/src/find-pypy.ts new file mode 100644 index 00000000..ef8586f3 --- /dev/null +++ b/src/find-pypy.ts @@ -0,0 +1,116 @@ +import * as path from 'path'; +import * as pypyInstall from './install-pypy'; +import {IS_WINDOWS} from './utils'; + +import * as semver from 'semver'; +import * as core from '@actions/core'; +import * as tc from '@actions/tool-cache'; + +interface IPyPyVersionSpec { + pypyVersion: string; + pythonVersion: string; +} + +export async function findPyPyVersion( + versionSpec: string, + architecture: string +): Promise<{resolvedPyPyVersion: string; resolvedPythonVersion: string}> { + let resolvedPyPyVersion = ''; + let resolvedPythonVersion = ''; + let installDir: string | null; + + const pypyVersionSpec = parsePyPyVersion(versionSpec); + + // PyPy only precompiles binaries for x86, but the architecture parameter defaults to x64. + if (IS_WINDOWS && architecture === 'x64') { + architecture = 'x86'; + } + + ({installDir, resolvedPythonVersion, resolvedPyPyVersion} = findPyPyToolCache( + pypyVersionSpec.pythonVersion, + pypyVersionSpec.pypyVersion, + architecture + )); + + if (!installDir) { + ({ + installDir, + resolvedPythonVersion, + resolvedPyPyVersion + } = await pypyInstall.installPyPy( + pypyVersionSpec.pypyVersion, + pypyVersionSpec.pythonVersion, + architecture + )); + } + + const pipDir = IS_WINDOWS ? 'Scripts' : 'bin'; + const _binDir = path.join(installDir, pipDir); + const pythonLocation = pypyInstall.getPyPyBinaryPath(installDir); + core.exportVariable('pythonLocation', pythonLocation); + core.addPath(pythonLocation); + core.addPath(_binDir); + + return {resolvedPyPyVersion, resolvedPythonVersion}; +} + +function findPyPyToolCache( + pythonVersion: string, + pypyVersion: string, + architecture: string +) { + let resolvedPyPyVersion = ''; + let resolvedPythonVersion = ''; + let installDir: string | null = tc.find('PyPy', pythonVersion, architecture); + + if (installDir) { + // 'tc.find' finds tool based on Python version but we also need to check + // whether PyPy version satisfies requested version. + resolvedPythonVersion = getPyPyVersionFromPath(installDir); + resolvedPyPyVersion = pypyInstall.readExactPyPyVersion(installDir); + + const isPyPyVersionSatisfies = semver.satisfies( + resolvedPyPyVersion, + pypyVersion + ); + if (!isPyPyVersionSatisfies) { + installDir = null; + resolvedPyPyVersion = ''; + resolvedPythonVersion = ''; + } + } + + if (!installDir) { + core.info( + `PyPy version ${pythonVersion} (${pypyVersion}) was not found in the local cache` + ); + } + + return {installDir, resolvedPythonVersion, resolvedPyPyVersion}; +} + +function parsePyPyVersion(versionSpec: string): IPyPyVersionSpec { + const versions = versionSpec.split('-').filter(item => !!item); + + if (versions.length < 2) { + throw new Error( + "Invalid 'version' property for PyPy. PyPy version should be specified as 'pypy-'. See readme for more examples." + ); + } + const pythonVersion = versions[1]; + let pypyVersion: string; + if (versions.length > 2) { + pypyVersion = pypyInstall.pypyVersionToSemantic(versions[2]); + } else { + pypyVersion = 'x'; + } + + return { + pypyVersion: pypyVersion, + pythonVersion: pythonVersion + }; +} + +function getPyPyVersionFromPath(installDir: string) { + return path.basename(path.dirname(installDir)); +} diff --git a/src/find-python.ts b/src/find-python.ts index 6702430c..68136bd2 100644 --- a/src/find-python.ts +++ b/src/find-python.ts @@ -1,5 +1,6 @@ import * as os from 'os'; import * as path from 'path'; +import {IS_WINDOWS, IS_LINUX} from './utils'; import * as semver from 'semver'; @@ -8,9 +9,6 @@ import * as installer from './install-python'; import * as core from '@actions/core'; import * as tc from '@actions/tool-cache'; -const IS_WINDOWS = process.platform === 'win32'; -const IS_LINUX = process.platform === 'linux'; - // Python has "scripts" or "bin" directories where command-line tools that come with packages are installed. // This is where pip is, along with anything that pip installs. // There is a seperate directory for `pip install --user`. diff --git a/src/install-pypy.ts b/src/install-pypy.ts new file mode 100644 index 00000000..b8ef167b --- /dev/null +++ b/src/install-pypy.ts @@ -0,0 +1,216 @@ +import * as path from 'path'; +import * as core from '@actions/core'; +import * as tc from '@actions/tool-cache'; +import * as semver from 'semver'; +import * as httpm from '@actions/http-client'; +import * as exec from '@actions/exec'; +import * as fs from 'fs'; + +import {IS_WINDOWS, IPyPyManifestRelease, createSymlinkInFolder} from './utils'; + +const PYPY_VERSION_FILE = 'PYPY_VERSION'; + +export async function installPyPy( + pypyVersion: string, + pythonVersion: string, + architecture: string +) { + let downloadDir; + + const releases = await getAvailablePyPyVersions(); + const releaseData = findRelease( + releases, + pythonVersion, + pypyVersion, + architecture + ); + + if (!releaseData || !releaseData.foundAsset) { + throw new Error( + `PyPy version ${pythonVersion} (${pypyVersion}) with arch ${architecture} not found` + ); + } + + const {foundAsset, resolvedPythonVersion, resolvedPyPyVersion} = releaseData; + let downloadUrl = `${foundAsset.download_url}`; + + core.info(`Downloading PyPy from "${downloadUrl}" ...`); + const pypyPath = await tc.downloadTool(downloadUrl); + + core.info('Extracting downloaded archive...'); + if (IS_WINDOWS) { + downloadDir = await tc.extractZip(pypyPath); + } else { + downloadDir = await tc.extractTar(pypyPath, undefined, 'x'); + } + + // root folder in archive can have unpredictable name so just take the first folder + // downloadDir is unique folder under TEMP and can't contain any other folders + const archiveName = fs.readdirSync(downloadDir)[0]; + + const toolDir = path.join(downloadDir, archiveName); + let installDir = toolDir; + if (!isNightlyKeyword(resolvedPyPyVersion)) { + installDir = await tc.cacheDir( + toolDir, + 'PyPy', + resolvedPythonVersion, + architecture + ); + } + + writeExactPyPyVersionFile(installDir, resolvedPyPyVersion); + + const binaryPath = getPyPyBinaryPath(installDir); + await createPyPySymlink(binaryPath, resolvedPythonVersion); + await installPip(binaryPath); + + return {installDir, resolvedPythonVersion, resolvedPyPyVersion}; +} + +async function getAvailablePyPyVersions() { + const url = 'https://downloads.python.org/pypy/versions.json'; + const http: httpm.HttpClient = new httpm.HttpClient('tool-cache'); + + const response = await http.getJson(url); + if (!response.result) { + throw new Error( + `Unable to retrieve the list of available PyPy versions from '${url}'` + ); + } + + return response.result; +} + +async function createPyPySymlink( + pypyBinaryPath: string, + pythonVersion: string +) { + const version = semver.coerce(pythonVersion)!; + const pythonBinaryPostfix = semver.major(version); + const pypyBinaryPostfix = pythonBinaryPostfix === 2 ? '' : '3'; + let binaryExtension = IS_WINDOWS ? '.exe' : ''; + + core.info('Creating symlinks...'); + createSymlinkInFolder( + pypyBinaryPath, + `pypy${pypyBinaryPostfix}${binaryExtension}`, + `python${pythonBinaryPostfix}${binaryExtension}`, + true + ); + + createSymlinkInFolder( + pypyBinaryPath, + `pypy${pypyBinaryPostfix}${binaryExtension}`, + `python${binaryExtension}`, + true + ); +} + +async function installPip(pythonLocation: string) { + core.info('Installing and updating pip'); + const pythonBinary = path.join(pythonLocation, 'python'); + await exec.exec(`${pythonBinary} -m ensurepip`); + // TO-DO should we skip updating of pip ? + await exec.exec( + `${pythonLocation}/python -m pip install --ignore-installed pip` + ); +} + +function findRelease( + releases: IPyPyManifestRelease[], + pythonVersion: string, + pypyVersion: string, + architecture: string +) { + const filterReleases = releases.filter(item => { + const isPythonVersionSatisfies = semver.satisfies( + semver.coerce(item.python_version)!, + pythonVersion + ); + const isPyPyNightly = + isNightlyKeyword(pypyVersion) && isNightlyKeyword(item.pypy_version); + const isPyPyVersionSatisfies = + isPyPyNightly || + semver.satisfies(pypyVersionToSemantic(item.pypy_version), pypyVersion); + const isArchExists = item.files.some( + file => file.arch === architecture && file.platform === process.platform + ); + return isPythonVersionSatisfies && isPyPyVersionSatisfies && isArchExists; + }); + + if (filterReleases.length === 0) { + return null; + } + + const sortedReleases = filterReleases.sort((previous, current) => { + return ( + semver.compare( + semver.coerce(pypyVersionToSemantic(current.pypy_version))!, + semver.coerce(pypyVersionToSemantic(previous.pypy_version))! + ) || + semver.compare( + semver.coerce(current.python_version)!, + semver.coerce(previous.python_version)! + ) + ); + }); + + const foundRelease = sortedReleases[0]; + const foundAsset = foundRelease.files.find( + item => item.arch === architecture && item.platform === process.platform + ); + + return { + foundAsset, + resolvedPythonVersion: foundRelease.python_version, + resolvedPyPyVersion: foundRelease.pypy_version + }; +} + +// helper functions + +/** + * In tool-cache, we put PyPy to '/PyPy//x64' + * There is no easy way to determine what PyPy version is located in specific folder + * 'pypy --version' is not reliable enough since it is not set properly for preview versions + * "7.3.3rc1" is marked as '7.3.3' in 'pypy --version' + * so we put PYPY_VERSION file to PyPy directory when install it to VM and read it when we need to know version + * PYPY_VERSION contains exact version from 'versions.json' + */ +export function readExactPyPyVersion(installDir: string) { + let pypyVersion = ''; + let fileVersion = path.join(installDir, PYPY_VERSION_FILE); + if (fs.existsSync(fileVersion)) { + pypyVersion = fs.readFileSync(fileVersion).toString(); + core.debug(`Version from ${PYPY_VERSION_FILE} file is ${pypyVersion}`); + } + + return pypyVersion; +} + +function writeExactPyPyVersionFile( + installDir: string, + resolvedPyPyVersion: string +) { + const pypyFilePath = path.join(installDir, PYPY_VERSION_FILE); + fs.writeFileSync(pypyFilePath, resolvedPyPyVersion); +} + +/** Get PyPy binary location from the tool of installation directory + * - On Linux and macOS, the Python interpreter is in 'bin'. + * - On Windows, it is in the installation root. + */ +export function getPyPyBinaryPath(installDir: string) { + const _binDir = path.join(installDir, 'bin'); + return IS_WINDOWS ? installDir : _binDir; +} + +function isNightlyKeyword(pypyVersion: string) { + return pypyVersion === 'nightly'; +} + +export function pypyVersionToSemantic(versionSpec: string) { + const prereleaseVersion = /(\d+\.\d+\.\d+)((?:a|b|rc))(\d*)/g; + return versionSpec.replace(prereleaseVersion, '$1-$2.$3'); +} diff --git a/src/install-python.ts b/src/install-python.ts index 8fcfe68e..526e7d59 100644 --- a/src/install-python.ts +++ b/src/install-python.ts @@ -3,7 +3,7 @@ import * as core from '@actions/core'; import * as tc from '@actions/tool-cache'; import * as exec from '@actions/exec'; import {ExecOptions} from '@actions/exec/lib/interfaces'; -import {stderr} from 'process'; +import {IS_WINDOWS, IS_LINUX} from './utils'; const TOKEN = core.getInput('token'); const AUTH = !TOKEN || isGhes() ? undefined : `token ${TOKEN}`; @@ -12,9 +12,6 @@ const MANIFEST_REPO_NAME = 'python-versions'; const MANIFEST_REPO_BRANCH = 'main'; export const MANIFEST_URL = `https://raw.githubusercontent.com/${MANIFEST_REPO_OWNER}/${MANIFEST_REPO_NAME}/${MANIFEST_REPO_BRANCH}/versions-manifest.json`; -const IS_WINDOWS = process.platform === 'win32'; -const IS_LINUX = process.platform === 'linux'; - export async function findReleaseFromManifest( semanticVersionSpec: string, architecture: string diff --git a/src/setup-python.ts b/src/setup-python.ts index c97f314c..15e46956 100644 --- a/src/setup-python.ts +++ b/src/setup-python.ts @@ -1,15 +1,29 @@ import * as core from '@actions/core'; import * as finder from './find-python'; +import * as finderPyPy from './find-pypy'; import * as path from 'path'; import * as os from 'os'; +function isPyPyVersion(versionSpec: string) { + return versionSpec.startsWith('pypy-'); +} + async function run() { try { let version = core.getInput('python-version'); if (version) { const arch: string = core.getInput('architecture') || os.arch(); - const installed = await finder.findPythonVersion(version, arch); - core.info(`Successfully setup ${installed.impl} (${installed.version})`); + if (isPyPyVersion(version)) { + const installed = await finderPyPy.findPyPyVersion(version, arch); + core.info( + `Successfully setup PyPy ${installed.resolvedPyPyVersion} with Python (${installed.resolvedPythonVersion})` + ); + } else { + const installed = await finder.findPythonVersion(version, arch); + core.info( + `Successfully setup ${installed.impl} (${installed.version})` + ); + } } const matchersPath = path.join(__dirname, '..', '.github'); core.info(`##[add-matcher]${path.join(matchersPath, 'python.json')}`); diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 00000000..039f1929 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,42 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +export const IS_WINDOWS = process.platform === 'win32'; +export const IS_LINUX = process.platform === 'linux'; + +export interface IPyPyManifestAsset { + filename: string; + arch: string; + platform: string; + download_url: string; +} + +export interface IPyPyManifestRelease { + pypy_version: string; + python_version: string; + stable: boolean; + latest_pypy: boolean; + files: IPyPyManifestAsset[]; +} + +/** create Symlinks for downloaded PyPy + * It should be executed only for downloaded versions in runtime, because + * toolcache versions have this setup. + */ +export function createSymlinkInFolder( + folderPath: string, + sourceName: string, + targetName: string, + setExecutable = false +) { + const sourcePath = path.join(folderPath, sourceName); + const targetPath = path.join(folderPath, targetName); + if (fs.existsSync(targetPath)) { + return; + } + + fs.symlinkSync(sourcePath, targetPath); + if (!IS_WINDOWS && setExecutable) { + fs.chmodSync(targetPath, '755'); + } +}