From cbf0bf8d3cb4b1c54d30e80658a07cfdb75582df Mon Sep 17 00:00:00 2001 From: Doug Schaefer <167107236+dschaefer2@users.noreply.github.com> Date: Thu, 17 Apr 2025 00:15:38 -0400 Subject: [PATCH 1/4] Ensure prebuilt Modules include path added to generated modules (#8485) --- Sources/Build/BuildPlan/BuildPlan+Test.swift | 35 ++++++- Sources/PackageModel/Module/SwiftModule.swift | 9 +- Tests/BuildTests/BuildPlanTests.swift | 94 +++++++++++++++++++ 3 files changed, 133 insertions(+), 5 deletions(-) diff --git a/Sources/Build/BuildPlan/BuildPlan+Test.swift b/Sources/Build/BuildPlan/BuildPlan+Test.swift index b8e3fc9638a..a75f55a8bf9 100644 --- a/Sources/Build/BuildPlan/BuildPlan+Test.swift +++ b/Sources/Build/BuildPlan/BuildPlan+Test.swift @@ -27,6 +27,7 @@ import struct PackageGraph.ResolvedProduct import struct PackageGraph.ResolvedModule import struct PackageModel.Sources +import enum PackageModel.BuildSettings import class PackageModel.SwiftModule import class PackageModel.Module import struct SPMBuildCore.BuildParameters @@ -80,18 +81,30 @@ extension BuildPlan { let discoveryMainFile = discoveryDerivedDir.appending(component: TestDiscoveryTool.mainFileName) var discoveryPaths: [AbsolutePath] = [] + var discoveryBuildSettings: BuildSettings.AssignmentTable = .init() discoveryPaths.append(discoveryMainFile) for testTarget in testProduct.modules { let path = discoveryDerivedDir.appending(components: testTarget.name + ".swift") discoveryPaths.append(path) + // Add in the include path from the test targets to ensure this module builds + if let flags = testTarget.underlying.buildSettings.assignments[.OTHER_SWIFT_FLAGS] { + for assignment in flags { + let values = assignment.values.filter({ $0.hasPrefix("-I") }) + if !values.isEmpty { + discoveryBuildSettings.add(.init(values: values, conditions: []), for: .OTHER_SWIFT_FLAGS) + } + } + } } let discoveryTarget = SwiftModule( name: discoveryTargetName, dependencies: testProduct.underlying.modules.map { .module($0, conditions: []) }, packageAccess: true, // test target is allowed access to package decls by default - testDiscoverySrc: Sources(paths: discoveryPaths, root: discoveryDerivedDir) + testDiscoverySrc: Sources(paths: discoveryPaths, root: discoveryDerivedDir), + buildSettings: discoveryBuildSettings ) + let discoveryResolvedModule = ResolvedModule( packageIdentity: testProduct.packageIdentity, underlying: discoveryTarget, @@ -127,12 +140,26 @@ extension BuildPlan { let entryPointMainFile = entryPointDerivedDir.appending(component: entryPointMainFileName) let entryPointSources = Sources(paths: [entryPointMainFile], root: entryPointDerivedDir) + var entryPointBuildSettings: BuildSettings.AssignmentTable = .init() + for testTarget in testProduct.modules { + // Add in the include path from the test targets to ensure this module builds + if let flags = testTarget.underlying.buildSettings.assignments[.OTHER_SWIFT_FLAGS] { + for assignment in flags { + let values = assignment.values.filter({ $0.hasPrefix("-I") }) + if !values.isEmpty { + entryPointBuildSettings.add(.init(values: values, conditions: []), for: .OTHER_SWIFT_FLAGS) + } + } + } + } + let entryPointTarget = SwiftModule( name: testProduct.name, type: .library, dependencies: testProduct.underlying.modules.map { .module($0, conditions: []) } + swiftTargetDependencies, packageAccess: true, // test target is allowed access to package decls - testEntryPointSources: entryPointSources + testEntryPointSources: entryPointSources, + buildSettings: entryPointBuildSettings ) let entryPointResolvedTarget = ResolvedModule( packageIdentity: testProduct.packageIdentity, @@ -249,7 +276,8 @@ private extension PackageModel.SwiftModule { type: PackageModel.Module.Kind? = nil, dependencies: [PackageModel.Module.Dependency], packageAccess: Bool, - testEntryPointSources sources: Sources + testEntryPointSources sources: Sources, + buildSettings: BuildSettings.AssignmentTable = .init() ) { self.init( name: name, @@ -258,6 +286,7 @@ private extension PackageModel.SwiftModule { sources: sources, dependencies: dependencies, packageAccess: packageAccess, + buildSettings: buildSettings, usesUnsafeFlags: false ) } diff --git a/Sources/PackageModel/Module/SwiftModule.swift b/Sources/PackageModel/Module/SwiftModule.swift index 222c62c59ad..93b083b1ebf 100644 --- a/Sources/PackageModel/Module/SwiftModule.swift +++ b/Sources/PackageModel/Module/SwiftModule.swift @@ -28,7 +28,12 @@ public final class SwiftModule: Module { [defaultTestEntryPointName, "LinuxMain.swift"] } - public init(name: String, dependencies: [Module.Dependency], packageAccess: Bool, testDiscoverySrc: Sources) { + public init( + name: String, + dependencies: [Module.Dependency], + packageAccess: Bool, + testDiscoverySrc: Sources, + buildSettings: BuildSettings.AssignmentTable = .init()) { self.declaredSwiftVersions = [] super.init( @@ -38,7 +43,7 @@ public final class SwiftModule: Module { sources: testDiscoverySrc, dependencies: dependencies, packageAccess: packageAccess, - buildSettings: .init(), + buildSettings: buildSettings, buildSettingsDescription: [], pluginUsages: [], usesUnsafeFlags: false diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index 96861cba0a3..0b91f213021 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -4571,6 +4571,100 @@ final class BuildPlanTests: XCTestCase { } } + func testPrebuiltsFlags() async throws { + // Make sure the include path for the prebuilts get passed to the + // generated test entry point and discover targets on Linux/Windows + let observability = ObservabilitySystem.makeForTesting() + + let prebuiltLibrary = PrebuiltLibrary( + identity: .plain("swift-syntax"), + libraryName: "MacroSupport", + path: "/MyPackage/.build/prebuilts/swift-syntax/600.0.1/6.1-MacroSupport-macos_aarch64", + products: [ + "SwiftBasicFormat", + "SwiftCompilerPlugin", + "SwiftDiagnostics", + "SwiftIDEUtils", + "SwiftOperators", + "SwiftParser", + "SwiftParserDiagnostics", + "SwiftRefactor", + "SwiftSyntax", + "SwiftSyntaxBuilder", + "SwiftSyntaxMacros", + "SwiftSyntaxMacroExpansion", + "SwiftSyntaxMacrosTestSupport", + "SwiftSyntaxMacrosGenericTestSupport", + "_SwiftCompilerPluginMessageHandling", + "_SwiftLibraryPluginProvider" + ], + cModules: ["_SwiftSyntaxCShims"] + ) + + let fs = InMemoryFileSystem( + emptyFiles: [ + "/MyPackage/Sources/MyMacroMacros/MyMacroMacros.swift", + "/MyPackage/Sources/MyMacros/MyMacros.swift", + "/MyPackage/Sources/MyMacroTests/MyMacroTests.swift" + ] + ) + + let graph = try loadModulesGraph( + fileSystem: fs, + manifests: [ + Manifest.createRootManifest( + displayName: "MyPackage", + path: "/MyPackage", + targets: [ + TargetDescription(name: "MyMacroMacros", type: .macro), + TargetDescription( + name: "MyMacros", + dependencies: [ + "MyMacroMacros", + .product(name: "SwiftSyntaxMacros", package: "swift-syntax"), + .product(name: "SwiftCompilerPlugin", package: "swift-syntax"), + ] + ), + TargetDescription( + name: "MyMacroTests", + dependencies: [ + "MyMacroMacros", + .product(name: "SwiftSyntaxMacrosTestSupport", package: "swift-syntax"), + ], + type: .test + ) + ] + ) + ], + prebuilts: [prebuiltLibrary.identity: prebuiltLibrary.products.reduce(into: [:]) { + $0[$1] = prebuiltLibrary + }], + observabilityScope: observability.topScope + ) + + func checkTriple(triple: Basics.Triple) async throws { + let result = try await BuildPlanResult( + plan: mockBuildPlan( + triple: triple, + graph: graph, + fileSystem: fs, + observabilityScope: observability.topScope + ) + ) + + let modulesDir = "-I\(prebuiltLibrary.path.pathString)/Modules" + let mytest = try XCTUnwrap(result.allTargets(named: "MyMacroTests").first) + XCTAssert(try mytest.swift().compileArguments().contains(modulesDir)) + let entryPoint = try XCTUnwrap(result.allTargets(named: "MyPackagePackageTests").first) + XCTAssert(try entryPoint.swift().compileArguments().contains(modulesDir)) + let discovery = try XCTUnwrap(result.allTargets(named: "MyPackagePackageDiscoveredTests").first) + XCTAssert(try discovery.swift().compileArguments().contains(modulesDir)) + } + + try await checkTriple(triple: .x86_64Linux) + try await checkTriple(triple: .x86_64Windows) + } + func testExtraBuildFlags() async throws { let fs = InMemoryFileSystem( emptyFiles: From d2b0e6b3e38aecb5e8e7dc315e03b12bd89e6f60 Mon Sep 17 00:00:00 2001 From: Doug Schaefer Date: Mon, 21 Apr 2025 10:32:47 -0400 Subject: [PATCH 2/4] Empty commit to force CI From a1a7aa79afa19055055a267600147071b3ac6f44 Mon Sep 17 00:00:00 2001 From: Doug Schaefer Date: Wed, 23 Apr 2025 10:10:56 -0400 Subject: [PATCH 3/4] Add in the python build-using-self scripts from main. The CI jobs are expecting this to be a python script. Make it so. --- Utilities/build-using-self | 234 +++++++++++++++++++++++++++---------- Utilities/helpers.py | 96 +++++++++++++-- 2 files changed, 258 insertions(+), 72 deletions(-) diff --git a/Utilities/build-using-self b/Utilities/build-using-self index c1277db00e1..3b59152b9ba 100755 --- a/Utilities/build-using-self +++ b/Utilities/build-using-self @@ -1,62 +1,176 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the Swift open source project -## -## Copyright (c) 2014-2022 Apple Inc. and the Swift project authors -## Licensed under Apache License v2.0 with Runtime Library Exception -## -## See http://swift.org/LICENSE.txt for license information -## See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -## -##===----------------------------------------------------------------------===## - -set -eu - -__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -root_dir="$(cd ${__dir}/.. && pwd)" - -cd "${root_dir}/" -echo "Current directory is ${PWD}" - -CONFIGURATION=debug -export SWIFTCI_IS_SELF_HOSTED=1 - -# Ensure SDKROOT is configured -host=$(uname) -if [ "${host}" == "Darwin" ]; then - export SDKROOT=$(xcrun --show-sdk-path --sdk macosx) -fi - -set -x - -env | sort - -# Display toolchain version -swift --version - -# Perform package update in order to get the latest commits for the dependencies. -swift package update -swift build -c $CONFIGURATION -swift test -c $CONFIGURATION --parallel - -# Run the integration tests with just built SwiftPM. -export SWIFTPM_BIN_DIR=$(swift build -c $CONFIGURATION --show-bin-path) -( - cd ${root_dir}/IntegrationTests - # Perform package update in order to get the latest commits for the dependencies. - swift package update - $SWIFTPM_BIN_DIR/swift-test --parallel +#!/usr/bin/env python3 +# ===----------------------------------------------------------------------===## +# +# This source file is part of the Swift open source project +# +# Copyright (c) 2025 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ===----------------------------------------------------------------------===## + +import argparse +import logging +import os +import pathlib +import platform +import shlex +import sys +import typing as t + +from helpers import ( + Configuration, + change_directory, + call, + call_output, +) + +logging.basicConfig( + format=" | ".join( + [ + # Prefix script name to the log in an attempt to avoid confusion when parsing logs + f"{pathlib.Path(sys.argv[0]).name}", + "%(asctime)s", + "%(levelname)-8s", + "%(module)s", + "%(funcName)s", + "Line:%(lineno)d", + "%(message)s", + ] + ), + level=logging.INFO, ) -if [ "${host}" == "Darwin" ]; then - echo "Current working directory is ${PWD}" - echo "Bootstrapping with the XCBuild codepath..." - ${root_dir}/Utilities/bootstrap \ - build \ - --release \ - --verbose \ - --cross-compile-hosts macosx-arm64 \ - --skip-cmake-bootstrap \ - --swift-build-path "${SWIFTPM_BIN_DIR}/swift-build" -fi + +REPO_ROOT_PATH = pathlib.Path(__file__).parent.parent.resolve() + + +def get_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + "-v", + "--verbose", + dest="is_verbose", + action="store_true", + help="When set, prints verbose information.", + ) + parser.add_argument( + "-c", + "--configuration", + type=Configuration, + dest="config", + default="debug", + choices=[e.value for e in Configuration], + help="The configuraiton to use.", + ) + parser.add_argument( + "--enable-swift-testing", + action="store_true", + ) + parser.add_argument( + "--enable-xctest", + action="store_true", + ) + args = parser.parse_args() + return args + + +def log_environment() -> None: + logging.info("Environment Variables") + for key, value in sorted(os.environ.items()): + logging.info(" --> %s=%r", key, value) + + +def get_swiftpm_bin_dir(config: Configuration) -> pathlib.Path: + logging.info("Retrieving Swift PM binary directory.") + swiftpm_bin_dir = pathlib.Path( + call_output(["swift", "build", "--configuration", config, "--show-bin-path"]) + ) + logging.info("SwiftPM BIN DIR: %s", swiftpm_bin_dir) + return swiftpm_bin_dir + + +def is_on_darwin() -> bool: + return platform.uname().system == "Darwin" + + +def set_environment(*, swiftpm_bin_dir: pathlib.Path,) -> None: + os.environ["SWIFTCI_IS_SELF_HOSTED"] = "1" + + # Set the SWIFTPM_CUSTOM_BIN_DIR path + os.environ["SWIFTPM_CUSTOM_BIN_DIR"] = str(swiftpm_bin_dir) + + # Ensure SDKROOT is configure + if is_on_darwin(): + sdk_root = call_output(shlex.split("xcrun --show-sdk-path --sdk macosx")) + logging.debug("macos sdk root = %r", sdk_root) + os.environ["SDKROOT"] = sdk_root + log_environment() + + +def run_bootstrap(swiftpm_bin_dir: pathlib.Path) -> None: + logging.info("Current working directory is %s", pathlib.Path.cwd()) + logging.info("Bootstrapping with the XCBuild codepath...") + call( + [ + REPO_ROOT_PATH / "Utilities" / "bootstrap", + "build", + "--release", + "--verbose", + "--cross-compile-hosts", + "macosx-arm64", + "--skip-cmake-bootstrap", + "--swift-build-path", + (swiftpm_bin_dir / "swift-build").resolve(), + ], + ) + + +def main() -> None: + args = get_arguments() + ignore = "-Xlinker /ignore:4217" if os.name == "nt" else "" + logging.getLogger().setLevel(logging.DEBUG if args.is_verbose else logging.INFO) + logging.debug("Args: %r", args) + + with change_directory(REPO_ROOT_PATH): + swiftpm_bin_dir = get_swiftpm_bin_dir(config=args.config) + set_environment(swiftpm_bin_dir=swiftpm_bin_dir) + + call( + shlex.split("swift --version"), + ) + + call( + shlex.split("swift package update"), + ) + call( + shlex.split(f"swift build --configuration {args.config} {ignore}"), + ) + + if os.name != "nt": # turn off for Windows until we get the hang resolved + swift_testing_arg= "--enable-swift-testing" if args.enable_swift_testing else "" + xctest_arg= "--enable-xctest" if args.enable_swift_testing else "" + call( + shlex.split(f"swift run swift-test --configuration {args.config} --parallel {swift_testing_arg} {xctest_arg} --scratch-path .test {ignore}"), + ) + + integration_test_dir = REPO_ROOT_PATH / "IntegrationTests" + call( + shlex.split(f"swift package --package-path {integration_test_dir} update"), + ) + call( + shlex.split(f"swift run swift-test --package-path {integration_test_dir} --parallel {ignore}"), + ) + + if is_on_darwin(): + run_bootstrap(swiftpm_bin_dir=swiftpm_bin_dir) + logging.info("Done") + + +if __name__ == "__main__": + main() diff --git a/Utilities/helpers.py b/Utilities/helpers.py index cea6d02f51b..b5a4a9df9a9 100644 --- a/Utilities/helpers.py +++ b/Utilities/helpers.py @@ -3,7 +3,7 @@ ## ## This source file is part of the Swift open source project ## -## Copyright (c) 2014-2020 Apple Inc. and the Swift project authors +## Copyright (c) 2014-2025 Apple Inc. and the Swift project authors ## Licensed under Apache License v2.0 with Runtime Library Exception ## ## See http://swift.org/LICENSE.txt for license information @@ -11,12 +11,37 @@ ## ##===----------------------------------------------------------------------===## -import datetime +import contextlib +import enum +import errno import logging -import subprocess -import sys import os -import errno +import pathlib +import subprocess +import typing as t + + +@contextlib.contextmanager +def change_directory(directory: pathlib.Path) -> t.Iterator[pathlib.Path]: + current_directory = pathlib.Path.cwd() + logging.info("Current directory is %s", current_directory) + logging.info("Changing directory to: %s", directory) + os.chdir(directory) + + try: + yield directory + finally: + logging.debug("Chaning directory back to %s", current_directory) + os.chdir(current_directory) + + +class Configuration(str, enum.Enum): + DEBUG = "debug" + RELEASE = "release" + + def __str__(self) -> str: + return self.value + def symlink_force(source, destination): try: @@ -26,6 +51,7 @@ def symlink_force(source, destination): os.remove(destination) os.symlink(source, destination) + def mkdir_p(path): """Create the given directory, if it does not exist.""" try: @@ -35,22 +61,68 @@ def mkdir_p(path): if e.errno != errno.EEXIST: raise + def call(cmd, cwd=None, verbose=False): """Calls a subprocess.""" - logging.info("executing command >>> %s", ' '.join(cmd)) + cwd = cwd or pathlib.Path.cwd() + logging.info("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd) try: subprocess.check_call(cmd, cwd=cwd) except subprocess.CalledProcessError as cpe: - logging.debug("executing command >>> %s", ' '.join(cmd)) - logging.error("Process failure: %s", str(cpe)) + logging.debug("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd) + logging.error( + "\n".join([ + "Process failure with return code %d: %s", + "[---- START stdout ----]", + "%s", + "[---- END stdout ----]", + "[---- START stderr ----]", + "%s", + "[---- END stderr ----]", + "[---- START OUTPUT ----]", + "%s", + "[---- END OUTPUT ----]", + ]), + cpe.returncode, + str(cpe), + cpe.stdout, + cpe.stderr, + cpe.output, + ) raise cpe + def call_output(cmd, cwd=None, stderr=False, verbose=False): """Calls a subprocess for its return data.""" - logging.info(' '.join(cmd)) + stderr = subprocess.STDOUT if stderr else False + cwd = cwd or pathlib.Path.cwd() + logging.info("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd) try: - return subprocess.check_output(cmd, cwd=cwd, stderr=stderr, universal_newlines=True).strip() + return subprocess.check_output( + cmd, + cwd=cwd, + stderr=stderr, + universal_newlines=True, + ).strip() except subprocess.CalledProcessError as cpe: - logging.debug(' '.join(cmd)) - logging.error(str(cpe)) + logging.debug("executing command >>> %r with cwd %s", " ".join([str(c) for c in cmd]), cwd) + logging.error( + "\n".join([ + "Process failure with return code %d: %s", + "[---- START stdout ----]", + "%s", + "[---- END stdout ----]", + "[---- START stderr ----]", + "%s", + "[---- END stderr ----]", + "[---- START OUTPUT ----]", + "%s", + "[---- END OUTPUT ----]", + ]), + cpe.returncode, + str(cpe), + cpe.stdout, + cpe.stderr, + cpe.output, + ) raise cpe From 755c51c39facfe57a58e98e3f9ed14b70b4e52fd Mon Sep 17 00:00:00 2001 From: Doug Schaefer Date: Wed, 23 Apr 2025 11:28:13 -0400 Subject: [PATCH 4/4] Update SWIFTPM_CUSTOM_BIN_DIR in IntegrationTests. --- IntegrationTests/Tests/IntegrationTests/Helpers.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IntegrationTests/Tests/IntegrationTests/Helpers.swift b/IntegrationTests/Tests/IntegrationTests/Helpers.swift index 0fb77e8eabe..082b6698f75 100644 --- a/IntegrationTests/Tests/IntegrationTests/Helpers.swift +++ b/IntegrationTests/Tests/IntegrationTests/Helpers.swift @@ -99,7 +99,7 @@ let lldb: AbsolutePath = { }() let swiftpmBinaryDirectory: AbsolutePath = { - if let environmentPath = ProcessInfo.processInfo.environment["SWIFTPM_BIN_DIR"] { + if let environmentPath = ProcessInfo.processInfo.environment["SWIFTPM_CUSTOM_BIN_DIR"] { return try! AbsolutePath(validating: environmentPath) }