build.gradle
Rama main · réplica de lectura del repositorio original.
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
buildscript {
ext {
shadowGradlePluginVersion = '8.3.5'
}
repositories {
mavenCentral()
maven {
url = 'https://plugins.gradle.org/m2/'
}
}
}
plugins {
id "de.undercouch.download" version "4.0.4"
id "com.dorongold.task-tree" version "2.1.0"
}
apply from: "${projectDir}/x-pack/distributions/internal/observabilitySRE/build-ext.gradle"
apply plugin: 'de.undercouch.download'
apply from: "rubyUtils.gradle"
import de.undercouch.gradle.tasks.download.Download
import groovy.json.JsonSlurper
import org.logstash.gradle.tooling.ListProjectDependencies
import org.logstash.gradle.tooling.SignAliasDefinitions
import org.logstash.gradle.tooling.ToolingUtils
import org.logstash.gradle.tooling.SnapshotArtifactURLs
import java.net.HttpURLConnection
allprojects {
group = 'org.logstash'
apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'java-library'
java {
sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
}
tasks.withType(JavaCompile).configureEach {
options.compilerArgs.add("-Xlint:all")
options.compilerArgs.add("-Xlint:-processing")
options.compilerArgs.add("-Werror")
}
tasks.withType(Javadoc).configureEach {
if (JavaVersion.current().compareTo(JavaVersion.VERSION_14) > 0) {
// with JDK 15 the -Xwerror undocumented feature becomes official with switch -Werror
options.addBooleanOption("Werror", true)
} else {
options.addBooleanOption("Xwerror", true)
}
options.addBooleanOption("Xdoclint:all,-missing", true)
if (JavaVersion.current().compareTo(JavaVersion.VERSION_1_9) > 0) {
options.addBooleanOption("html5", true)
}
}
tasks.withType(Copy).configureEach {
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
}
clean {
delete "${projectDir}/out/"
}
tasks.withType(Test).configureEach {
// Add Exports to enable tests to run in JDK17
jvmArgs = [
"-Djruby.compile.invokedynamic=true",
"-Dlog4j2.isThreadContextMapInheritable=true",
"--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED",
"--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED",
"--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
"--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
"--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
"--add-opens=java.base/java.lang=ALL-UNNAMED",
"--add-opens=java.base/java.util=ALL-UNNAMED"
]
maxHeapSize = "2g"
//https://stackoverflow.com/questions/3963708/gradle-how-to-display-test-results-in-the-console-in-real-time
testLogging {
// set options for log level LIFECYCLE
events "passed", "skipped", "failed", "standardOut"
showExceptions = true
exceptionFormat = "full"
showCauses = true
showStackTraces = true
enableAssertions = false
// set options for log level DEBUG and INFO
debug {
events "started", "passed", "skipped", "failed", "standardOut", "standardError"
exceptionFormat = "full"
}
info.events = debug.events
info.exceptionFormat = debug.exceptionFormat
}
addTestListener(new TestListener() {
void beforeSuite(TestDescriptor suite) {}
void afterSuite(TestDescriptor desc, TestResult result) {
if (!desc.parent) { // will match the outermost suite
def output = "Results: ${result.resultType} (${result.testCount} tests, ${result.successfulTestCount} successes, ${result.failedTestCount} failures, ${result.skippedTestCount} skipped)"
def startItem = '| ', endItem = ' |'
def repeatLength = startItem.length() + output.length() + endItem.length()
println('\n' + ('-' * repeatLength) + '\n' + startItem + output + endItem + '\n' + ('-' * repeatLength))
}
}
void beforeTest(TestDescriptor testDescriptor) {}
void afterTest(TestDescriptor testDescriptor, TestResult result) {}
})
}
}
subprojects {
repositories {
mavenCentral()
maven {
url = 'https://plugins.gradle.org/m2/'
}
}
tasks.register("generateLicenseReport", ListProjectDependencies) {
}
}
version = versionMap['logstash-core']
tasks.register("configureArchitecture") {
String arch = System.properties['os.arch']
String beatsArch = arch
String esArch = arch
String osName = (System.properties['os.name'] ==~ /Mac OS X/) ? "darwin" : "linux"
// For aarch64 architectures, beats and elasticsearch name their artifacts differently
if (arch == "aarch64") {
beatsArch=(osName == "darwin") ? "aarch64" : "arm64"
esArch="aarch64"
} else if (arch == "amd64") {
beatsArch="x86_64"
esArch="x86_64"
}
project.ext.set("beatsArchitecture", "${osName}-${beatsArch}")
project.ext.set("esArchitecture", "${osName}-${esArch}")
}
tasks.register("configureArtifactInfo") {
dependsOn configureArchitecture
description = "Set the url to download stack artifacts for select stack version"
def projectRef = project
doLast {
def splitVersion = version.split('\\.')
int major = splitVersion[0].toInteger()
int minor = splitVersion[1].toInteger()
String branch = "${major}.${minor}"
String fallbackMajorX = "${major}.x"
boolean isFallBackPreviousMajor = minor - 1 < 0
String fallbackBranch = isFallBackPreviousMajor ? "${major-1}.x" : "${major}.${minor-1}"
def qualifiedVersion = ""
for (b in [branch, fallbackMajorX, fallbackBranch]) {
def url = "https://storage.googleapis.com/artifacts-api/snapshots/${b}.json"
try {
def snapshotInfo = new JsonSlurper().parseText(url.toURL().text)
qualifiedVersion = snapshotInfo.version
println "ArtifactInfo version: ${qualifiedVersion}"
break
} catch (Exception e) {
println "Failed to fetch branch ${branch} from ${url}: ${e.message}"
}
}
projectRef.ext.set("artifactApiVersion", qualifiedVersion)
}
}
tasks.register("markAliasDefinitions", SignAliasDefinitions) {
description = "Create an hashes aliases file from original aliases yml definition"
hashedFile = layout.buildDirectory.file("plugin_aliases_hashed.yml").get().asFile
}
tasks.register("markTestAliasDefinitions", SignAliasDefinitions) {
description = "Create an hashes aliases file for testing aliases yml definition"
stage SignAliasDefinitions.Stage.test
hashedFile = layout.buildDirectory.file("plugin_aliases_hashed_test.yml").get().asFile
}
tasks.register("copyPluginAlias", Copy) {
description = "Copy the marked plugin_aliases.yml file to destination folders"
dependsOn = [copyPluginAlias_ruby, copyPluginAlias_java]
}
tasks.register("copyPluginAlias_ruby", Copy) {
description = "Copy the marked plugin_aliases.yml file to destination folders"
dependsOn "markAliasDefinitions"
def aliasFile = layout.buildDirectory.file("plugin_aliases_hashed.yml")
inputs.file(aliasFile)
from(aliasFile) {
rename "plugin_aliases_hashed.yml", "plugin_aliases.yml"
}
into "lib/pluginmanager/"
}
tasks.register("copyPluginAlias_java", Copy) {
description = "Copy the marked plugin_aliases.yml file to destination folders"
dependsOn "markAliasDefinitions"
def aliasFile = layout.buildDirectory.file("plugin_aliases_hashed.yml")
inputs.file(aliasFile)
from(aliasFile) {
rename "plugin_aliases_hashed.yml", "plugin_aliases.yml"
}
into "logstash-core/src/main/resources/org/logstash/plugins/"
}
tasks.register("copyPluginTestAlias") {
description = "Copy the marked test plugin_aliases.yml file to destination folders"
dependsOn = [copyPluginTestAlias_ruby, copyPluginTestAlias_java]
}
tasks.register("copyPluginTestAlias_ruby", Copy) {
description = "Copy the marked test plugin_aliases.yml file into Ruby's plugin_manager specs"
dependsOn "markTestAliasDefinitions"
def aliasTestFile = layout.buildDirectory.file("plugin_aliases_hashed_test.yml")
inputs.file(aliasTestFile)
from(aliasTestFile) {
rename "plugin_aliases_hashed_test.yml", "plugin_aliases.yml"
}
into "spec/unit/plugin_manager/"
}
tasks.register("copyPluginTestAlias_java", Copy) {
description = "Copy the marked test plugin_aliases.yml file into logstash-core's test resources"
dependsOn "markTestAliasDefinitions"
def aliasTestFile = layout.buildDirectory.file("plugin_aliases_hashed_test.yml")
inputs.file(aliasTestFile)
from(aliasTestFile) {
rename "plugin_aliases_hashed_test.yml", "plugin_aliases.yml"
}
into "logstash-core/src/test/resources/org/logstash/plugins/"
}
project(':logstash-core').tasks.named('processResources').configure { dependsOn(copyPluginAlias) }
project(':logstash-core').tasks.named('processTestResources').configure { dependsOn(copyPluginTestAlias) }
// Tasks
tasks.register("printStackVersion") {
description = "Print the stack version from logstash-plugins CI"
doLast {
def releaseTrack = gradle.ext.versions['logstash-release-track']
def url = new URL("https://raw.githubusercontent.com/logstash-plugins/.ci/1.x/logstash-versions.yml")
def remoteVersions = new org.yaml.snakeyaml.Yaml().load(url.text)
println remoteVersions['snapshots'][releaseTrack]
}
}
clean {
delete "${projectDir}/Gemfile"
delete "${projectDir}/Gemfile.lock"
delete "${projectDir}/vendor"
delete "${projectDir}/.bundle"
delete "${projectDir}/qa/integration/Gemfile.lock"
delete "${projectDir}/qa/integration/.bundle"
delete "${projectDir}/build/licenseReportFolders.txt"
delete "${projectDir}/build/rubyDependencies.csv"
delete "${projectDir}/lib/pluginmanager/plugin_aliases.yml"
delete "${projectDir}/spec/unit/plugin_manager/plugin_aliases.yml"
delete "${projectDir}/logstash-core/build/resources/test/org/logstash/plugins/plugin_aliases.yml"
delete "${projectDir}/logstash-core/build/resources/main/org/logstash/plugins/plugin_aliases.yml"
delete "${projectDir}/logstash-core/src/test/resources/org/logstash/plugins/plugin_aliases.yml"
delete "${projectDir}/logstash-core/src/main/resources/org/logstash/plugins/plugin_aliases.yml"
}
def assemblyDeps = [downloadAndInstallJRuby, assemble] + subprojects.collect {
it.tasks.findByName("assemble")
}
tasks.register("bootstrap") {
dependsOn assemblyDeps
doLast {
setupJruby(projectDir, buildDir)
}
}
tasks.register("dockerBootstrap") {
description = "Docker bootstrap ensures env2yaml java is compiled and staged for inclusion in tarballs"
dependsOn bootstrap
dependsOn ':docker:data:logstash:env2yaml:compileJava'
}
tasks.register("installDefaultGems") {
dependsOn bootstrap
doLast {
rake(projectDir, buildDir, 'plugin:install-default')
}
}
tasks.register("installDevelopmentGems") {
dependsOn bootstrap
doLast {
rake(projectDir, buildDir, 'plugin:install-development-dependencies')
}
}
tasks.register("artifactDocker") {
description = "Build docker image"
dependsOn dockerBootstrap
dependsOn copyAllJdks
doLast {
rake(projectDir, buildDir, 'artifact:docker')
}
}
tasks.register("artifactDockerOss") {
description = "Build OSS docker image"
dependsOn dockerBootstrap
dependsOn copyAllJdks
doLast {
rake(projectDir, buildDir, 'artifact:docker_oss')
}
}
tasks.register("artifactDockerWolfi") {
description = "Build Wolfi docker image"
dependsOn dockerBootstrap
dependsOn copyAllJdks
doLast {
rake(projectDir, buildDir, 'artifact:docker_wolfi')
}
}
tasks.register("artifactDockerIronbank") {
description = "Build Ironbank docker image"
dependsOn dockerBootstrap
dependsOn copyAllJdks
doLast {
rake(projectDir, buildDir, 'artifact:docker_ironbank')
}
}
tasks.register("artifactDockerfiles") {
description = "Generate all Dockerfiles and build contexts"
doLast {
rake(projectDir, buildDir, 'artifact:dockerfiles')
}
}
tasks.register("artifactAll", Exec) {
description = "Build all artifacts except docker"
dependsOn bootstrap
dependsOn copyAllJdks
workingDir projectDir
environment 'GEM_HOME', "${projectDir}/vendor/bundle/jruby/3.4.0"
environment 'GEM_PATH', "${projectDir}/vendor/bundle/jruby/3.4.0"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "rake", "artifact:all"
}
tasks.register("artifactDeb", Exec) {
description = "Build DEB package"
dependsOn bootstrap
dependsOn copyAllJdks
workingDir projectDir
environment 'ARCH', System.getenv("ARCH") ?: "x86_64"
environment 'GEM_HOME', "${projectDir}/vendor/bundle/jruby/3.4.0"
environment 'GEM_PATH', "${projectDir}/vendor/bundle/jruby/3.4.0"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "rake", "artifact:deb"
}
tasks.register("artifactRpm", Exec) {
description = "Build RPM package"
dependsOn bootstrap
dependsOn copyAllJdks
workingDir projectDir
environment 'ARCH', System.getenv("ARCH") ?: "x86_64"
environment 'GEM_HOME', "${projectDir}/vendor/bundle/jruby/3.4.0"
environment 'GEM_PATH', "${projectDir}/vendor/bundle/jruby/3.4.0"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "rake", "artifact:rpm"
}
tasks.register("compileGrammar") {
dependsOn bootstrap
doLast {
rake(projectDir, buildDir, 'compile:grammar')
}
}
tasks.register("artifactDockerOnly") {
description = "Build docker image without OSS"
dependsOn dockerBootstrap
dependsOn copyAllJdks
doLast {
rake(projectDir, buildDir, 'artifact:docker_only')
}
}
tasks.register("generatePluginsVersion") {
description = "Generate plugins version documentation"
dependsOn bootstrap
dependsOn installDefaultGems
doLast {
rake(projectDir, buildDir, 'generate_plugins_version')
}
}
tasks.register("artifactDockerObservabilitySRE") {
dependsOn bootstrap
dependsOn "copyAllJdks"
inputs.files fileTree("${projectDir}/rakelib")
inputs.files fileTree("${projectDir}/bin")
inputs.files fileTree("${projectDir}/config")
inputs.files fileTree("${projectDir}/lib")
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
inputs.files fileTree("${projectDir}/logstash-core/lib")
inputs.files fileTree("${projectDir}/logstash-core/src")
inputs.files fileTree("${projectDir}/x-pack")
outputs.files fileTree("${buildDir}") {
include "Dockerfile-observability-sre"
include "logstash-observability-sre-${project.version}-SNAPSHOT-linux-*.tar.gz"
include "logstash-observability-sre-${project.version}-SNAPSHOT-docker-build-context.tar.gz"
include "plugin_aliases_hashed.yml"
include "jdk-*-linux-*.tar.gz"
}
doFirst {
if (!fedrampHighMode) {
logger.error("NOT in Fedramp High mode. Aborting.")
throw new GradleException("cannot build docker artifact for observabilitySRE without `-PfedrampHighMode=true`")
}
}
doLast {
rake(projectDir, buildDir, 'artifact:docker_observabilitySRE')
}
}
tasks.register("assembleTarDistribution") {
dependsOn bootstrap
dependsOn "copyAllJdks"
inputs.files fileTree("${projectDir}/rakelib")
inputs.files fileTree("${projectDir}/bin")
inputs.files fileTree("${projectDir}/config")
inputs.files fileTree("${projectDir}/lib")
inputs.files fileTree("${projectDir}/logstash-core-plugin-api")
inputs.files fileTree("${projectDir}/logstash-core/lib")
inputs.files fileTree("${projectDir}/logstash-core/src")
inputs.files fileTree("${projectDir}/x-pack")
outputs.files file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz")
doLast {
rake(projectDir, buildDir, 'artifact:bundle_jdk_tar')
}
}
evaluationDependsOn(":logstash-core")
project(":logstash-core") {
["rubyTests", "test"].each { tsk ->
tasks.getByPath(":logstash-core:" + tsk).configure {
dependsOn copyPluginTestAlias
dependsOn installDevelopmentGems
dependsOn installDefaultGems
}
}
}
def logstashBuildDir = "${buildDir}/logstash-${project.version}-SNAPSHOT"
tasks.register("unpackTarDistribution", Copy) {
dependsOn assembleTarDistribution
def tar = file("${buildDir}/logstash-${project.version}-SNAPSHOT.tar.gz")
inputs.files tar
outputs.files fileTree(logstashBuildDir)
from tarTree(tar)
into {buildDir}
}
def qaBuildPath = "${buildDir}/qa/integration"
def qaVendorPath = "${qaBuildPath}/vendor"
tasks.register("installIntegrationTestGems") {
dependsOn assembleTarDistribution
dependsOn unpackTarDistribution
def gemfilePath = file("${projectDir}/qa/integration/Gemfile")
inputs.files gemfilePath
inputs.files file("${projectDir}/qa/integration/integration_tests.gemspec")
inputs.files file("${logstashBuildDir}/Gemfile")
inputs.files file("${logstashBuildDir}/Gemfile.lock")
inputs.files file("${logstashBuildDir}/logstash-core/logstash-core.gemspec")
outputs.files fileTree("${qaVendorPath}")
outputs.files file("${projectDir}/qa/integration/Gemfile.lock")
doLast {
bundleQAGems(projectDir, qaBuildPath)
}
}
tasks.register("prepareFilebeatDownload") {
dependsOn configureArtifactInfo
def projectRef = project
doLast {
String beatsVersion = projectRef.ext.get("artifactApiVersion")
String downloadedFilebeatName = "filebeat-${beatsVersion}-${projectRef.ext.get("beatsArchitecture")}"
projectRef.ext.set("unpackedFilebeatName", downloadedFilebeatName)
def res = SnapshotArtifactURLs.packageUrls("beats", beatsVersion, downloadedFilebeatName)
projectRef.ext.set("filebeatSnapshotUrl", System.getenv("FILEBEAT_SNAPSHOT_URL") ?: res.packageUrl)
projectRef.ext.set("filebeatDownloadLocation", "${projectDir}/build/${downloadedFilebeatName}.tar.gz")
}
}
def verifyPackageSHA(String artifactProject, String version, String packageName) {
def res = SnapshotArtifactURLs.packageUrls(artifactProject, version, packageName)
String remoteSHACode = res.packageShaUrl.toURL().text.split(' ')[0]
def localArchive = new File("${projectDir}/build/${packageName}.tar.gz")
if (localArchive.exists()) {
ant.checksum(file: localArchive, algorithm: "SHA-512", forceoverwrite: true)
String localSHA = new File("${projectDir}/build/${packageName}.tar.gz.SHA-512").text.trim()
if (localSHA != remoteSHACode) {
println "${artifactProject} package calculated fingerprint is different from remote, deleting local archive"
delete(localArchive)
} else {
println "Local ${artifactProject} package is already the latest"
}
}
}
tasks.register("checkFilebeatSHA") {
dependsOn configureArtifactInfo
description = "Download Filebeat version remote's fingerprint file"
def projectRef = project
doLast {
String beatsVersion = projectRef.ext.get("artifactApiVersion")
String downloadedFilebeatName = "filebeat-${beatsVersion}-${projectRef.ext.get("beatsArchitecture")}"
verifyPackageSHA("beats", beatsVersion, downloadedFilebeatName)
}
}
tasks.register("downloadFilebeat", Download) {
dependsOn prepareFilebeatDownload, checkFilebeatSHA
description = "Download Filebeat Snapshot for current branch version: ${version}"
project.ext.set("versionFound", true)
inputs.file("${projectDir}/versions.yml")
def projectRef = project
// Use lazy configuration to get values after prepareFilebeatDownload runs
src { projectRef.ext.filebeatSnapshotUrl }
dest { new File(projectRef.ext.filebeatDownloadLocation) }
onlyIfNewer true
retries 3
doLast {
System.out.println "Downloaded to ${projectRef.ext.filebeatDownloadLocation}"
}
}
tasks.register("deleteLocalFilebeat", Delete) {
delete ('./build/filebeat')
}
tasks.register("copyFilebeat") {
dependsOn downloadFilebeat
inputs.files(tasks.named("downloadFilebeat"))
outputs.dir('./build/filebeat')
mustRunAfter tasks.named("unpackTarDistribution")
doLast {
delete('./build/filebeat')
copy {
from tarTree(resources.gzip(project.ext.filebeatDownloadLocation))
into "./build/"
}
file("./build/${project.ext.unpackedFilebeatName}").renameTo('./build/filebeat')
System.out.println "Unzipped ${project.ext.filebeatDownloadLocation} to ./build/filebeat"
}
}
tasks.register("checkEsSHA") {
dependsOn configureArtifactInfo
description = "Download ES version remote's fingerprint file"
def projectRef = project
doLast {
String esVersion = projectRef.ext.get("artifactApiVersion")
String downloadedElasticsearchName = "elasticsearch-${esVersion}-${projectRef.ext.get("esArchitecture")}"
verifyPackageSHA("elasticsearch", esVersion, downloadedElasticsearchName)
}
}
tasks.register("prepareEsDownload") {
dependsOn configureArtifactInfo
def projectRef = project
doLast {
String esVersion = projectRef.ext.get("artifactApiVersion")
String downloadedElasticsearchName = "elasticsearch-${esVersion}-${projectRef.ext.get("esArchitecture")}"
projectRef.ext.set("unpackedElasticsearchName", "elasticsearch-${esVersion}")
def res = SnapshotArtifactURLs.packageUrls("elasticsearch", esVersion, downloadedElasticsearchName)
projectRef.ext.set("elasticsearchSnapshotURL", System.getenv("ELASTICSEARCH_SNAPSHOT_URL") ?: res.packageUrl)
projectRef.ext.set("elasticsearchDownloadLocation", "${projectDir}/build/${downloadedElasticsearchName}.tar.gz")
}
}
tasks.register("downloadEs", Download) {
dependsOn = [prepareEsDownload, checkEsSHA]
description = "Download ES Snapshot for current branch version: ${version}"
inputs.file("${projectDir}/versions.yml")
def projectRef = project
src { projectRef.ext.elasticsearchSnapshotURL }
dest { new File(projectRef.ext.elasticsearchDownloadLocation) }
onlyIfNewer true
overwrite false
retries 3
doLast {
System.out.println "Downloaded to ${projectRef.ext.elasticsearchDownloadLocation}"
}
}
tasks.register("deleteLocalEs", Delete) {
delete ('./build/elasticsearch')
}
tasks.register("copyEs") {
dependsOn downloadEs
inputs.files(tasks.named("downloadEs"))
outputs.dir('./build/elasticsearch')
mustRunAfter tasks.named("unpackTarDistribution")
doLast {
println "copyEs executing.."
delete('./build/elasticsearch')
copy {
from tarTree(resources.gzip(project.ext.elasticsearchDownloadLocation))
into "./build/"
}
file("./build/${project.ext.unpackedElasticsearchName}").renameTo('./build/elasticsearch')
println "Unzipped ${project.ext.elasticsearchDownloadLocation} to ./build/elasticsearch"
}
}
def rubyIntegrationSpecs = project.hasProperty("rubyIntegrationSpecs") ? ((String) project.property("rubyIntegrationSpecs")).split(/\s+/).join(",") : "specs/**/*_spec.rb"
def integrationTestPwd = "${projectDir}/qa/integration"
evaluationDependsOn(":logstash-integration-tests")
project(":logstash-integration-tests") {
tasks.getByPath(":logstash-integration-tests:integrationTests").configure {
systemProperty 'org.logstash.integration.specs', rubyIntegrationSpecs
environment "FEATURE_FLAG", System.getenv('FEATURE_FLAG')
workingDir integrationTestPwd
dependsOn installIntegrationTestGems
dependsOn copyProductionLog4jConfiguration
}
}
tasks.register("runIntegrationTests") {
dependsOn ":logstash-integration-tests:integrationTests"
dependsOn copyEs
dependsOn copyFilebeat
shouldRunAfter ":logstash-core:test"
}
tasks.register("installAcceptanceTestGems", Exec) {
dependsOn bootstrap
inputs.file("${projectDir}/qa/Gemfile")
outputs.file("${projectDir}/qa/Gemfile.lock")
workingDir "${projectDir}/qa"
environment "BUNDLE_PATH", "vendor/bundle"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "install"
}
tasks.register("runAcceptanceTests", Exec) {
dependsOn installAcceptanceTestGems
workingDir "${projectDir}/qa"
environment "BUNDLE_PATH", "vendor/bundle"
// NOTE: We sub-process here to avoid issues in java 21+ with Jruby whereby access to some modules that
// are required for file descriptors to be properly inherited when ruby does sub-processing.
// https://github.com/elastic/logstash/pull/18536#discussion_r2655013858
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "exec", "rake", "qa:acceptance:all"
}
// Docker acceptance tests - reuses installAcceptanceTestGems for gem installation
tasks.register("runDockerOssTests", Exec) {
description = "Run OSS docker acceptance tests"
dependsOn installAcceptanceTestGems
dependsOn artifactDockerOss
workingDir "${projectDir}/qa"
environment "BUNDLE_PATH", "vendor/bundle"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "exec", "rspec", "docker/spec/oss"
}
tasks.register("runDockerFullTests", Exec) {
description = "Run full docker acceptance tests"
dependsOn installAcceptanceTestGems
dependsOn artifactDocker
workingDir "${projectDir}/qa"
environment "BUNDLE_PATH", "vendor/bundle"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "exec", "rspec", "docker/spec/full"
}
tasks.register("runDockerWolfiTests", Exec) {
description = "Run Wolfi docker acceptance tests"
dependsOn installAcceptanceTestGems
dependsOn artifactDockerWolfi
workingDir "${projectDir}/qa"
environment "BUNDLE_PATH", "vendor/bundle"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "exec", "rspec", "docker/spec/wolfi"
}
tasks.register("runAllDockerTests", Exec) {
description = "Run all docker acceptance tests"
dependsOn installAcceptanceTestGems
dependsOn artifactDockerOnly
workingDir "${projectDir}/qa"
environment "BUNDLE_PATH", "vendor/bundle"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "exec", "rspec", "docker/spec"
}
tasks.register("runDockerIronbankTests", Exec) {
description = "Run Ironbank docker acceptance tests"
dependsOn installAcceptanceTestGems
dependsOn artifactDockerIronbank
workingDir "${projectDir}/qa"
environment "BUNDLE_PATH", "vendor/bundle"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "exec", "rspec", "docker/spec/ironbank"
}
// Serverless integration tests
tasks.register("runServerlessDLQTests", Exec) {
description = "Run serverless DLQ integration tests"
dependsOn installIntegrationTestGems
dependsOn unpackTarDistribution
dependsOn ':logstash-core:copyGemjar'
workingDir "${projectDir}/qa/integration"
environment "BUNDLE_PATH", "${buildDir}/qa/integration/vendor"
environment "SERVERLESS", "true"
commandLine "${projectDir}/vendor/jruby/bin/jruby", "-S", "bundle", "exec", "rspec",
"-fd", "specs/dlq_spec.rb", "-e", "using pipelines.yml"
}
tasks.register("generateLicenseReport", JavaExec) {
dependsOn generateLicenseReportInputs
dependsOn ":dependencies-report:assemble"
def jarFile = project('dependencies-report').layout.buildDirectory.dir("libs").get().asFile.toString() + "/dependencies-report.jar"
String licenseReportInputCSV = project.hasProperty("licenseReportInputCSV") ? project.property("licenseReportInputCSV") : "build/dependencies.csv.ruby"
String licenseReportOutputCSV = project.hasProperty("licenseReportOutputCSV") ? project.property("licenseReportOutputCSV") : "build/dependencies.csv"
String noticePath = "NOTICE.txt"
classpath = project.files([jarFile])
mainClass = "org.logstash.dependencies.Main"
args licenseReportInputCSV,
layout.buildDirectory.file("licenseReportFolders.txt").get().asFile.toString(),
licenseReportOutputCSV, noticePath
}
tasks.register("generateLicenseReportInputs") {
dependsOn subprojects.generateLicenseReport
doLast {
// write location of all license reports for subprojects containing artifacts that are distributed to single file
StringBuilder licenseReportFolders = new StringBuilder()
subprojects.findAll { s1 -> !s1.hasProperty("isDistributedArtifact") || s1.property("isDistributedArtifact") == 'true' }.each { s ->
s.tasks.findAll { t2 -> t2.getName() == "generateLicenseReport" }.each { t3 ->
licenseReportFolders.append(t3.reportDir.toString() + "\n")
}
}
if (gradle.startParameter.taskNames.contains("generateLicenseReport")) {
def buildDirFile = layout.buildDirectory.get().asFile
if (!buildDirFile.exists()) {
buildDirFile.mkdirs()
}
def licenseReportPath = buildDirFile.toString() + "/licenseReportFolders.txt"
def licenseReportFolder = new File(licenseReportPath)
licenseReportFolder.delete()
licenseReportFolder = new File(licenseReportPath)
licenseReportFolder.createNewFile()
if (licenseReportFolder.canWrite()) {
licenseReportFolder.text = licenseReportFolders.toString()
}
}
}
}
bootstrap.dependsOn assemblyDeps
// FIXME: adding the integration tests task to check will mean
// that any registered task will be evaluated. This creates an issue
// where the downloadES task may throw an error on versions where
// Elasticsearch doesn't yet have a build we can fetch
// So for now we'll remove this to unblock builds, but finding a way
// to compartimentalize failures is needed going forward
//check.dependsOn runIntegrationTest
def selectArch() {
// 1. Check Gradle project property (e.g., -Pjdk_arch=arm64)
if (project.ext.has("jdk_arch")) {
return project.ext.jdk_arch
}
// 2. Check ARCH environment variable (used in CI for cross-compilation)
// Matches rake pattern: x86_64 → x86_64, anything else (aarch64, arm64, etc.) → arm64
String envArch = System.getenv("ARCH")
if (envArch) {
return (envArch == "x86_64") ? "x86_64" : "arm64"
}
// 3. Fall back to host architecture
String cpu_arch = System.properties["os.arch"]
switch (cpu_arch) {
case "amd64":
case "x86_64":
return "x86_64"
case "aarch64":
case "arm64":
return "arm64"
default:
throw new IllegalArgumentException("Can't handle os.arch of type $cpu_arch")
}
}
class JDKDetails {
final String revision
final String build
final String vendor
final int major
private final String osName
private final String extension
final String localPackageName
final String unpackedJdkName
private String arch
JDKDetails(bundledJdk, osName, jdkArch) {
revision = bundledJdk.revision
build = bundledJdk.build
vendor = bundledJdk.vendor
major = revision.split('\\.').first() as int
this.osName = osName
switch (osName) {
case "windows":
extension = "zip"
break
default:
extension = "tar.gz"
}
arch = parseJdkArchitecture(jdkArch)
unpackedJdkName = "jdk-${revision}-${osName}"
localPackageName = "${unpackedJdkName}-${arch}.${extension}"
}
String createDownloadUrl() {
return createElasticCatalogDownloadUrl()
}
private String createElasticCatalogDownloadUrl() {
// Ask details to catalog https://jvm-catalog.elastic.co/jdk and return the url to download the JDK
// arch x86_64 is default, aarch64 if macos or linux
def url = "https://jvm-catalog.elastic.co/jdk/adoptiumjdk-${revision}+${build}-${osName}"
// Append the cpu's arch only if not x86_64, which is the default
if (arch == "aarch64") {
url += "-${arch}"
}
println "Retrieving JDK location URL from Elastic catalog..."
def catalogMetadataUrl = URI.create(url).toURL()
int maxAttempts = 5
long retryDelayMillis = 3000
int connectTimeoutMillis = 15000
int readTimeoutMillis = 60000
int lastResponseCode = -1
String lastResponseMessage = null
String lastErrorBody = null
Exception lastException = null
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
HttpURLConnection catalogConnection = null
try {
catalogConnection = catalogMetadataUrl.openConnection() as HttpURLConnection
catalogConnection.requestMethod = 'GET'
catalogConnection.connectTimeout = connectTimeoutMillis
catalogConnection.readTimeout = readTimeoutMillis
int responseCode = catalogConnection.responseCode
if (responseCode == 200) {
def metadataRetrieved = catalogConnection.inputStream.withCloseable { stream -> stream.getText('UTF-8') }
println "Retrieved!"
def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved)
validateMetadata(catalogMetadata)
return catalogMetadata.url
}
lastResponseCode = responseCode
lastResponseMessage = catalogConnection.responseMessage
lastErrorBody = catalogConnection.errorStream?.withCloseable { stream -> stream.getText('UTF-8') }?.trim()
lastException = null
def statusMessage = lastResponseMessage ? "${responseCode} ${lastResponseMessage}" : "${responseCode}"
if (lastErrorBody) {
statusMessage += ": ${lastErrorBody.take(200)}"
}
if (attempt < maxAttempts) {
println "Attempt ${attempt} of ${maxAttempts} failed with HTTP ${statusMessage}. Retrying..."
try {
Thread.sleep(retryDelayMillis)
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt()
throw new GradleException("Interrupted while retrying Elastic JVM catalog metadata request", interruptedException)
}
}
} catch (Exception e) {
lastException = e
lastResponseCode = -1
lastResponseMessage = null
lastErrorBody = null
if (attempt < maxAttempts) {
println "Attempt ${attempt} of ${maxAttempts} failed: ${e.message}. Retrying..."
try {
Thread.sleep(retryDelayMillis)
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt()
throw new GradleException("Interrupted while retrying Elastic JVM catalog metadata request", interruptedException)
}
}
} finally {
catalogConnection?.disconnect()
}
}
def requestedArtifact = "adoptiumjdk-${revision}+${build}-${osName}${arch == "aarch64" ? "-${arch}" : ""}"
if (lastResponseCode == 404) {
throw new GradleException("Elastic JVM catalog returned 404 for ${requestedArtifact}. Verify the artifact exists.", lastException)
}
if (lastResponseCode != -1) {
def statusMessage = lastResponseMessage ? "${lastResponseCode} ${lastResponseMessage}" : "${lastResponseCode}"
if (lastErrorBody) {
statusMessage += " (${lastErrorBody.take(200)})"
}
throw new GradleException("Failed to fetch Elastic JVM catalog metadata for ${requestedArtifact}. Last HTTP response: ${statusMessage}", lastException)
}
if (lastException != null) {
throw new GradleException("Failed to fetch Elastic JVM catalog metadata for ${requestedArtifact}: ${lastException.message}", lastException)
}
throw new GradleException("Failed to fetch Elastic JVM catalog metadata for ${requestedArtifact}.")
}
//Verify that the artifact metadata correspond to the request, if not throws an error
private void validateMetadata(Map metadata) {
if (metadata.version != revision) {
throw new GradleException("Expected to retrieve a JDK for version ${revision} but received: ${metadata.version}")
}
if (!isSameArchitecture(metadata.architecture)) {
throw new GradleException("Expected to retrieve a JDK for architecture ${arch} but received: ${metadata.architecture}")
}
}
private boolean isSameArchitecture(String metadataArch) {
if (arch == 'x64') {
return metadataArch == 'x86_64'
}
return metadataArch == arch
}
private String parseJdkArchitecture(String jdkArch) {
switch (jdkArch) {
case "x86_64":
return "x64"
break
case "arm64":
return "aarch64"
break
default:
throw new GradleException("Can't handle CPU architechture: ${jdkArch}")
}
}
}
tasks.register("lint") {
description = "Lint Ruby source files. Use -PrubySource=file1.rb,file2.rb to specify files"
dependsOn installDevelopmentGems
doLast {
if (project.hasProperty("rubySource")) {
// Split the comma-separated files and pass them as separate arguments
def files = project.property("rubySource").split(",")
rake(projectDir, buildDir, "lint:report", *files)
} else {
rake(projectDir, buildDir, "lint:report")
}
}
}
def getTargetPlatforms() {
String arch = selectArch()
if (arch == "x86_64") {
return [[os: "linux", arch: "x86_64"], [os: "windows", arch: "x86_64"], [os: "darwin", arch: "x86_64"]]
} else {
return [[os: "linux", arch: "arm64"], [os: "darwin", arch: "arm64"]]
}
}
["linux", "windows", "darwin"].each { osName ->
["x86_64", "arm64"].each { archName ->
if (osName == "windows" && archName == "arm64") { return }
String taskSuffix = "${osName.capitalize()}${archName == 'x86_64' ? 'X64' : 'Arm64'}"
String jdkFolderName = ToolingUtils.jdkFolderName(osName, archName)
tasks.register("downloadJdk${taskSuffix}", Download) {
def jdkDetails = new JDKDetails(gradle.ext.versions.bundled_jdk, osName, archName)
// Use provider for lazy evaluation - only resolve URL when task executes
src provider { System.getenv("JDK_URL") ?: jdkDetails.createDownloadUrl() }
onlyIfNewer true
overwrite false
retries 3
quiet true
inputs.file("${projectDir}/versions.yml")
dest new File("${projectDir}/build/${jdkDetails.localPackageName}")
doFirst { mkdir "${projectDir}/${jdkFolderName}" }
}
if (osName == "windows") {
tasks.register("unpackJdk${taskSuffix}", Copy) {
dependsOn "downloadJdk${taskSuffix}"
def jdkDetails = new JDKDetails(gradle.ext.versions.bundled_jdk, osName, archName)
String rootName = null
from(zipTree("${projectDir}/build/${jdkDetails.localPackageName}")) {
eachFile { fcd ->
rootName = rootName ?: fcd.relativePath.segments[0]
fcd.relativePath = new RelativePath(true, fcd.relativePath.segments.drop(1))
}
}
into "${projectDir}/${jdkFolderName}"
doFirst { delete "${projectDir}/${jdkFolderName}" }
doLast { delete "${projectDir}/${jdkFolderName}/$rootName" }
}
} else {
tasks.register("unpackJdk${taskSuffix}", Exec) {
dependsOn "downloadJdk${taskSuffix}"
def jdkDetails = new JDKDetails(gradle.ext.versions.bundled_jdk, osName, archName)
commandLine 'tar', 'xf', "${projectDir}/build/${jdkDetails.localPackageName}", '-C', "${projectDir}/${jdkFolderName}", '--strip-components', '1'
doFirst { mkdir "${projectDir}/${jdkFolderName}" }
}
}
}
}
tasks.register("copyAllJdks") {
dependsOn bootstrap
getTargetPlatforms().each { platform ->
String taskSuffix = "${platform.os.capitalize()}${platform.arch == 'x86_64' ? 'X64' : 'Arm64'}"
dependsOn "unpackJdk${taskSuffix}"
}
doLast {
String arch = selectArch()
String jdkFolder = ToolingUtils.jdkFolderName("linux", arch)
File releaseFile = file("${projectDir}/${jdkFolder}/release")
if (releaseFile.exists()) {
def content = releaseFile.text
def matcher = content =~ /IMPLEMENTOR_VERSION="([^"]+)"/
if (matcher.find()) {
file("${projectDir}/JDK_VERSION").write(matcher.group(1))
}
}
}
}
tasks.register("deleteAllJdks", Delete) {
delete fileTree(projectDir) { include 'jdk-*' }
}
tasks.register("javaTests") {
dependsOn ":logstash-core:javaTests"
dependsOn ":jvm-options-parser:test"
}
clean {
dependsOn deleteAllJdks
delete "${projectDir}/JDK_VERSION"
delete fileTree(projectDir) {
include 'output*.csv'
include 'output*.json'
}
delete fileTree("${projectDir}/.buildkite/scripts/snyk/artifact-scan") {
include 'output*.csv'
include 'output*.json'
}
}
tasks.register("extractArtifactVersions") {
description = "Extract versions from a Logstash artifact (generates SBOM for Snyk scanning)"
dependsOn bootstrap
doLast {
def artifactDir = project.findProperty('artifactDir')
def outputFile = project.findProperty('outputFile') ?: 'output.csv'
if (!artifactDir) {
throw new GradleException("artifactDir property required. Usage: ./gradlew extractArtifactVersions -PartifactDir=/path/to/logstash -PoutputFile=output.csv")
}
rake(projectDir, buildDir, 'extract:artifact_versions', artifactDir, outputFile)
}
}
tasks.register("cleanExtraction") {
description = "Clean artifact extraction output files"
doLast {
rake(projectDir, buildDir, 'extract:clean')
}
}
if (System.getenv('OSS') != 'true') {
evaluationDependsOn(":logstash-xpack")
project(":logstash-xpack") {
["rubyTests", "rubyIntegrationTests", "test"].each { tsk ->
tasks.getByPath(":logstash-xpack:" + tsk).configure {
dependsOn installDevelopmentGems
dependsOn installDefaultGems
}
}
}
}
tasks.register("runXPackUnitTests") {
dependsOn copyPluginTestAlias
dependsOn ":logstash-xpack:rubyTests"
}
tasks.register("runXPackIntegrationTests") {
dependsOn copyPluginTestAlias
dependsOn ":logstash-xpack:rubyIntegrationTests"
}