Multi-module project phantom dependencies

Steve_Yegge · February 21, 2022, 10:50pm

Since upgrading to Gradle 7.x, my build has been reporting task dependencies between subprojects that are essentially wholly unrelated and share no code.

I do not get the warnings when building the subprojects individually; only when I run the build at the top level. Here are two examples (out of 174 total), in which the build claims that my ‘server’ subproject tests depend on the ‘web’ subproject tests (and war) tasks:

  - Gradle detected a problem with the following location:
    '/Users/stevey/wyvern'. Reason: Task ':wyvern-server:test' uses
    this output of task ':wyvern-web:test' without declaring an
    explicit or implicit dependency. This can lead to incorrect
    results being produced, depending on what order the tasks are
    executed. Please refer to
    https://docs.gradle.org/7.4/userguide/validation_problems.html#implicit_dependency
    for more details about this problem.

  - Gradle detected a problem with the following location:
    '/Users/stevey/wyvern'. Reason: Task ':wyvern-server:test' uses
    this output of task ':wyvern-web:war' without declaring an
    explicit or implicit dependency. This can lead to incorrect
    results being produced, depending on what order the tasks are
    executed. Please refer to
    https://docs.gradle.org/7.4/userguide/validation_problems.html#implicit_dependency
    for more details about this problem.

I will include the three relevant build scripts (root, ‘server’, and ‘web’) here in all their ugly glory. Apologies that they are such a mess.

root build.gradle:

buildscript {
   ext.kotlin_version = '1.5.31'
   ext.kotlin_jdk7_version = '1.5.31'
   ext.kotlin_jdk8_version = '1.5.31'

   ext.wyvernHome = new File(System.getProperty("user.home"), "wyvern").toString()
   ext.jvm_target = "15"

   repositories {
     jcenter()
     mavenLocal()
     mavenCentral()
     maven { url "https://mvnrepository.com/artifact/com.google.guava/guava" }
   }

   dependencies {
     classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
   }
}
apply plugin: "kotlin"

// https://youtrack.jetbrains.com/issue/KT-16520#focus=streamItem-27-4128145.0-0
// Note:  We also need to enable the new IR at runtime; not sure how to do that.
compileKotlin {
  kotlinOptions {
    useIR = true
    freeCompilerArgs += "-XXLanguage:+ProperArrayConventionSetterWithDefaultCalls"
    jvmTarget = jvm_target
  }
}
tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach {
    kotlinOptions {
      jvmTarget = jvm_target
      useIR = true
      freeCompilerArgs += "-XXLanguage:+ProperArrayConventionSetterWithDefaultCalls"
    }
}
compileTestKotlin {
    kotlinOptions.jvmTarget = jvm_target
}

allprojects {
  apply plugin: "java"
  sourceCompatibility = jvm_target
  targetCompatibility = jvm_target
  apply plugin: "application"
  mainClassName = ""

  apply plugin: "idea"

  repositories {
    google()
    mavenCentral()
    maven { url "https://repo1.maven.org/maven2" }
  }

  // Top-level dependencies are included by all submodules, so be frugal here.
  dependencies {
    implementation "org.apache.commons:commons-lang3:3.9"
    implementation group: "commons-io", name: "commons-io", version:"2.4"
    implementation group: "commons-codec", name: "commons-codec", version:"1.11"

    implementation "com.google.guava:guava:28.2-jre"

    // TODO:  Remove this.  We're already on JUnit 5 in some places.
    implementation group: "junit", name: "junit", version:"4.12"
    testImplementation group: "junit", name: "junit", version:"4.12"

    implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"

    implementation "javax.annotation:javax.annotation-api:1.3.2"
  }

  compileJava.options.encoding = "UTF-8"

  tasks.withType(JavaCompile) {
    options.encoding = "UTF-8"
  }
}

subprojects {
  group = "wyvern"
  version = "1.0-SNAPSHOT"
  description = """Wyvern"""

  jar {
    manifest.attributes provider: "gradle"
  }
}

apply from: rootProject.file("server/src/main/scripts/setup.gradle")
apply from: rootProject.file("server/src/main/scripts/archives.gradle")
apply from: rootProject.file("server/src/main/scripts/buildUtils.gradle")

‘server’ subproject build.gradle:

apply plugin: "application"
apply plugin: "kotlin"
apply plugin: "groovy"  // Spock
mainClassName = "wyvern.server.Server"
buildscript {
    repositories {
        maven { url "https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core" }
        maven { url "https://mvnrepository.com/artifact/com.google.guava/guava" }
        maven { url "https://oss.sonatype.org/content/repositories/snapshots/" }
        jcenter()
        mavenCentral()
    }
    dependencies {
        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
        classpath "com.github.jengelman.gradle.plugins:shadow:5.2.0"
    }
}
java {
  sourceCompatibility = jvm_target
  targetCompatibility = jvm_target
}
compileKotlin {
    kotlinOptions.jvmTarget = jvm_target
}
compileTestKotlin {
    kotlinOptions.jvmTarget = jvm_target
}
dependencies {
    implementation "org.python:jython-standalone:2.7.2"
    implementation "org.apache.xmlrpc:xmlrpc-server:3.1.3"
    implementation "gnu.getopt:java-getopt:1.0.13"
    implementation "org.jdom:jdom:1.1"
    implementation "net.sf.jung:jung-api:2.1.1"
    implementation "net.sf.jung:jung-graph-impl:2.1.1"
    implementation "net.sf.jung:jung-algorithms:2.1.1"
    implementation "org.apache.commons:commons-text:1.9"
    implementation "org.apache.commons:commons-math3:3.6.1"
    runtimeOnly "mysql:mysql-connector-java:8.0.24"
    implementation "commons-logging:commons-logging:1.2"
    implementation "io.netty:netty-all:4.1.63.Final"
    implementation "org.json:json:20210307"
    api project(":wyvern-common")
    testApi project(":wyvern-common").sourceSets.test.output
    implementation 'com.google.apis:google-api-services-androidpublisher:v3-rev20210429-1.31.0'
    implementation "com.google.api-client:google-api-client:1.31.4"
    implementation "com.google.auth:google-auth-library-oauth2-http:0.25.5"
    implementation project(":lib:wyvern-auth")
    implementation project(":lib:wyvern-clientlib")
    implementation project(":lib:wyvern-datasource")
    implementation project(":lib:wyvern-rpc")
    implementation 'org.redisson:redisson:3.15.4'
    implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_jdk8_version"
    implementation "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.3"
    implementation group: "com.konghq",
            name: "unirest-java",
            version: "3.11.11",
            classifier: "standalone"
    implementation group: "org.apache.thrift", name: "libthrift", version: "0.13.0"
    implementation group: "org.slf4j", name: "slf4j-api", version: "1.7.30"
    implementation group: "org.slf4j", name: "slf4j-log4j12", version: "1.7.30"
    implementation "io.dropwizard.metrics:metrics-core:4.1.9"
    implementation "io.grpc:grpc-all:1.37.0"
    implementation "com.google.api:api-common:1.10.3"
    implementation "com.google.api:gax-grpc:1.63.3"
    implementation "com.google.cloud:google-cloud-translate:1.97.1"
    implementation "org.fluentd:fluent-logger:0.3.4"
    implementation "org.codehaus.groovy:groovy-all:3.0.8"
    testImplementation platform("org.spockframework:spock-bom:2.0-M2-groovy-3.0")
    testImplementation "org.spockframework:spock-core"
    testImplementation "net.bytebuddy:byte-buddy:1.11.0"
    testImplementation "org.objenesis:objenesis:3.2"
    implementation "io.prometheus:simpleclient:0.10.0"
    implementation "io.prometheus:simpleclient_httpserver:0.10.0"
    implementation "io.prometheus:simpleclient_hotspot:0.10.0"
    testImplementation 'org.assertj:assertj-core:3.17.2'
    testImplementation(platform('org.junit:junit-bom:5.7.0'))
    testImplementation 'org.junit.jupiter:junit-jupiter'
    testImplementation 'com.github.stefanbirkner:system-rules:1.19.0'
    testImplementation testFixtures(project(path: ":lib:wyvern-datasource"))
}
sourceSets {
    main.kotlin.srcDirs += "/src/main/java"
}
application {
    applicationDefaultJvmArgs = defaultJvmArgs
}
test {
    systemProperty "test.server", "true"
    classpath = localGitClasspath + sourceSets.main.runtimeClasspath + sourceSets.test.runtimeClasspath
    environment "WYVERN_HOME", "$wyvernHome"
    environment "GOOGLE_APPLICATION_CREDENTIALS", "$wyvernHome/ssl/Wyvern-e2c4fad9081f5.json"
    useJUnitPlatform()
    jvmArgs += defaultJvmArgs
    environment "REDIS_HOST", "127.0.0.1"
    environment "REDIS_PORT", "6380"
}
test {
}
run {
    classpath = localGitClasspath + sourceSets.main.runtimeClasspath
    systemProperties commonSystemProps
    environment commonEnvironment
    environment "GOOGLE_APPLICATION_CREDENTIALS", "$wyvernHome/ssl/Wyvern-e2c4fad9081f5.json"
    jvmArgs += defaultJvmArgs
}
task(dbg, dependsOn: "classes", type: JavaExec) {
    classpath = localGitClasspath + sourceSets.main.runtimeClasspath
    main = "$mainClassName"
    systemProperties commonSystemProps
    jvmArgs += ["-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005"]
    jvmArgs += defaultJvmArgs
    environment "WYVERN_HOME", "$wyvernHome"
    environment "GOOGLE_APPLICATION_CREDENTIALS", "$wyvernHome/ssl/Wyvern-e2c4fad9081f5.json"
}
defaultTasks "run"
task finalize {
    doLast {
        cleanlogs
    }
}
build.finalizedBy(finalize)
task customCleanUp(type: Delete) {
    delete "$wyvernHome/ssl", "$wyvernHome/config"
}
tasks.clean.dependsOn(tasks.customCleanUp)
apply plugin: "war"
task explodedWar(type: Sync) {
    into "$buildDir/explodedWar"
    with war
}
war.finalizedBy "explodedWar"
task printClasspath() {
    doFirst {
        sourceSets.main.runtimeClasspath.each { println it }
        localGitContentClasspath.each { println it }
        localGitWizClasspath.each { println it }
    }
}
task cleanlogs(type: Delete) {
    delete fileTree("$wyvernHome/log").matching {
        include "**/*"
    }
}
task graph(type: JavaExec) {
  main = "wyvern.kernel.graph.MapCrawler"
  classpath = localGitClasspath + sourceSets.main.runtimeClasspath
  environment commonEnvironment
  systemProperties commonSystemProps
  jvmArgs += defaultJvmArgs
}

‘web’ subproject build.gradle:

plugins {
  id 'java'
  id 'application'
  id 'war'  // Used by our Dockerfile.
}
repositories {
  mavenLocal()
  maven {
    url = uri('https://repo.maven.apache.org/maven2')
  }
}
dependencies {
  providedCompile 'org.apache.tomcat:tomcat-servlet-api:10.0.0-M9'
  implementation 'org.apache.commons:commons-lang3:3.11'
  implementation 'commons-io:commons-io:2.4'
  implementation group: 'org.glassfish.web', name: 'jakarta.servlet.jsp.jstl', version: '2.0.0'
  implementation 'org.apache.tomcat:jsp-api:6.0.53'
  implementation 'com.google.cloud:google-cloud-storage:1.113.1'
  implementation group: 'com.google.api', name: 'gax', version: '1.5.0'
  implementation 'org.apache.logging.log4j:log4j-api:2.13.3'
  implementation 'org.apache.logging.log4j:log4j-core:2.13.3'
  testImplementation 'junit:junit:4.10'
  testImplementation 'org.mockito:mockito-all:1.9.0'
}
group = 'com.ghosttrack.wyvern'
version = '2.0'
sourceCompatibility = '1.8'
tasks.withType(JavaCompile) {
  options.encoding = 'UTF-8'
}

I’m having trouble seeing where the dependency from ‘server’ to ‘web’ is created. Some of the subprojects do indeed depend on others, but the ‘web’ subproject is standalone, so I thought it might serve as a good representative example.

Again, apologies for both the ignorance of my question and the awfulness of the build files.

Thank you.

Vampire · February 26, 2022, 3:42am

I’m not going to read through your build files, but the error says the opposite of what you think it is saying.
It says the same you say, that there is no dependency.

Here a small translation:

Gradle detected a problem with the following location: ‘/Users/stevey/wyvern’.

This location is used in a fishy way

Reason: Task ‘:wyvern-server:test’ uses this output of task ‘:wyvern-web:test’

That location (/Users/stevey/wyvern) is an output of task :wyvern-web:test
That location (/Users/stevey/wyvern) is an input of task :wyvern-server:test

without declaring an explicit or implicit dependency.

But there is no dependency from :wyvern-server:test to :wyvern-web:test

The two most common reasons for this is, that this is indeed the case, one task using the output of another task without having a task dependency,
or the location is wrongly defined as output of the one or input of the other task.

Tasks should usually have non-overlapping dedicated outputs.
Having /Users/stevey/wyvern as a task output definitely sounds fishy.
Even as task input it sounds incorrect.

Topic		Replies	Views
Implicit dependency but only when the task is not in the graph Help/Discuss	2	866	November 27, 2021
Explicit dependency issue Help/Discuss	1	18	December 25, 2024
Implicit dependeny to subproject in multi module project Help/Discuss	8	1551	April 26, 2023
Task A uses this output of task B without declaring an explicit or implicit dependency Help/Discuss	0	11321	May 11, 2022
Gradle 7.0 seems to take an overzealous approach to inter-task dependencies Help/Discuss	17	7087	February 1, 2022

Multi-module project phantom dependencies

Related topics