diff --git a/maven-core/src/main/java/org/apache/maven/lifecycle/ConcurrencyDependencyGraph.java b/maven-core/src/main/java/org/apache/maven/lifecycle/ConcurrencyDependencyGraph.java new file mode 100644 index 0000000..c0b3065 --- /dev/null +++ b/maven-core/src/main/java/org/apache/maven/lifecycle/ConcurrencyDependencyGraph.java @@ -0,0 +1,76 @@ +package org.apache.maven.lifecycle; + +import org.apache.maven.execution.ProjectDependencyGraph; +import org.apache.maven.project.MavenProject; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; + +/** + * Presents a view of the Dependency Graph that is suited for concurrent building. + */ +class ConcurrencyDependencyGraph { + private final HashMap projectBuildMap = new HashMap(); + private final List projectBuilds; + private final ProjectDependencyGraph projectDependencyGraph; + private final HashSet finishedProjects = new HashSet(); + + + ConcurrencyDependencyGraph(List projectBuilds, ProjectDependencyGraph projectDependencyGraph) + { + this.projectDependencyGraph = projectDependencyGraph; + this.projectBuilds = projectBuilds; + for (DefaultLifecycleExecutor.ProjectBuild projectBuild : projectBuilds) { + projectBuildMap.put(projectBuild.project, projectBuild); + } + } + + + public int getNumberOfBuilds() { + return projectBuilds.size(); + } + + /** + * Gets all the builds that have no reactor-dependencies + * @return A list of all the initial builds + */ + + public List getRootSchedulableBuilds() + { + List result = new ArrayList(); + for (DefaultLifecycleExecutor.ProjectBuild projectBuild : projectBuilds) { + if (projectDependencyGraph.getUpstreamProjects(projectBuild.project, false).size() == 0) { + result.add(projectBuild); + } + } + return result; + } + + /** + * Marks the provided project as finished. Returns a list of + * + * @param mavenProject The project + * @return The list of builds that are eligible for starting now that the provided project is done + */ + public List markAsFinished(MavenProject mavenProject) + { + finishedProjects.add(mavenProject); + return getSchedulableNewProcesses(mavenProject); + } + + private List getSchedulableNewProcesses(MavenProject finishedProject) + { + List result = new ArrayList(); + // schedule dependent projects, if all of their requirements are met + for (MavenProject dependentProject : projectDependencyGraph.getDownstreamProjects(finishedProject, false)) { + final List upstreamProjects = projectDependencyGraph.getUpstreamProjects(dependentProject, false); + if (finishedProjects.containsAll(upstreamProjects)) { + result.add(projectBuildMap.get(dependentProject)); + } + } + return result; + } + +} diff --git a/maven-core/src/main/java/org/apache/maven/lifecycle/DefaultLifecycleExecutor.java b/maven-core/src/main/java/org/apache/maven/lifecycle/DefaultLifecycleExecutor.java index 2cf46e5..b7d620e 100644 --- a/maven-core/src/main/java/org/apache/maven/lifecycle/DefaultLifecycleExecutor.java +++ b/maven-core/src/main/java/org/apache/maven/lifecycle/DefaultLifecycleExecutor.java @@ -31,12 +31,7 @@ import java.util.Set; import java.util.StringTokenizer; import java.util.TreeMap; import java.util.TreeSet; -import java.util.concurrent.Callable; -import java.util.concurrent.CompletionService; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; +import java.util.concurrent.*; import org.apache.maven.ProjectDependenciesResolver; import org.apache.maven.artifact.Artifact; @@ -282,15 +277,18 @@ public class DefaultLifecycleExecutor if ( threadCount > 0 ) { - multithreadedBuild( session, result, projectBuilds, projectIndex, - oldContextClassLoader, threadCount ); + ConcurrencyDependencyGraph analyzer = new ConcurrencyDependencyGraph(projectBuilds, session.getProjectDependencyGraph()); + Executor executor = Executors.newFixedThreadPool( threadCount ); + CompletionService service = new ExecutorCompletionService( executor ); + CallableContext callableContext = new CallableContext(result, projectIndex, oldContextClassLoader, session); + multithreadedBuild( analyzer, callableContext, session, service); } else { for ( ProjectBuild projectBuild : projectBuilds ) { - CallableBuilder cb = new CallableBuilder( result, projectBuild, projectIndex, - oldContextClassLoader, session ); + CallableContext callableContext = new CallableContext( result, projectIndex, oldContextClassLoader, session); + CallableBuilder cb = new CallableBuilder( callableContext, projectBuild ); try { cb.call(); @@ -306,124 +304,156 @@ public class DefaultLifecycleExecutor fireEvent( session, null, LifecycleEventCatapult.SESSION_ENDED ); } - private void multithreadedBuild( MavenSession session, MavenExecutionResult result, - List projectBuilds, ProjectIndex projectIndex, - ClassLoader oldContextClassLoader, - int threadCount ) - { - - HashMap projectBuildMap = new HashMap(); + /** + * Presents a view of the Dependency Graph that is suited for concurrent building. + */ + static class ConcurrencyDependencyGraph { + private final HashMap projectBuildMap = new HashMap(); + private final List projectBuilds; + private final ProjectDependencyGraph projectDependencyGraph; + private final HashSet finishedProjects = new HashSet(); + + + ConcurrencyDependencyGraph(List projectBuilds, ProjectDependencyGraph projectDependencyGraph) { + this.projectDependencyGraph = projectDependencyGraph; + this.projectBuilds = projectBuilds; + for ( ProjectBuild projectBuild : projectBuilds ) + { + projectBuildMap.put( projectBuild.project, projectBuild ); + } + } + + + public int getNumberOfBuilds(){ + return projectBuilds.size(); + } + public List getRootSchedulableBuilds(){ + List result = new ArrayList(); for ( ProjectBuild projectBuild : projectBuilds ) { - projectBuildMap.put( projectBuild.project, projectBuild ); + if ( projectDependencyGraph.getUpstreamProjects( projectBuild.project, false ).size() == 0 ) + { + result.add ( projectBuild); + } + } + return result; } - ProjectDependencyGraph pdg = session.getProjectDependencyGraph(); - Executor executor = Executors.newFixedThreadPool( threadCount ); - CompletionService service = - new ExecutorCompletionService( executor ); - HashSet> futures = new HashSet>(); + /** + * Marks the provided project as finished + * @param mavenProject The project + * @return The list of builds that are eligible for starting now that the provided project is done + */ + public List markAsFinished( MavenProject mavenProject){ + finishedProjects.add( mavenProject); + return getSchedulableNewProcesses( mavenProject); + } - // schedule independent projects - for ( ProjectBuild projectBuild : projectBuilds ) - { - if ( pdg.getUpstreamProjects( projectBuild.project, false ).size() == 0 ) + public List getSchedulableNewProcesses( MavenProject finishedProject){ + List result = new ArrayList(); + // schedule dependent projects, if all of their requirements are met + for ( MavenProject dependentProject : projectDependencyGraph.getDownstreamProjects( finishedProject, false ) ) { - if ( logger.isDebugEnabled() ) - { - logger.debug( "Scheduling: " + projectBuild.project ); + final List upstreamProjects = projectDependencyGraph.getUpstreamProjects(dependentProject, false); + if (finishedProjects.containsAll( upstreamProjects)){ + result.add( projectBuildMap.get( dependentProject )); } - CallableBuilder cb = - new CallableBuilder( result, projectBuild, projectIndex, oldContextClassLoader, session ); - futures.add( service.submit( cb ) ); } + return result; } - HashSet finishedProjects = new HashSet(); + } + + private void multithreadedBuild(ConcurrencyDependencyGraph analyzer, CallableContext callableContext, + MavenSession session, + CompletionService service) { + + HashSet> futures = new HashSet>(); + + // schedule independent projects + for (ProjectBuild projectBuild : analyzer.getRootSchedulableBuilds()) + { + logger.debug("Scheduling: " + projectBuild.project); + CallableBuilder cb = new CallableBuilder(callableContext, projectBuild); + futures.add(service.submit(cb)); + } // for each finished project - for ( int i = 0; i < projectBuilds.size(); i++ ) + for (int i = 0; i < analyzer.getNumberOfBuilds(); i++) { - ProjectBuild projectBuild; - try - { - projectBuild = service.take().get(); - } - catch ( Exception e ) - { + try { + ProjectBuild projectBuild = service.take().get(); + if (session.isHalted()) + break; + final List newItemsThatCanBeBuilt = analyzer.markAsFinished(projectBuild.project); + for (ProjectBuild scheduledDependent : newItemsThatCanBeBuilt) { + logger.debug("Scheduling: " + scheduledDependent); + CallableBuilder cb = new CallableBuilder(callableContext, scheduledDependent); + futures.add(service.submit(cb)); + } + } catch (InterruptedException e) { break; - } - if ( session.isHalted() ) + } catch (ExecutionException e) { break; - MavenProject finishedProject = projectBuild.project; - finishedProjects.add( finishedProject ); - // schedule dependent projects, if all of their requirements are met - for ( MavenProject dependentProject : pdg.getDownstreamProjects( finishedProject, false ) ) - { - boolean allRequirementsMet = true; - for ( MavenProject requirement : pdg.getUpstreamProjects( dependentProject, false ) ) - { - if ( !finishedProjects.contains( requirement ) ) - { - if ( logger.isDebugEnabled() ) - { - logger.debug( "Not scheduling " + dependentProject + " because requirement not met: " - + requirement ); - } - allRequirementsMet = false; - break; - } - } - if ( allRequirementsMet ) - { - ProjectBuild scheduledDependent = projectBuildMap.get( dependentProject ); - if ( logger.isDebugEnabled() ) - { - logger.debug( "Scheduling: " + dependentProject ); - } - CallableBuilder cb = - new CallableBuilder( result, scheduledDependent, projectIndex, oldContextClassLoader, session ); - futures.add( service.submit( cb ) ); - } } } - // cancel outstanding builds (if any) - for ( Future future : futures ) + // cancel outstanding builds (if any) - this can happen if an exception is thrown in above block + for (Future future : futures) { - future.cancel( true /* mayInterruptIfRunning */); + future.cancel(true /* mayInterruptIfRunning */); } } - private class CallableBuilder - implements Callable - { - + private class CallableContext { final MavenExecutionResult result; - - final ProjectBuild projectBuild; - final ProjectIndex projectIndex; - final ClassLoader oldContextClassLoader; - final MavenSession originalSession; - public CallableBuilder( MavenExecutionResult result, ProjectBuild projectBuild, ProjectIndex projectIndex, - ClassLoader oldContextClassLoader, MavenSession originalSession ) - { + + private CallableContext(MavenExecutionResult result, ProjectIndex projectIndex, ClassLoader oldContextClassLoader, MavenSession originalSession) { this.result = result; - this.projectBuild = projectBuild; this.projectIndex = projectIndex; this.oldContextClassLoader = oldContextClassLoader; this.originalSession = originalSession; } + public MavenExecutionResult getResult() { + return result; + } + + public ProjectIndex getProjectIndex() { + return projectIndex; + } + + public ClassLoader getOldContextClassLoader() { + return oldContextClassLoader; + } + + public MavenSession getOriginalSession() { + return originalSession; + } + } + private class CallableBuilder + implements Callable + { + + final CallableContext callableContext; + + final ProjectBuild projectBuild; + + private CallableBuilder(CallableContext callableContext, ProjectBuild projectBuild) { + this.callableContext = callableContext; + this.projectBuild = projectBuild; + } + public ProjectBuild call() throws Exception { - buildProject( result, projectBuild, projectIndex, oldContextClassLoader, originalSession ); + buildProject( callableContext.getResult(), projectBuild, callableContext.getProjectIndex(), + callableContext.getOldContextClassLoader(), callableContext.originalSession); return projectBuild; } } @@ -1035,7 +1065,7 @@ public class DefaultLifecycleExecutor return task.indexOf( ':' ) >= 0; } - private static final class ProjectBuild + static final class ProjectBuild { final MavenProject project; @@ -1056,7 +1086,7 @@ public class DefaultLifecycleExecutor } - private static final class TaskSegment + static final class TaskSegment { final List tasks; diff --git a/maven-core/src/test/java/org/apache/maven/lifecycle/ConcurrencyDependencyGraphTest.java b/maven-core/src/test/java/org/apache/maven/lifecycle/ConcurrencyDependencyGraphTest.java new file mode 100644 index 0000000..df253b6 --- /dev/null +++ b/maven-core/src/test/java/org/apache/maven/lifecycle/ConcurrencyDependencyGraphTest.java @@ -0,0 +1,49 @@ +package org.apache.maven.lifecycle; + +import org.apache.maven.execution.ProjectDependencyGraph; +import static org.apache.maven.lifecycle.StubProjectDependencyGraph.*; +import java.util.List; + +/** + * @author Kristian Rosenvold + */ +public class ConcurrencyDependencyGraphTest extends junit.framework.TestCase { + + public void testConcurrencyGraphPrimaryVersion(){ + ProjectDependencyGraph dependencyGraph = new StubProjectDependencyGraph(); + ConcurrencyDependencyGraph graph = new ConcurrencyDependencyGraph( StubProjectDependencyGraph.getProjectBuilds(), dependencyGraph); + + final List projectBuilds = graph.getRootSchedulableBuilds(); + assertEquals(1, projectBuilds.size()); + assertEquals(A, projectBuilds.get(0).project); + + + final List subsequent = graph.markAsFinished(StubProjectDependencyGraph.A); + assertEquals(2, subsequent.size()); + assertEquals(StubProjectDependencyGraph.B, subsequent.get(0).project); + assertEquals(StubProjectDependencyGraph.C, subsequent.get(1).project); + + final List bDescendants = graph.markAsFinished(B); + assertEquals(1, bDescendants.size()); + assertEquals(StubProjectDependencyGraph.Y, bDescendants.get(0).project); + + final List cDescendants = graph.markAsFinished(C); + assertEquals(2, cDescendants.size()); + assertEquals(StubProjectDependencyGraph.X, cDescendants.get(0).project); + assertEquals(StubProjectDependencyGraph.Z, cDescendants.get(1).project); + } + public void testConcurrencyGraphDifferentCompletionOrder(){ + ProjectDependencyGraph dependencyGraph = new StubProjectDependencyGraph(); + ConcurrencyDependencyGraph graph = new ConcurrencyDependencyGraph( StubProjectDependencyGraph.getProjectBuilds(), dependencyGraph); + + graph.markAsFinished(StubProjectDependencyGraph.A); + final List cDescendants = graph.markAsFinished(C); + assertEquals(1, cDescendants.size()); + assertEquals(StubProjectDependencyGraph.Z, cDescendants.get(0).project); + + final List bDescendants = graph.markAsFinished(B); + assertEquals(2, bDescendants.size()); + assertEquals(StubProjectDependencyGraph.X, bDescendants.get(0).project); + assertEquals(StubProjectDependencyGraph.Y, bDescendants.get(1).project); + } +} diff --git a/maven-core/src/test/java/org/apache/maven/lifecycle/StubProjectDependencyGraph.java b/maven-core/src/test/java/org/apache/maven/lifecycle/StubProjectDependencyGraph.java new file mode 100644 index 0000000..375142e --- /dev/null +++ b/maven-core/src/test/java/org/apache/maven/lifecycle/StubProjectDependencyGraph.java @@ -0,0 +1,97 @@ +package org.apache.maven.lifecycle; + +import org.apache.maven.execution.ProjectDependencyGraph; +import org.apache.maven.project.MavenProject; + +import java.util.*; + +/** + * A stub dependency graph that is custom made for testing concurrent build graph evaluations. + * + * Implements a graph as follows: + * A has no dependencies + * B depends on A + * C depends on A + * X depends on B & C + * Y depends on B + * Z depends on C + + * @author Kristian Rosenvold + */ +public class StubProjectDependencyGraph implements ProjectDependencyGraph { + public static final MavenProject A = new MavenProject(); + public static final MavenProject B = new MavenProject(); + public static final MavenProject C = new MavenProject(); + public static final MavenProject X = new MavenProject(); + public static final MavenProject Y = new MavenProject(); + public static final MavenProject Z = new MavenProject(); + + static { + A.setArtifactId("A"); + B.setArtifactId("B"); + C.setArtifactId("C"); + X.setArtifactId("X"); + Y.setArtifactId("Y"); + Z.setArtifactId("Z"); + } + public static List getProjectBuilds(){ + List projectBuilds = new ArrayList(); + projectBuilds.add( new DefaultLifecycleExecutor.ProjectBuild(A, new DefaultLifecycleExecutor.TaskSegment(false))); + projectBuilds.add( new DefaultLifecycleExecutor.ProjectBuild(B, new DefaultLifecycleExecutor.TaskSegment(false))); + projectBuilds.add( new DefaultLifecycleExecutor.ProjectBuild(C, new DefaultLifecycleExecutor.TaskSegment(false))); + projectBuilds.add( new DefaultLifecycleExecutor.ProjectBuild(X, new DefaultLifecycleExecutor.TaskSegment(false))); + projectBuilds.add( new DefaultLifecycleExecutor.ProjectBuild(Y, new DefaultLifecycleExecutor.TaskSegment(false))); + projectBuilds.add( new DefaultLifecycleExecutor.ProjectBuild(Z, new DefaultLifecycleExecutor.TaskSegment(false))); + return projectBuilds; + } + + + class Dependency { + MavenProject dependant; + MavenProject dependency; + + Dependency(MavenProject dependant, MavenProject dependency) { + this.dependant = dependant; + this.dependency = dependency; + } + void addIfDownstream( MavenProject mavenProject, List result){ + if (dependency == mavenProject) result.add( dependant); + } + + void addIfUpstreamOf( MavenProject mavenProject, List result){ + if (dependant == mavenProject) result.add( dependency); // All projects are the statics from this class + } + } + private List getDependencies(){ + List dependencies = new ArrayList(); + dependencies.add( new Dependency( B,A)); + dependencies.add( new Dependency( C,A)); + dependencies.add( new Dependency( X,B)); + dependencies.add( new Dependency( X,C)); + dependencies.add( new Dependency( Y,B)); + dependencies.add( new Dependency( Z,C)); + return dependencies; + } + + public List getSortedProjects() { + return Arrays.asList(A, B, C, X, Y, Z); // I'm not entirely sure about the order but this shold do... + } + + public List getDownstreamProjects(MavenProject project, boolean transitive) { + if (transitive) throw new RuntimeException("Not implemented yet"); + List result = new ArrayList(); + for ( Dependency dependency : getDependencies()){ + dependency.addIfDownstream( project, result); + } + return result; + } + + public List getUpstreamProjects(MavenProject project, boolean transitive) { + if (transitive) throw new RuntimeException("Not implemented yet"); + List result = new ArrayList(); + for ( Dependency dependency : getDependencies()){ + dependency.addIfUpstreamOf( project, result); + } + return result; + } +}