From d58142b3f004eb74095165dbd72e0d59e0cf18ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 Aug 2013 01:59:29 +0200 Subject: [PATCH] Store aggregate members in the database For presentation purposes, we need to know what builds are part of an aggregate build. So at evaluation time, look at the "members" attribute, find the corresponding builds in the eval, and create a mapping in the AggregateMembers table. --- src/c/hydra-eval-jobs.cc | 17 ++++ src/lib/Hydra/Controller/Build.pm | 2 +- src/lib/Hydra/Helper/AddBuilds.pm | 24 ++--- src/lib/Hydra/Schema/AggregateMembers.pm | 111 +++++++++++++++++++++++ src/lib/Hydra/Schema/Builds.pm | 54 ++++++++++- src/script/hydra-evaluator | 31 +++++-- src/sql/hydra.sql | 7 ++ src/sql/upgrade-19.sql | 5 + 8 files changed, 226 insertions(+), 25 deletions(-) create mode 100644 src/lib/Hydra/Schema/AggregateMembers.pm create mode 100644 src/sql/upgrade-19.sql diff --git a/src/c/hydra-eval-jobs.cc b/src/c/hydra-eval-jobs.cc index 4ff010572..ca2fb7fbb 100644 --- a/src/c/hydra-eval-jobs.cc +++ b/src/c/hydra-eval-jobs.cc @@ -160,6 +160,23 @@ static void findJobsWrapped(EvalState & state, XMLWriter & doc, } xmlAttrs["maintainers"] = maintainers; + /* If this is an aggregate, then get its members. */ + Bindings::iterator a = v.attrs->find(state.symbols.create("_hydraAggregate")); + if (a != v.attrs->end() && state.forceBool(*a->value)) { + Bindings::iterator a = v.attrs->find(state.symbols.create("members")); + if (a == v.attrs->end()) + throw EvalError("derivation must have a ‘members’ attribute"); + PathSet context; + state.coerceToString(*a->value, context, true, false); + PathSet drvs; + foreach (PathSet::iterator, i, context) + if (i->at(0) == '!') { + size_t index = i->find("!", 1); + drvs.insert(string(*i, index + 1)); + } + xmlAttrs["members"] = concatStringsSep(" ", drvs); + } + /* Register the derivation as a GC root. !!! This registers roots for jobs that we may have already done. */ diff --git a/src/lib/Hydra/Controller/Build.pm b/src/lib/Hydra/Controller/Build.pm index cded877fd..6fef3ed8e 100644 --- a/src/lib/Hydra/Controller/Build.pm +++ b/src/lib/Hydra/Controller/Build.pm @@ -578,7 +578,7 @@ sub clone_submit : Chained('buildChain') PathPart('clone/submit') Args(0) { my %currentBuilds; my $newBuild = checkBuild( - $c->model('DB'), $build->project, $build->jobset, + $c->model('DB'), $build->jobset, $inputInfo, $nixExprInput, $job, \%currentBuilds, undef, {}, $c->hydra_plugins); error($c, "This build has already been performed.") unless $newBuild; diff --git a/src/lib/Hydra/Helper/AddBuilds.pm b/src/lib/Hydra/Helper/AddBuilds.pm index 6f51b98d8..f1e21b49c 100644 --- a/src/lib/Hydra/Helper/AddBuilds.pm +++ b/src/lib/Hydra/Helper/AddBuilds.pm @@ -288,13 +288,9 @@ sub evalJobs { my $validJob = 1; foreach my $arg (@{$job->{arg}}) { my $input = $inputInfo->{$arg->{name}}->[$arg->{altnr}]; - if ($input->{type} eq "sysbuild" && $input->{system} ne $job->{system}) { - $validJob = 0; - } - } - if ($validJob) { - push(@filteredJobs, $job); + $validJob = 0 if $input->{type} eq "sysbuild" && $input->{system} ne $job->{system}; } + push(@filteredJobs, $job) if $validJob; } $jobs->{job} = \@filteredJobs; @@ -390,7 +386,7 @@ sub getPrevJobsetEval { # Check whether to add the build described by $buildInfo. sub checkBuild { - my ($db, $project, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildIds, $prevEval, $jobOutPathMap, $plugins) = @_; + my ($db, $jobset, $inputInfo, $nixExprInput, $buildInfo, $buildMap, $prevEval, $jobOutPathMap, $plugins) = @_; my @outputNames = sort keys %{$buildInfo->{output}}; die unless scalar @outputNames; @@ -411,9 +407,7 @@ sub checkBuild { my $build; txn_do($db, sub { - my $job = $jobset->jobs->update_or_create( - { name => $jobName - }); + my $job = $jobset->jobs->update_or_create({ name => $jobName }); # Don't add a build that has already been scheduled for this # job, or has been built but is still a "current" build for @@ -434,19 +428,19 @@ sub checkBuild { # semantically unnecessary (because they're implied by # the eval), but they give a factor 1000 speedup on # the Nixpkgs jobset with PostgreSQL. - { project => $project->name, jobset => $jobset->name, job => $job->name, + { project => $jobset->project->name, jobset => $jobset->name, job => $jobName, name => $firstOutputName, path => $firstOutputPath }, { rows => 1, columns => ['id'], join => ['buildoutputs'] }); if (defined $prevBuild) { print STDERR " already scheduled/built as build ", $prevBuild->id, "\n"; - $buildIds->{$prevBuild->id} = 0; + $buildMap->{$prevBuild->id} = { new => 0, drvPath => $drvPath }; return; } } # Prevent multiple builds with the same (job, outPath) from # being added. - my $prev = $$jobOutPathMap{$job->name . "\t" . $firstOutputPath}; + my $prev = $$jobOutPathMap{$jobName . "\t" . $firstOutputPath}; if (defined $prev) { print STDERR " already scheduled as build ", $prev, "\n"; return; @@ -512,8 +506,8 @@ sub checkBuild { $build->buildoutputs->create({ name => $_, path => $buildInfo->{output}->{$_}->{path} }) foreach @outputNames; - $buildIds->{$build->id} = 1; - $$jobOutPathMap{$job->name . "\t" . $firstOutputPath} = $build->id; + $buildMap->{$build->id} = { new => 1, drvPath => $drvPath }; + $$jobOutPathMap{$jobName . "\t" . $firstOutputPath} = $build->id; if ($build->iscachedbuild) { print STDERR " marked as cached build ", $build->id, "\n"; diff --git a/src/lib/Hydra/Schema/AggregateMembers.pm b/src/lib/Hydra/Schema/AggregateMembers.pm new file mode 100644 index 000000000..4a0376630 --- /dev/null +++ b/src/lib/Hydra/Schema/AggregateMembers.pm @@ -0,0 +1,111 @@ +use utf8; +package Hydra::Schema::AggregateMembers; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Hydra::Schema::AggregateMembers + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 COMPONENTS LOADED + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->load_components("+Hydra::Component::ToJSON"); + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("AggregateMembers"); + +=head1 ACCESSORS + +=head2 aggregate + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=head2 member + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "aggregate", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, + "member", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("aggregate", "member"); + +=head1 RELATIONS + +=head2 aggregate + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "aggregate", + "Hydra::Schema::Builds", + { id => "aggregate" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, +); + +=head2 member + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "member", + "Hydra::Schema::Builds", + { id => "member" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:jHJtO2baXiprv0OcWCLZ+w + + +# You can replace this text with custom code or comments, and it will be preserved on regeneration +1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index ea27d5793..6280e08fe 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -288,6 +288,36 @@ __PACKAGE__->set_primary_key("id"); =head1 RELATIONS +=head2 aggregatemembers_aggregates + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "aggregatemembers_aggregates", + "Hydra::Schema::AggregateMembers", + { "foreign.aggregate" => "self.id" }, + undef, +); + +=head2 aggregatemembers_members + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "aggregatemembers_members", + "Hydra::Schema::AggregateMembers", + { "foreign.member" => "self.id" }, + undef, +); + =head2 buildinputs_builds Type: has_many @@ -468,9 +498,29 @@ __PACKAGE__->has_many( undef, ); +=head2 aggregates -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-06-13 01:54:50 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:isCEXACY/PwkvgKHcXvAIg +Type: many_to_many + +Composing rels: L -> aggregate + +=cut + +__PACKAGE__->many_to_many("aggregates", "aggregatemembers_members", "aggregate"); + +=head2 members + +Type: many_to_many + +Composing rels: L -> member + +=cut + +__PACKAGE__->many_to_many("members", "aggregatemembers_members", "member"); + + +# Created by DBIx::Class::Schema::Loader v0.07033 @ 2013-08-13 22:17:52 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:9jqsol/evbHYjusT09hLtw __PACKAGE__->has_many( "dependents", diff --git a/src/script/hydra-evaluator b/src/script/hydra-evaluator index 5bfed9e3d..a6dfde463 100755 --- a/src/script/hydra-evaluator +++ b/src/script/hydra-evaluator @@ -144,11 +144,11 @@ sub checkJobsetWrapped { $jobset->builds->search({iscurrent => 1})->update({iscurrent => 0}); # Schedule each successfully evaluated job. - my %buildIds; + my %buildMap; foreach my $job (permute @{$jobs->{job}}) { next if $job->{jobName} eq ""; print STDERR " considering job " . $project->name, ":", $jobset->name, ":", $job->{jobName} . "\n"; - checkBuild($db, $project, $jobset, $inputInfo, $nixExprInput, $job, \%buildIds, $prevEval, $jobOutPathMap, $plugins); + checkBuild($db, $jobset, $inputInfo, $nixExprInput, $job, \%buildMap, $prevEval, $jobOutPathMap, $plugins); } # Update the last checked times and error messages for each @@ -162,8 +162,8 @@ sub checkJobsetWrapped { foreach $jobset->jobs->all; my $hasNewBuilds = 0; - while (my ($id, $new) = each %buildIds) { - $hasNewBuilds = 1 if $new; + while (my ($id, $x) = each %buildMap) { + $hasNewBuilds = 1 if $x->{new}; } my $ev = $jobset->jobsetevals->create( @@ -172,12 +172,29 @@ sub checkJobsetWrapped { , checkouttime => abs($checkoutStop - $checkoutStart) , evaltime => abs($evalStop - $evalStart) , hasnewbuilds => $hasNewBuilds - , nrbuilds => $hasNewBuilds ? scalar(keys %buildIds) : undef + , nrbuilds => $hasNewBuilds ? scalar(keys %buildMap) : undef }); if ($hasNewBuilds) { - while (my ($id, $new) = each %buildIds) { - $ev->jobsetevalmembers->create({ build => $id, isnew => $new }); + # Create JobsetEvalMembers mappings. + my %drvPathToId; + while (my ($id, $x) = each %buildMap) { + $ev->jobsetevalmembers->create({ build => $id, isnew => $x->{new} }); + $drvPathToId{$x->{drvPath}} = $id; + } + + # Create AggregateMembers mappings. + foreach my $job (@{$jobs->{job}}) { + next unless $job->{members}; + my $id = $drvPathToId{$job->{drvPath}} or die; + foreach my $drvPath (split / /, $job->{members}) { + my $member = $drvPathToId{$drvPath}; + if (defined $member) { + $db->resultset('AggregateMembers')->update_or_create({aggregate => $id, member => $member}); + } else { + warn "aggregate job ‘$job->{jobName}’ has a member ‘$drvPath’ that doesn't correspond to a Hydra build\n"; + } + } } foreach my $name (keys %{$inputInfo}) { diff --git a/src/sql/hydra.sql b/src/sql/hydra.sql index 3001ab108..ee601c627 100644 --- a/src/sql/hydra.sql +++ b/src/sql/hydra.sql @@ -514,6 +514,13 @@ create table NewsItems ( ); +create table AggregateMembers ( + aggregate integer not null references Builds(id) on delete cascade, + member integer not null references Builds(id) on delete cascade, + primary key (aggregate, member) +); + + -- Cache of the number of finished builds. create table NrBuilds ( what text primary key not null, diff --git a/src/sql/upgrade-19.sql b/src/sql/upgrade-19.sql new file mode 100644 index 000000000..3d1e849bd --- /dev/null +++ b/src/sql/upgrade-19.sql @@ -0,0 +1,5 @@ +create table AggregateMembers ( + aggregate integer not null references Builds(id) on delete cascade, + member integer not null references Builds(id) on delete cascade, + primary key (aggregate, member) +);