From 4d26546d3c137cf4afb2403c60c28a1053599894 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 31 Jul 2015 00:57:30 +0200 Subject: [PATCH] Add support for tracking custom metrics Builds can now emit metrics that Hydra will store in its database and render as time series via flot charts. Typical applications are to keep track of performance indicators, coverage percentages, artifact sizes, and so on. For example, a coverage build can emit the coverage percentage as follows: echo "lineCoverage $pct %" > $out/nix-support/hydra-metrics Graphs of all metrics for a job can be seen at http://.../job///#tabs-charts Specific metrics are also visible at http://.../job////metric/ The latter URL also allows getting the data in JSON format (e.g. via "curl -H 'Accept: application/json'"). --- src/hydra-queue-runner/build-result.cc | 47 +++-- src/hydra-queue-runner/build-result.hh | 8 + src/hydra-queue-runner/hydra-queue-runner.cc | 13 ++ src/hydra-queue-runner/queue-monitor.cc | 9 +- src/hydra-queue-runner/state.hh | 9 +- src/lib/Hydra/Controller/Job.pm | 17 ++ src/lib/Hydra/Schema/BuildMetrics.pm | 187 +++++++++++++++++++ src/lib/Hydra/Schema/Builds.pm | 20 +- src/lib/Hydra/Schema/Jobs.pm | 23 ++- src/lib/Hydra/Schema/Jobsets.pm | 22 ++- src/lib/Hydra/Schema/Projects.pm | 19 +- src/root/build.tt | 21 ++- src/root/common.tt | 8 +- src/root/job.tt | 8 + src/root/metric.tt | 7 + src/root/static/css/hydra.css | 4 + src/sql/hydra.sql | 23 +++ src/sql/upgrade-39.sql | 22 +++ 18 files changed, 437 insertions(+), 30 deletions(-) create mode 100644 src/lib/Hydra/Schema/BuildMetrics.pm create mode 100644 src/root/metric.tt create mode 100644 src/sql/upgrade-39.sql diff --git a/src/hydra-queue-runner/build-result.cc b/src/hydra-queue-runner/build-result.cc index c2c2c8afc..5f516d23c 100644 --- a/src/hydra-queue-runner/build-result.cc +++ b/src/hydra-queue-runner/build-result.cc @@ -7,6 +7,21 @@ using namespace nix; +static std::tuple secureRead(Path fileName) +{ + auto fail = std::make_tuple(false, ""); + + if (!pathExists(fileName)) return fail; + + try { + /* For security, resolve symlinks. */ + fileName = canonPath(fileName, true); + if (!isInStore(fileName)) return fail; + return std::make_tuple(true, readFile(fileName)); + } catch (Error & e) { return fail; } +} + + BuildOutput getBuildOutput(std::shared_ptr store, const Derivation & drv) { BuildOutput res; @@ -40,22 +55,12 @@ BuildOutput getBuildOutput(std::shared_ptr store, const Derivation & d Path failedFile = output + "/nix-support/failed"; if (pathExists(failedFile)) res.failed = true; - Path productsFile = output + "/nix-support/hydra-build-products"; - if (!pathExists(productsFile)) continue; + auto file = secureRead(output + "/nix-support/hydra-build-products"); + if (!std::get<0>(file)) continue; + explicitProducts = true; - /* For security, resolve symlinks. */ - try { - productsFile = canonPath(productsFile, true); - } catch (Error & e) { continue; } - if (!isInStore(productsFile)) continue; - - string contents; - try { - contents = readFile(productsFile); - } catch (Error & e) { continue; } - - for (auto & line : tokenizeString(contents, "\n")) { + for (auto & line : tokenizeString(std::get<1>(file), "\n")) { BuildProduct product; Regex::Subs subs; @@ -122,5 +127,19 @@ BuildOutput getBuildOutput(std::shared_ptr store, const Derivation & d // FIXME: validate release name } + /* Get metrics. */ + for (auto & output : outputs) { + auto file = secureRead(output + "/nix-support/hydra-metrics"); + for (auto & line : tokenizeString(std::get<1>(file), "\n")) { + auto fields = tokenizeString>(line); + if (fields.size() < 2) continue; + BuildMetric metric; + metric.name = fields[0]; // FIXME: validate + metric.value = atof(fields[1].c_str()); // FIXME + metric.unit = fields.size() >= 3 ? fields[2] : ""; + res.metrics[metric.name] = metric; + } + } + return res; } diff --git a/src/hydra-queue-runner/build-result.hh b/src/hydra-queue-runner/build-result.hh index 65fbf9480..e40e7dcdc 100644 --- a/src/hydra-queue-runner/build-result.hh +++ b/src/hydra-queue-runner/build-result.hh @@ -15,6 +15,12 @@ struct BuildProduct BuildProduct() { } }; +struct BuildMetric +{ + std::string name, unit; + double value; +}; + struct BuildOutput { /* Whether this build has failed with output, i.e., the build @@ -27,6 +33,8 @@ struct BuildOutput unsigned long long closureSize = 0, size = 0; std::list products; + + std::map metrics; }; BuildOutput getBuildOutput(std::shared_ptr store, const nix::Derivation & drv); diff --git a/src/hydra-queue-runner/hydra-queue-runner.cc b/src/hydra-queue-runner/hydra-queue-runner.cc index 2860b854d..10fd1cd1d 100644 --- a/src/hydra-queue-runner/hydra-queue-runner.cc +++ b/src/hydra-queue-runner/hydra-queue-runner.cc @@ -238,6 +238,19 @@ void State::markSucceededBuild(pqxx::work & txn, Build::ptr build, (product.defaultPath).exec(); } + for (auto & metric : res.metrics) { + txn.parameterized + ("insert into BuildMetrics (build, name, unit, value, project, jobset, job, timestamp) values ($1, $2, $3, $4, $5, $6, $7, $8)") + (build->id) + (metric.second.name) + (metric.second.unit, metric.second.unit != "") + (metric.second.value) + (build->projectName) + (build->jobsetName) + (build->jobName) + (build->timestamp).exec(); + } + nrBuildsDone++; } diff --git a/src/hydra-queue-runner/queue-monitor.cc b/src/hydra-queue-runner/queue-monitor.cc index bd9b2aefa..8448cb68a 100644 --- a/src/hydra-queue-runner/queue-monitor.cc +++ b/src/hydra-queue-runner/queue-monitor.cc @@ -64,7 +64,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, { pqxx::work txn(conn); - auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec(); + auto res = txn.parameterized("select id, project, jobset, job, drvPath, maxsilent, timeout, timestamp from Builds where id > $1 and finished = 0 order by id")(lastBuildId).exec(); for (auto const & row : res) { auto builds_(builds.lock()); @@ -76,9 +76,12 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, auto build = std::make_shared(); build->id = id; build->drvPath = row["drvPath"].as(); - build->fullJobName = row["project"].as() + ":" + row["jobset"].as() + ":" + row["job"].as(); + build->projectName = row["project"].as(); + build->jobsetName = row["jobset"].as(); + build->jobName = row["job"].as(); build->maxSilentTime = row["maxsilent"].as(); build->buildTimeout = row["timeout"].as(); + build->timestamp = row["timestamp"].as(); newBuilds.emplace(std::make_pair(build->drvPath, build)); } @@ -89,7 +92,7 @@ void State::getQueuedBuilds(Connection & conn, std::shared_ptr store, std::function createBuild; createBuild = [&](Build::ptr build) { - printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName); + printMsg(lvlTalkative, format("loading build %1% (%2%)") % build->id % build->fullJobName()); nrAdded++; if (!store->isValidPath(build->drvPath)) { diff --git a/src/hydra-queue-runner/state.hh b/src/hydra-queue-runner/state.hh index 761a100b0..2bc3a56c8 100644 --- a/src/hydra-queue-runner/state.hh +++ b/src/hydra-queue-runner/state.hh @@ -38,6 +38,7 @@ typedef enum { bssFailed = 1, bssAborted = 4, bssTimedOut = 7, + bssCachedFailure = 8, bssUnsupported = 9, bssBusy = 100, // not stored } BuildStepStatus; @@ -67,12 +68,18 @@ struct Build BuildID id; nix::Path drvPath; std::map outputs; - std::string fullJobName; + std::string projectName, jobsetName, jobName; + time_t timestamp; unsigned int maxSilentTime, buildTimeout; std::shared_ptr toplevel; std::atomic_bool finishedInDB{false}; + + std::string fullJobName() + { + return projectName + ":" + jobsetName + ":" + jobName; + } }; diff --git a/src/lib/Hydra/Controller/Job.pm b/src/lib/Hydra/Controller/Job.pm index 274d96046..475eca376 100644 --- a/src/lib/Hydra/Controller/Job.pm +++ b/src/lib/Hydra/Controller/Job.pm @@ -77,6 +77,9 @@ sub overview : Chained('job') PathPart('') Args(0) { , jobset => $c->stash->{jobset}->name , job => $c->stash->{job}->name })->count == 1 if $c->user_exists; + + $c->stash->{metrics} = [ $job->buildmetrics->search( + { }, { select => ["name"], distinct => 1, order_by => "timestamp desc", }) ]; } @@ -110,6 +113,20 @@ sub output_sizes : Chained('job') PathPart('output-sizes') Args(0) { } +sub metric : Chained('job') PathPart('metric') Args(1) { + my ($self, $c, $metricName) = @_; + + $c->stash->{template} = 'metric.tt'; + $c->stash->{metricName} = $metricName; + + my @res = $c->stash->{job}->buildmetrics->search( + { name => $metricName }, + { order_by => "timestamp", columns => [ "build", "name", "timestamp", "value", "unit" ] }); + + $self->status_ok($c, entity => [ map { { id => $_->get_column("build"), timestamp => $_ ->timestamp, value => $_->value, unit => $_->unit } } @res ]); +} + + # Hydra::Base::Controller::ListBuilds needs this. sub get_builds : Chained('job') PathPart('') CaptureArgs(0) { my ($self, $c) = @_; diff --git a/src/lib/Hydra/Schema/BuildMetrics.pm b/src/lib/Hydra/Schema/BuildMetrics.pm new file mode 100644 index 000000000..58bbed945 --- /dev/null +++ b/src/lib/Hydra/Schema/BuildMetrics.pm @@ -0,0 +1,187 @@ +use utf8; +package Hydra::Schema::BuildMetrics; + +# Created by DBIx::Class::Schema::Loader +# DO NOT MODIFY THE FIRST PART OF THIS FILE + +=head1 NAME + +Hydra::Schema::BuildMetrics + +=cut + +use strict; +use warnings; + +use base 'DBIx::Class::Core'; + +=head1 COMPONENTS LOADED + +=over 4 + +=item * L + +=back + +=cut + +__PACKAGE__->load_components("+Hydra::Component::ToJSON"); + +=head1 TABLE: C + +=cut + +__PACKAGE__->table("BuildMetrics"); + +=head1 ACCESSORS + +=head2 build + + data_type: 'integer' + is_foreign_key: 1 + is_nullable: 0 + +=head2 name + + data_type: 'text' + is_nullable: 0 + +=head2 unit + + data_type: 'text' + is_nullable: 1 + +=head2 value + + data_type: 'double precision' + is_nullable: 0 + +=head2 project + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 jobset + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 job + + data_type: 'text' + is_foreign_key: 1 + is_nullable: 0 + +=head2 timestamp + + data_type: 'integer' + is_nullable: 0 + +=cut + +__PACKAGE__->add_columns( + "build", + { data_type => "integer", is_foreign_key => 1, is_nullable => 0 }, + "name", + { data_type => "text", is_nullable => 0 }, + "unit", + { data_type => "text", is_nullable => 1 }, + "value", + { data_type => "double precision", is_nullable => 0 }, + "project", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "jobset", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "job", + { data_type => "text", is_foreign_key => 1, is_nullable => 0 }, + "timestamp", + { data_type => "integer", is_nullable => 0 }, +); + +=head1 PRIMARY KEY + +=over 4 + +=item * L + +=item * L + +=back + +=cut + +__PACKAGE__->set_primary_key("build", "name"); + +=head1 RELATIONS + +=head2 build + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "build", + "Hydra::Schema::Builds", + { id => "build" }, + { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" }, +); + +=head2 job + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "job", + "Hydra::Schema::Jobs", + { jobset => "jobset", name => "job", project => "project" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + +=head2 jobset + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "jobset", + "Hydra::Schema::Jobsets", + { name => "jobset", project => "project" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + +=head2 project + +Type: belongs_to + +Related object: L + +=cut + +__PACKAGE__->belongs_to( + "project", + "Hydra::Schema::Projects", + { name => "project" }, + { is_deferrable => 0, on_delete => "NO ACTION", on_update => "CASCADE" }, +); + + +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:qoPm5/le+sVHigW4Dmum2Q + +sub json_hint { + return { columns => ['value', 'unit'] }; +} + +1; diff --git a/src/lib/Hydra/Schema/Builds.pm b/src/lib/Hydra/Schema/Builds.pm index a5abce08a..5e5024495 100644 --- a/src/lib/Hydra/Schema/Builds.pm +++ b/src/lib/Hydra/Schema/Builds.pm @@ -341,6 +341,21 @@ __PACKAGE__->has_many( undef, ); +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { "foreign.build" => "self.id" }, + undef, +); + =head2 buildoutputs Type: has_many @@ -535,8 +550,8 @@ __PACKAGE__->many_to_many( ); -# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:03:55 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:EwxiaQpqbdzI9RvU0uUtLQ +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Y2lDtgY8EBLOuCHAI8fWRQ __PACKAGE__->has_many( "dependents", @@ -630,6 +645,7 @@ my %hint = ( buildoutputs => 'name', buildinputs_builds => 'name', buildproducts => 'productnr', + buildmetrics => 'name', } ); diff --git a/src/lib/Hydra/Schema/Jobs.pm b/src/lib/Hydra/Schema/Jobs.pm index 883c1bee1..cd89ed3d0 100644 --- a/src/lib/Hydra/Schema/Jobs.pm +++ b/src/lib/Hydra/Schema/Jobs.pm @@ -81,6 +81,25 @@ __PACKAGE__->set_primary_key("project", "jobset", "name"); =head1 RELATIONS +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { + "foreign.job" => "self.name", + "foreign.jobset" => "self.jobset", + "foreign.project" => "self.project", + }, + undef, +); + =head2 builds Type: has_many @@ -150,7 +169,7 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-09-29 19:41:42 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:lnZSd0gDXgLk8WQeAFqByA +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:vDAo9bzLca+QWfhOb9OLMg 1; diff --git a/src/lib/Hydra/Schema/Jobsets.pm b/src/lib/Hydra/Schema/Jobsets.pm index f304f770d..682b3ab0d 100644 --- a/src/lib/Hydra/Schema/Jobsets.pm +++ b/src/lib/Hydra/Schema/Jobsets.pm @@ -184,6 +184,24 @@ __PACKAGE__->set_primary_key("project", "name"); =head1 RELATIONS +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { + "foreign.jobset" => "self.name", + "foreign.project" => "self.project", + }, + undef, +); + =head2 builds Type: has_many @@ -320,8 +338,8 @@ __PACKAGE__->has_many( ); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-04-23 23:13:51 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:CO0aE+jrjB+UrwGRzWZLlw +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:Coci9FdBAvUO9T3st2NEqA my %hint = ( columns => [ diff --git a/src/lib/Hydra/Schema/Projects.pm b/src/lib/Hydra/Schema/Projects.pm index 9bf9dde3e..e04b1f8ef 100644 --- a/src/lib/Hydra/Schema/Projects.pm +++ b/src/lib/Hydra/Schema/Projects.pm @@ -106,6 +106,21 @@ __PACKAGE__->set_primary_key("name"); =head1 RELATIONS +=head2 buildmetrics + +Type: has_many + +Related object: L + +=cut + +__PACKAGE__->has_many( + "buildmetrics", + "Hydra::Schema::BuildMetrics", + { "foreign.project" => "self.name" }, + undef, +); + =head2 builds Type: has_many @@ -267,8 +282,8 @@ Composing rels: L -> username __PACKAGE__->many_to_many("usernames", "projectmembers", "username"); -# Created by DBIx::Class::Schema::Loader v0.07033 @ 2014-04-23 23:13:08 -# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:fkd9ruEoVSBGIktmAj4u4g +# Created by DBIx::Class::Schema::Loader v0.07043 @ 2015-07-30 16:52:20 +# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:67kWIE0IGmEJTvOIATAKaw my %hint = ( columns => [ diff --git a/src/root/build.tt b/src/root/build.tt index 9ecd5cd61..e3eb43f90 100644 --- a/src/root/build.tt +++ b/src/root/build.tt @@ -106,7 +106,7 @@ [% END %] - +
  • Summary
    • [% IF isAggregate %]
  • Constituents
    • [% END %]
  • Details
    • @@ -385,6 +385,25 @@ [% END %] + + [% IF build.finished && build.buildmetrics %] +

    Metrics

    + + + + + + + [% FOREACH metric IN build.buildmetrics %] + + + + + [% END %] + +
    NameValue
    [%HTML.escape(metric.name)%][%metric.value%][%metric.unit%]
    + [% END %] +
    diff --git a/src/root/common.tt b/src/root/common.tt index 19e2462ea..dafae8aab 100644 --- a/src/root/common.tt +++ b/src/root/common.tt @@ -566,12 +566,14 @@ BLOCK createChart %] success: function(data) { var ids = []; var d = []; - var max = 0; + var maxTime = 0; + var minTime = Number.MAX_SAFE_INTEGER; data.forEach(function(x) { var t = x.timestamp * 1000; ids[t] = x.id; d.push([t, x.value [% IF yaxis == "mib" %] / (1024.0 * 1024.0)[% END %]]); - max = Math.max(t, max); + maxTime = Math.max(t, maxTime); + minTime = Math.min(t, minTime); }); var options = { @@ -634,7 +636,7 @@ BLOCK createChart %] }); // Zoom in to the last two months by default. - plot.setSelection({ xaxis: { from: max - 60 * 24 * 60 * 60 * 1000, to: max } }); + plot.setSelection({ xaxis: { from: Math.max(minTime, maxTime - 60 * 24 * 60 * 60 * 1000), to: maxTime } }); } }); }); diff --git a/src/root/job.tt b/src/root/job.tt index 998bc0831..a9e1ca779 100644 --- a/src/root/job.tt +++ b/src/root/job.tt @@ -98,6 +98,14 @@ removed or had an evaluation error.
    [% INCLUDE createChart id="output-size" yaxis="mib" dataUrl=c.uri_for('/job' project.name jobset.name job.name 'output-sizes') %] + [% FOREACH metric IN metrics %] + +

    Metric: [%HTML.escape(metric.name)%]

    + + [% INCLUDE createChart id="metric-${metric.name}" dataUrl=c.uri_for('/job' project.name jobset.name job.name 'metric' metric.name) %] + + [% END %] +