Console.WriteLine("Execution Date/time is " + DateTime.Now);
Execution Date/time is 11/24/2023 7:53:12 PM
System.Security.Principal.WindowsIdentity.GetCurrent().Name
FDY\JenkinsCI
#r "BoSSSpad.dll"
using System;
using System.Collections.Generic;
using System.Linq;
using ilPSP;
using ilPSP.Utils;
using BoSSS.Platform;
using BoSSS.Platform.LinAlg;
using BoSSS.Foundation;
using BoSSS.Foundation.XDG;
using BoSSS.Foundation.Grid;
using BoSSS.Foundation.Grid.Classic;
using BoSSS.Foundation.Grid.RefElements;
using BoSSS.Foundation.IO;
using BoSSS.Solution;
using BoSSS.Solution.Control;
using BoSSS.Solution.GridImport;
using BoSSS.Solution.Statistic;
using BoSSS.Solution.Utils;
using BoSSS.Solution.AdvancedSolvers;
using BoSSS.Solution.Gnuplot;
using BoSSS.Application.BoSSSpad;
using BoSSS.Application.XNSE_Solver;
using BoSSS.Application.XNSFE_Solver;
using static BoSSS.Application.BoSSSpad.BoSSSshell;
Init();
string PROJECT_NAME = System.Environment.GetEnvironmentVariable("LinslvPerfSer") ?? "LinslvPerfSer"; // this allows to modify the project name for testing purposes
wmg.Init(PROJECT_NAME);
wmg.SetNameBasedSessionJobControlCorrelation();
wmg.AllJobs
Project name is set to 'LinslvPerfSer'. Default Execution queue is chosen for the database. Opening existing database '\\fdygitrunner\ValidationTests\databases\LinslvPerfSer'.
// for examination on the local workstation,
//OpenDatabase(@"\\fdygitrunner\ValidationTests\LinslvPerf_XdgStokes");
/*
// extract the control object to reproduce specific simulation on local workstation
string dest = @"C:\Users\flori\Documents\BoSSS-kummer\public\src\L4-application\XNSE_Solver\bin\Release\net5.0\BenchControls";
foreach(var s in wmg.Sessions) {
string name = s.Name;
Console.Write(name + ": ");
try {
var ctrl = s.GetControl();
var txt = ctrl.Serialize();
string DestPath = System.IO.Path.Combine(dest, name + ".obj");
System.IO.File.WriteAllText(DestPath, txt);
Console.WriteLine("written");
} catch(Exception e) {
Console.WriteLine(e.Message);
}
}
*/
//var ss = wmg.Sessions.Where(sess => sess.Name.Contains("XdgStokes"));
//foreach(var s in ss)
// s.Delete(true);
static class Utils {
// DOF per cell in 3D for one variable
public static int Np(int p) {
return (p*p*p + 6*p*p + 11*p + 6)/6;
}
/*
//Non-equidistant nodes
public static double[] SinLinSpacing(double l, double r, double a, int n) {
double[] linnodes = GenericBlas.Linspace(-Math.PI * 0.5, Math.PI * 0.5, n);
double[] linnodes2 = GenericBlas.Linspace(-1, 1, n);
double[] nodes = new double[n];
for (int i = 0; i < n; i++)
//nodes[i] = linnodes2[i] * (1 - a) + (1.0 - Math.Sin(linnodes[i])) * a;
nodes[i] = linnodes2[i] * (1 - a) + Math.Sin(linnodes[i])*a;
for (int i = 0; i < n; i++)
nodes[i] = nodes[i] * (r - l)*0.5 + l;
return nodes;
}
*/
}
Note: The dimension of the domain $(-1,1)^3$ are assumed to be centimeters! In this benchmark, realistic physical values are used, i.e. densities and viscosities of
int[] Resolutions_3D = new int[] { 8, 16, 24, 32, 48, 64 };
//int[] Resolutions_3D = new int[] { 2 };
IGridInfo[] grids = new IGridInfo[Resolutions_3D.Length];
for(int cnt = 0; cnt < Resolutions_3D.Length; cnt++) {
int Res = Resolutions_3D[cnt];
double[] xNodes = GenericBlas.Linspace(-1, +1, Res + 1);
double[] yNodes = GenericBlas.Linspace(-1, +1, Res + 1);
double[] zNodes = GenericBlas.Linspace(-1, +1, Res + 1);
int J = (xNodes.Length - 1)*(yNodes.Length - 1)*(zNodes.Length - 1);
string GridName = string.Format(wmg.CurrentProject + "-XdgStokes_J" + J);
grids[cnt] = wmg.Grids.SingleOrDefault(grd => grd.Name.Contains(GridName)); // check if an appropriate grid is already present in the database
if(grids[cnt] == null){
Console.WriteLine("Creating grid with " + J + " cells.");
GridCommons g;
g = Grid3D.Cartesian3DGrid(xNodes, yNodes, zNodes);
g.Name = GridName;
g.DefineEdgeTags(delegate (double[] X) {
if (Math.Abs(X[0] - (-1)) <= 1.0e-8)
return "wall_left";
if (Math.Abs(X[0] - (+1)) <= 1.0e-8)
return "wall_right";
if (Math.Abs(X[1] - (-1)) <= 1.0e-8)
return "wall_front";
if (Math.Abs(X[1] - (+1)) <= 1.0e-8)
return "wall_back";
if (Math.Abs(X[2] - (-1)) <= 1.0e-8)
return "wall_top";
if (Math.Abs(X[2] - (+1)) <= 1.0e-8)
return "wall_bottom";
throw new ArgumentException("unknown wall");
});
g = wmg.SaveGrid(g);
grids[cnt] = g;
} else {
Console.WriteLine("Found Grid: " + grids[cnt]);
if(grids[cnt].NumberOfCells != J)
throw new Exception("J mismatch");
if(grids[cnt].SpatialDimension != 3)
throw new Exception("D mismatch");
}
}
Opening existing database '\\fdygitrunner\BoSSStests\LinslvPerfSer'.
Found Grid: { Guid = 06efeda7-55ef-452d-a5a2-d60c3114d5a5; Name = LinslvPerfSer-XdgStokes_J512; Cell Count = 512; Dim = 3 }
Found Grid: { Guid = a413248c-89e0-42ac-955a-653b84831644; Name = LinslvPerfSer-XdgStokes_J4096; Cell Count = 4096; Dim = 3 }
Found Grid: { Guid = a7dab9da-d03e-4714-a40f-b8e094f945a5; Name = LinslvPerfSer-XdgStokes_J13824; Cell Count = 13824; Dim = 3 }
Found Grid: { Guid = a1d4cdf5-c0ae-4cf7-9424-47bd090072ec; Name = LinslvPerfSer-XdgStokes_J32768; Cell Count = 32768; Dim = 3 }
Found Grid: { Guid = b42078e0-4f47-4336-94fe-920600283b77; Name = LinslvPerfSer-XdgStokes_J110592; Cell Count = 110592; Dim = 3 }
Found Grid: { Guid = da153024-457d-4086-8b98-e0e42b7d07ad; Name = LinslvPerfSer-XdgStokes_J262144; Cell Count = 262144; Dim = 3 }
grids
#0: { Guid = 06efeda7-55ef-452d-a5a2-d60c3114d5a5; Name = LinslvPerfSer-XdgStokes_J512; Cell Count = 512; Dim = 3 }
#1: { Guid = a413248c-89e0-42ac-955a-653b84831644; Name = LinslvPerfSer-XdgStokes_J4096; Cell Count = 4096; Dim = 3 }
#2: { Guid = a7dab9da-d03e-4714-a40f-b8e094f945a5; Name = LinslvPerfSer-XdgStokes_J13824; Cell Count = 13824; Dim = 3 }
#3: { Guid = a1d4cdf5-c0ae-4cf7-9424-47bd090072ec; Name = LinslvPerfSer-XdgStokes_J32768; Cell Count = 32768; Dim = 3 }
#4: { Guid = b42078e0-4f47-4336-94fe-920600283b77; Name = LinslvPerfSer-XdgStokes_J110592; Cell Count = 110592; Dim = 3 }
#5: { Guid = da153024-457d-4086-8b98-e0e42b7d07ad; Name = LinslvPerfSer-XdgStokes_J262144; Cell Count = 262144; Dim = 3 }
//var ss = wmg.Sessions.Where(sess => sess.Name.Contains("XdgStokes") && sess.Name.Contains("exp_Kcycle_schwarz"));
//foreach(var s in ss)
// s.Delete(true);
// - - - - - - - - - - - - - - - - - - -
// Initial Values & Boundary conditions
// - - - - - - - - - - - - - - - - - - -
using BoSSS.Application.XNSE_Solver;
using BoSSS.Solution.LevelSetTools;
using BoSSS.Solution.AdvancedSolvers;
using BoSSS.Solution.XNSECommon;
using BoSSS.Solution.Timestepping;
using BoSSS.Solution.XdgTimestepping ;
Polynomial degrees to test:
int[] PolyDegS = new int[] {2, 3 ,5};
Solvers which we want to instrument:
// Solvers which we want to instrument:
LinearSolverCode[] solver_nameS = new LinearSolverCode[] {
LinearSolverCode.direct_pardiso,
LinearSolverCode.exp_gmres_levelpmg,
LinearSolverCode.exp_Kcycle_schwarz,
LinearSolverCode.pMultigrid
};
Maximum Dof for one calculation (we skip fine grids for higher polynomial orders):
int GetMaxAllowedDOF(LinearSolverCode code) {
switch(code) {
case LinearSolverCode.direct_pardiso:
case LinearSolverCode.direct_mumps:
case LinearSolverCode.pMultigrid:
return 700000; // 0.7 Million for direct solvers and p-MG at maximum
default:
return 3000000; // Up to 3 Million for iterative solvers
}
}
Loop over all combinations of parameters and define a control object for each combo:
using BoSSS.Solution.XNSECommon;
using BoSSS.Foundation.XDG;
List<XNSE_Control> controls = new List<XNSE_Control>();
controls.Clear();
foreach(LinearSolverCode solver in solver_nameS) {
foreach(int k in PolyDegS) {
foreach(IGridInfo grd in grids) {
int Np_V = Utils.Np(k);
int Np_p = Utils.Np(k-1);
int J = grd.NumberOfCells;
int DOF = Np_V*3 + Np_p;
if(J*DOF > GetMaxAllowedDOF(solver))
continue;
if(solver == LinearSolverCode.pMultigrid && k <= 2)
continue; // p-multigrid cannot really work with only two p-levels
if(solver == LinearSolverCode.exp_gmres_levelpmg && k > 3)
continue; // two-grid is ineffective for higher polynomial degrees
if(k >= 5 && J >= 13824) // 09oct23: calculation lasts more than one day
continue;
if(k >= 3 && J >= 32768) // 09oct23: calculation lasts more than one day
continue;
// Control Instance, grid, DG degree, etc.
// =======================================
XNSE_Control C = new XNSE_Control();
controls.Add(C);
string caseName = string.Format("XdgStokes-J{0}_p{1}_{2}", J, k, solver);
Console.WriteLine("setting up: " + caseName);
C.SessionName = caseName;
C.SetGrid(grd);
C.savetodb = true;
C.SetDGdegree(k);
// Phys. Parameters
// ================
// Species A: Water; Species B: Air
C.PhysicalParameters.rho_A = 1.0; //1e-3; // kg / cm³
C.PhysicalParameters.rho_B = 1.0; //1.2e-6; // kg / cm³
C.PhysicalParameters.mu_A = 1.0; //1e-5; // kg / cm / sec
C.PhysicalParameters.mu_B = 1.0; //17.1e-8; // kg / cm / sec
C.PhysicalParameters.Sigma = 72.75e-3; // kg / sec²
C.PhysicalParameters.IncludeConvection = false;
C.PhysicalParameters.Material = true;
// Dont know
// ============
double r = 0.5;
double nonsp = 0.5;
C.AddInitialValue("Phi", new Formula($"X => (X[0]/{r*nonsp}).Pow2() + (X[1]/{r}).Pow2() + (X[2]/{r}).Pow2() - 1", false));
C.LSContiProjectionMethod = BoSSS.Solution.LevelSetTools.ContinuityProjectionOption.None;
//C.CutCellQuadratureType = BoSSS.Foundation.XDG.XQuadFactoryHelper.MomentFittingVariants.Saye;
C.ComputeEnergyProperties = false;
// Solver Stuff
// ============
C.LinearSolver = solver.GetConfig();
if(C.LinearSolver is IterativeSolverConfig isc) {
isc.ConvergenceCriterion = 1e-8;
}
C.NoOfMultigridLevels = 100;
C.TracingNamespaces = "BoSSS.Solution";
C.LevelSet_ConvergenceCriterion = 1e-6;
//C.Option_LevelSetEvolution = LevelSetEvolution.FastMarching;
C.AdvancedDiscretizationOptions.SST_isotropicMode = SurfaceStressTensor_IsotropicMode.Curvature_Projected;
//C.AdvancedDiscretizationOptions.ViscosityMode = ViscosityMode.Standard;
C.AdvancedDiscretizationOptions.FilterConfiguration = CurvatureAlgorithms.FilterConfiguration.NoFilter;
// Timestepping / Instationary
// ===========================
C.TimesteppingMode = AppControl._TimesteppingMode.Steady;
//C.dtFixed = 0.01;
}
}
}
setting up: XdgStokes-J512_p2_direct_pardiso setting up: XdgStokes-J4096_p2_direct_pardiso setting up: XdgStokes-J13824_p2_direct_pardiso setting up: XdgStokes-J512_p3_direct_pardiso setting up: XdgStokes-J4096_p3_direct_pardiso setting up: XdgStokes-J512_p5_direct_pardiso setting up: XdgStokes-J512_p2_exp_gmres_levelpmg setting up: XdgStokes-J4096_p2_exp_gmres_levelpmg setting up: XdgStokes-J13824_p2_exp_gmres_levelpmg setting up: XdgStokes-J32768_p2_exp_gmres_levelpmg setting up: XdgStokes-J512_p3_exp_gmres_levelpmg setting up: XdgStokes-J4096_p3_exp_gmres_levelpmg setting up: XdgStokes-J13824_p3_exp_gmres_levelpmg setting up: XdgStokes-J512_p2_exp_Kcycle_schwarz setting up: XdgStokes-J4096_p2_exp_Kcycle_schwarz setting up: XdgStokes-J13824_p2_exp_Kcycle_schwarz setting up: XdgStokes-J32768_p2_exp_Kcycle_schwarz setting up: XdgStokes-J512_p3_exp_Kcycle_schwarz setting up: XdgStokes-J4096_p3_exp_Kcycle_schwarz setting up: XdgStokes-J13824_p3_exp_Kcycle_schwarz setting up: XdgStokes-J512_p5_exp_Kcycle_schwarz setting up: XdgStokes-J4096_p5_exp_Kcycle_schwarz setting up: XdgStokes-J512_p3_pMultigrid setting up: XdgStokes-J4096_p3_pMultigrid setting up: XdgStokes-J512_p5_pMultigrid
Total number of simulations:
controls.Count
// Assert that the location of the fluid phases is as desired:
NUnit.Framework.Assert.Negative(controls[0].InitialValues["Phi"].Evaluate(new double[]{0,0.0,0}, 0.0), "phase A (negative) must be inside");
NUnit.Framework.Assert.Positive(controls[0].InitialValues["Phi"].Evaluate(new double[]{0,4.0,0}, 0.0), "phase B (positive) must be inside");
// Basic checks on the multigrid configuration
foreach(var ctrl in controls) {
NUnit.Framework.Assert.Greater(ctrl.NoOfMultigridLevels, 1, "More than 1 multigrid level must be set");
if(ctrl.LinearSolver is OrthoMGSchwarzConfig osc)
NUnit.Framework.Assert.Greater(osc.NoOfMultigridLevels, 1, "More than 1 multigrid level must be set");
}
//string path = @"C:\Users\flori\Documents\BoSSS-kummer\public\src\L4-application\XNSE_Solver\bin\Release\net6.0\control-ser";
//foreach(var ctrl in controls) {
// ctrl.savetodb = false;
// ctrl.SaveToFile(System.IO.Path.Combine(path, ctrl.SessionName + ".obj"));
//}
Use the default queue defined on this machine:
ExecutionQueues
| index | RuntimeLocation | DeploymentBaseDirectory | DeployRuntime | Name | DotnetRuntime | Username | ServerName | ComputeNodes | DefaultJobPriority | SingleNode | AllowedDatabasesPaths |
|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | win\amd64 | \\fdygitrunner\BoSSStests | True | MSHPC-Gitrunner-HighPrio | dotnet | FDY\jenkinsci | DC2 | [ fdygitrunner ] | Highest | True | [ \\fdygitrunner\BoSSStests, \\fdygitrunner\ValidationTests\databases ] |
| 1 | win\amd64 | \\fdygitrunner\BoSSStests | False | MSHPC-Gitrunner-DefaultTest | dotnet | FDY\jenkinsci | DC2 | [ hpccluster, hpccluster2, hpcluster3, hpccluster4, fdygitrunner ] | Normal | True | [ \\fdygitrunner\BoSSStests, \\fdygitrunner\ValidationTests\databases ] |
| 2 | win\amd64 | \\fdygitrunner\ValidationTests\deploy | True | MSHPC-AllNodes | dotnet | FDY\jenkinsci | DC2 | [ hpccluster, hpccluster2, hpcluster3, hpccluster4, fdygitrunner ] | Normal | True | [ \\fdygitrunner\ValidationTests\databases ] |
| 3 | win\amd64 | \\fdygitrunner\BoSSStests | True | MSHPC-AllNodes-test | dotnet | FDY\jenkinsci | DC2 | [ hpccluster, hpccluster2, hpcluster3, hpccluster4, fdygitrunner ] | Normal | True | [ \\fdygitrunner\BoSSStests ] |
| 4 | win\amd64 | \\fdygitrunner\ValidationTests\deploy | True | MSHPC-FastNodes | dotnet | FDY\jenkinsci | DC2 | [ hpcluster3, hpccluster4 ] | Normal | True | [ \\fdygitrunner\ValidationTests\databases ] |
var myBatch = GetDefaultQueue();
myBatch
| RuntimeLocation | DeploymentBaseDirectory | DeployRuntime | Name | DotnetRuntime | Username | ServerName | ComputeNodes | DefaultJobPriority | SingleNode | AllowedDatabasesPaths |
|---|---|---|---|---|---|---|---|---|---|---|
win\amd64 | \\fdygitrunner\ValidationTests\deploy | True | MSHPC-FastNodes | dotnet | FDY\jenkinsci | DC2 | [ hpcluster3, hpccluster4 ] | Normal | True | [ \\fdygitrunner\ValidationTests\databases ] |
foreach(var ctrl in controls) {
Console.WriteLine(" Submitting: " + ctrl.SessionName);
var j = ctrl.CreateJob();
j.RetryCount = 1;
j.Activate(myBatch);
//ctrl.RunBatch();
}
Submitting: XdgStokes-J512_p2_direct_pardiso Deployments so far (1): (Job token: 802787, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125105.140834' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p2_direct_pardiso 10/29/2023 12:51:24 94eb907b..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J4096_p2_direct_pardiso Deployments so far (1): (Job token: 802788, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125122.431399' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J4096_p2_direct_pardiso 10/29/2023 12:51:40 eeaa7f35..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J13824_p2_direct_pardiso Deployments so far (1): (Job token: 802789, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125140.492559' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J13824_p2_direct_pardiso 10/29/2023 12:51:59 922153a6..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J512_p3_direct_pardiso Deployments so far (1): (Job token: 802790, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125158.648686' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p3_direct_pardiso 10/29/2023 12:52:17 90faae1d..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J4096_p3_direct_pardiso Deployments so far (1): (Job token: 802791, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125217.307282' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J4096_p3_direct_pardiso 10/29/2023 12:52:35 7956c7e2..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J512_p5_direct_pardiso Deployments so far (1): (Job token: 802792, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125235.893166' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p5_direct_pardiso 10/29/2023 12:52:54 888e2466..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J512_p2_exp_gmres_levelpmg Deployments so far (1): (Job token: 802793, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125253.993337' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p2_exp_gmres_levelpmg 10/29/2023 12:53:13 f95c76c4..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J4096_p2_exp_gmres_levelpmg Deployments so far (1): (Job token: 802794, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125312.764337' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J4096_p2_exp_gmres_levelpmg 10/29/2023 12:53:32 a8683c7d..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J13824_p2_exp_gmres_levelpmg Deployments so far (1): (Job token: 802795, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125331.599634' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J13824_p2_exp_gmres_levelpmg 10/29/2023 12:53:52 a77248ee..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J32768_p2_exp_gmres_levelpmg Deployments so far (1): (Job token: 802796, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125350.743560' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J32768_p2_exp_gmres_levelpmg 10/29/2023 12:54:14 6873a35d..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J512_p3_exp_gmres_levelpmg Deployments so far (1): (Job token: 802797, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125411.966361' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p3_exp_gmres_levelpmg 10/29/2023 12:54:31 784be56e..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J4096_p3_exp_gmres_levelpmg Deployments so far (1): (Job token: 802798, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125434.721149' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J4096_p3_exp_gmres_levelpmg 10/29/2023 12:54:53 7a1ba509..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J13824_p3_exp_gmres_levelpmg Deployments so far (1): (Job token: 802799, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125452.372679' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J13824_p3_exp_gmres_levelpmg 10/29/2023 12:55:12 0250840e..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J512_p2_exp_Kcycle_schwarz Deployments so far (1): (Job token: 802800, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125511.070971' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p2_exp_Kcycle_schwarz 10/29/2023 12:55:30 7870a1e7..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J4096_p2_exp_Kcycle_schwarz Deployments so far (1): (Job token: 802801, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125530.626047' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J4096_p2_exp_Kcycle_schwarz 10/29/2023 12:55:51 164cd04a..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J13824_p2_exp_Kcycle_schwarz Deployments so far (1): (Job token: 802802, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125549.254293' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J13824_p2_exp_Kcycle_schwarz 10/29/2023 12:56:12 7ea65a09..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J32768_p2_exp_Kcycle_schwarz Deployments so far (1): (Job token: 802803, FailedOrCanceled 'LinslvPerfSer-XNSE_Solver2023Oct29_125610.535923' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FailedOrCanceled); Success: 0 job submit count: 1 Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action. Hint: you might either remove old deployments or increase the 'RetryCount'. No submission, because job status is: FailedOrCanceled Submitting: XdgStokes-J512_p3_exp_Kcycle_schwarz Deployments so far (1): (Job token: 802804, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125629.564118' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p3_exp_Kcycle_schwarz 10/29/2023 12:56:50 4976dcdc..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J4096_p3_exp_Kcycle_schwarz Deployments so far (1): (Job token: 802805, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125648.664299' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J4096_p3_exp_Kcycle_schwarz 10/29/2023 12:57:08 c0046bc4..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J13824_p3_exp_Kcycle_schwarz Deployments so far (1): (Job token: 802806, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125708.871111' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J13824_p3_exp_Kcycle_schwarz 10/29/2023 12:57:28 92f17ffd..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J512_p5_exp_Kcycle_schwarz Deployments so far (1): (Job token: 802807, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125728.112810' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p5_exp_Kcycle_schwarz 10/29/2023 12:57:48 cf4efdde..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J4096_p5_exp_Kcycle_schwarz Deployments so far (1): (Job token: 802808, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125747.022024' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J4096_p5_exp_Kcycle_schwarz 10/29/2023 12:58:07 d128cf24..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J512_p3_pMultigrid Deployments so far (1): (Job token: 802809, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125806.379176' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p3_pMultigrid 10/29/2023 12:58:28 985dd0ce..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J4096_p3_pMultigrid Deployments so far (1): (Job token: 802810, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125825.802845' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J4096_p3_pMultigrid 10/29/2023 12:58:48 17fdc5b2..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful Submitting: XdgStokes-J512_p5_pMultigrid Deployments so far (1): (Job token: 802811, FinishedSuccessful 'LinslvPerfSer-XNSE_Solver2023Oct29_125844.900161' @ MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy, FinishedSuccessful); Success: 1 Info: Found successful session "LinslvPerfSer XdgStokes-J512_p5_pMultigrid 10/29/2023 12:59:05 bac65688..." -- job is marked as successful, no further action. No submission, because job status is: FinishedSuccessful
wmg.BlockUntilAllJobsTerminate(3600*24*2); // wait at maximum two days for the jobs to finish
All jobs finished.
wmg.AllJobs
#0: XdgStokes-J512_p2_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p2_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #1: XdgStokes-J4096_p2_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J4096_p2_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #2: XdgStokes-J13824_p2_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J13824_p2_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #3: XdgStokes-J512_p3_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p3_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #4: XdgStokes-J4096_p3_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J4096_p3_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #5: XdgStokes-J512_p5_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p5_direct_pardiso: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #6: XdgStokes-J512_p2_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p2_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #7: XdgStokes-J4096_p2_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J4096_p2_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #8: XdgStokes-J13824_p2_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J13824_p2_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #9: XdgStokes-J32768_p2_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J32768_p2_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #10: XdgStokes-J512_p3_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p3_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #11: XdgStokes-J4096_p3_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J4096_p3_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #12: XdgStokes-J13824_p3_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J13824_p3_exp_gmres_levelpmg: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #13: XdgStokes-J512_p2_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p2_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #14: XdgStokes-J4096_p2_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J4096_p2_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #15: XdgStokes-J13824_p2_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J13824_p2_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #16: XdgStokes-J32768_p2_exp_Kcycle_schwarz: FailedOrCanceled (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J32768_p2_exp_Kcycle_schwarz: FailedOrCanceled (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #17: XdgStokes-J512_p3_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p3_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #18: XdgStokes-J4096_p3_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J4096_p3_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #19: XdgStokes-J13824_p3_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J13824_p3_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #20: XdgStokes-J512_p5_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p5_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #21: XdgStokes-J4096_p5_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J4096_p5_exp_Kcycle_schwarz: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #22: XdgStokes-J512_p3_pMultigrid: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p3_pMultigrid: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #23: XdgStokes-J4096_p3_pMultigrid: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J4096_p3_pMultigrid: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) #24: XdgStokes-J512_p5_pMultigrid: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy) XdgStokes-J512_p5_pMultigrid: FinishedSuccessful (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy)
//foreach(var j in wmg.AllJobs.Values) {
// j.DeleteOldDeploymentsAndSessions();
//}
wmg.Sessions.Where(sess => sess.Name.StartsWith("XdgStokes_J"))
//wmg.Sessions[3].Export().WithSupersampling(2).Do()
2+2
var NoSuccess = controls.Select(ctrl => ctrl.GetJob()).Where(job => job.Status != JobStatus.FinishedSuccessful).ToArray();
NoSuccess
#0: XdgStokes-J32768_p2_exp_Kcycle_schwarz: FailedOrCanceled (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy)
// In the case of some failed job, print the directory name for further inspection:
foreach(var fail in NoSuccess)
Console.WriteLine(fail + ": @" + ((fail.LatestDeployment?.DeploymentDirectory?.FullName) ?? " no deployment directory"));
//Console.WriteLine(fail.LatestDeployment);
XdgStokes-J32768_p2_exp_Kcycle_schwarz: FailedOrCanceled (MS HPC client MSHPC-FastNodes @DC2, @\\fdygitrunner\ValidationTests\deploy): @\\fdygitrunner\ValidationTests\deploy\LinslvPerfSer-XNSE_Solver2023Oct29_125610.535923
/*
string PathOffset = @"C:\Users\jenkinsci\Desktop\LinSlvPerfFail-20apr22";
foreach(var fail in NoSuccess) {
var C = fail.GetControl();
C.savetodb = false;
C.SaveToFile(System.IO.Path.Combine(PathOffset, fail.Name + ".obj"));
string Stdout = fail.Stdout;
System.IO.File.WriteAllText(System.IO.Path.Combine(PathOffset, fail.Name + "-stdout.txt"), Stdout);
string Stderr = fail.Stderr;
System.IO.File.WriteAllText(System.IO.Path.Combine(PathOffset, fail.Name + "-stderr.txt"), Stderr);
}
*/
var FailedSessions = wmg.Sessions.Where(Si => Si.Name.Contains("XdgStokes") && Si.SuccessfulTermination == false);
FailedSessions
#0: LinslvPerfSer XdgStokes-J32768_p2_exp_Kcycle_schwarz* 10/29/2023 12:56:31 8b973436...
//foreach(var n in wmg.Sessions.Where(Si => Si.Name.Contains("XdgStokes") && Si.SuccessfulTermination == false).Select(s => s.Name)) {
//Console.WriteLine("\"" + n + "\",");
//}
Remark: since this is currently (22 Apr. 2022) work-in-progress, we allow for some jobs to fail; At this intermediate milestone, I want to record (by the means of tests) what is already working. Thereby, I hope I won't break the working cases while trying to fix the failing ones.
var prelim_allowedFails = new[] {
"XdgStokes-J512_p5_pMultigrid",
"XdgStokes-J4096_p3_pMultigrid",
"XdgStokes-J13824_p5_exp_Kcycle_schwarz",
"XdgStokes-J512_p3_pMultigrid",
"XdgStokes-J4096_p5_exp_Kcycle_schwarz",
"XdgStokes-J32768_p2_exp_Kcycle_schwarz",
"XdgStokes-J32768_p3_exp_gmres_levelpmg",
"XdgStokes-J13824_p3_exp_gmres_levelpmg",
"XdgStokes-J4096_p3_exp_gmres_levelpmg",
"XdgStokes-J32768_p2_exp_gmres_levelpmg",
"XdgStokes-J13824_p2_exp_gmres_levelpmg",
"XdgStokes-J512_p5_direct_pardiso",
"XdgStokes-J4096_p3_direct_pardiso",
"XdgStokes-J13824_p2_direct_pardiso",
"XdgStokes-J4096_p2_exp_Kcycle_schwarz"
};
bool FailAllowed(string name) {
foreach(string s in prelim_allowedFails) {
if(s.WildcardMatch(name))
return true;
}
return false;
}
NoSuccess.Where(job => !FailAllowed(job.Name))
NUnit.Framework.Assert.Zero(NoSuccess.Where(job => !FailAllowed(job.Name)).Count(), "Some Jobs Failed");
NUnit.Framework.Assert.Zero(FailedSessions.Where(s => !FailAllowed(s.Name)).Count(), "Some Sessions did not terminate successfully.");