Linear Solver Performance: Constant Coefficient Poisson, MPI-parallel

Part 1, Benchmark Setup and Execution

Console.WriteLine("Execution Date/time is " + DateTime.Now);

The below script needs to be able to find the current output cell; this is an easy method to get it.

Execution Date/time is 12/4/2023 9:33:56 PM

#r "BoSSSpad.dll"
//#r "../../../../src/L4-application/BoSSSpad/bin/Debug/net6.0/BoSSSpad.dll"
using System;
using System.Collections.Generic;
using System.Linq;
using ilPSP;
using ilPSP.Utils;
using BoSSS.Platform;
using BoSSS.Platform.LinAlg;
using BoSSS.Platform.Utils.Geom;
using BoSSS.Foundation;
using BoSSS.Foundation.XDG;
using BoSSS.Foundation.Grid;
using BoSSS.Foundation.Grid.Classic;
using BoSSS.Foundation.Grid.RefElements;
using BoSSS.Foundation.IO;
using BoSSS.Solution;
using BoSSS.Solution.Control;
using BoSSS.Solution.GridImport;
using BoSSS.Solution.Statistic;
using BoSSS.Solution.Utils;
using BoSSS.Solution.AdvancedSolvers;
using BoSSS.Solution.Gnuplot;
using BoSSS.Application.BoSSSpad;
using BoSSS.Application.XNSE_Solver;
using BoSSS.Application.GridGen;
using static BoSSS.Application.BoSSSpad.BoSSSshell;
Init();

using BoSSS.Application.SipPoisson;

string PROJECT_NAME = System.Environment.GetEnvironmentVariable("LinslvPerfPar") ?? "LinslvPerfPar"; // this allows to modify the project name for testing purposes
wmg.Init(PROJECT_NAME);
wmg.SetNameBasedSessionJobControlCorrelation();
wmg.AllJobs

Project name is set to 'LinslvPerfPar'.
Default Execution queue is chosen for the database.
Opening existing database '/mnt/lb17-scratch-fk69umer/jenkins/databases/LinslvPerfPar'.

ExecutionQueues

index	Username	Password	ServerName	PrivateKeyFilePath	AdditionalBatchCommands	ExecutionTime	DeploymentBaseDirectoryAtRemote	RuntimeLocation	SlurmAccount	Email	DeploymentBaseDirectory	DeployRuntime	Name	DotnetRuntime	MonoDebug	AllowedDatabasesPaths
0	fk69umer	<null>	lcluster18.hrz.tu-darmstadt.de	/home/administrator/.ssh/id_ed25519	[ #SBATCH -C avx512, #SBATCH --mem-per-cpu=8000 ]	02:00:00	/work/scratch/fk69umer/jenkins/deploy	linux/amd64-openmpi	special00006	<null>	/mnt/lb17-scratch-fk69umer/jenkins/deploy	False	Lb2-specialPrj	dotnet	False	[ /mnt/lb17-scratch-fk69umer/jenkins/databases == /work/scratch/fk69umer/jenkins/databases ]
1	fk69umer	<null>	lcluster18.hrz.tu-darmstadt.de	/home/administrator/.ssh/id_ed25519	[ #SBATCH -C avx512, #SBATCH --mem-per-cpu=8000 ]	02:00:00	/work/scratch/fk69umer/jenkins/deploy	linux/amd64-openmpi	special00006	<null>	/mnt/lb17-scratch-fk69umer/jenkins/deploy	False	Lb2-specialPrj	dotnet	False	[ /mnt/lb17-scratch-fk69umer/jenkins/databases == /work/scratch/fk69umer/jenkins/databases ]

GetDefaultQueue()

Username	Password	ServerName	PrivateKeyFilePath	AdditionalBatchCommands	ExecutionTime	DeploymentBaseDirectoryAtRemote	RuntimeLocation	SlurmAccount	Email	DeploymentBaseDirectory	DeployRuntime	Name	DotnetRuntime	MonoDebug	AllowedDatabasesPaths
fk69umer	<null>	lcluster18.hrz.tu-darmstadt.de	/home/administrator/.ssh/id_ed25519	[ #SBATCH -C avx512, #SBATCH --mem-per-cpu=8000 ]	02:00:00	/work/scratch/fk69umer/jenkins/deploy	linux/amd64-openmpi	special00006	<null>	/mnt/lb17-scratch-fk69umer/jenkins/deploy	False	Lb2-specialPrj	dotnet	False	[ /mnt/lb17-scratch-fk69umer/jenkins/databases == /work/scratch/fk69umer/jenkins/databases ]

Utility definitions

static class Utils {
    // DOF per cell in 3D
    public static int Np(int p) {
        return (p*p*p + 6*p*p + 11*p + 6)/6;
    }    
    
    //Non-equidistant nodes
    public static double[] SinLinSpacing(double l, double r, double a, int n) {
        double[] linnodes = GenericBlas.Linspace(-Math.PI * 0.5, Math.PI * 0.5, n);
        double[] linnodes2 = GenericBlas.Linspace(-1, 1, n);
        double[] nodes = new double[n];

        for (int i = 0; i < n; i++)
            //nodes[i] = linnodes2[i] * (1 - a) + (1.0 - Math.Sin(linnodes[i])) * a;
            nodes[i] = linnodes2[i] * (1 - a) + Math.Sin(linnodes[i])*a;

        for (int i = 0; i < n; i++)
            nodes[i] = nodes[i] * (r - l)*0.5 + l;
        return nodes;
    }
}

Init grids and save to database

wmg.Grids

#0: { Guid = 7888a41b-7d45-4c9c-8a50-ec318741e353; Name = LinslvPerfPar-XdgStokes_J16777216; Cell Count = 16777216; Dim = 3 }
#1: { Guid = 63c4d60e-06b8-47b0-96c2-d30120f68ecc; Name = LinslvPerfPar-XdgStokes_J2097152; Cell Count = 2097152; Dim = 3 }
#2: { Guid = a39a884a-ac03-405e-8112-284fc6bd555b; Name = LinslvPerfPar-XdgStokes_J262144; Cell Count = 262144; Dim = 3 }
#3: { Guid = 978a4247-d201-4e33-ba89-285b7f9a031d; Name = LinslvPerfPar-XdgStokes_J110592; Cell Count = 110592; Dim = 3 }
#4: { Guid = b617648c-d4b8-4c33-b30d-a2b7b1efaf6a; Name = LinslvPerfPar-XdgStokes_J13824; Cell Count = 13824; Dim = 3 }
#5: { Guid = 9bcaf4a1-1e21-4442-99ee-ef72b47431be; Name = LinslvPerfPar-XdgStokes_J32768; Cell Count = 32768; Dim = 3 }
#6: { Guid = 38d02bed-9f9e-464f-bb50-060a767db5a6; Name = LinslvPerfPar-XdgStokes_J4096; Cell Count = 4096; Dim = 3 }
#7: { Guid = d197075e-ed8d-44fc-8048-d3f4d8277428; Name = LinslvPerfPar-XdgStokes_J512; Cell Count = 512; Dim = 3 }
#8: { Guid = c5535517-fef8-4869-9452-63e8841711c8; Name = LinslvPerfPar-Stokes3D_J16777216; Cell Count = 16777216; Dim = 3 }
#9: { Guid = 9847b1b8-a5d1-41b3-b68a-337f7d19198d; Name = LinslvPerfPar-Stokes3D_J2097152; Cell Count = 2097152; Dim = 3 }
#10: { Guid = b9655b1d-6bd5-4570-9c4d-c2e2dafcc4ed; Name = LinslvPerfPar-Stokes3D_J262144; Cell Count = 262144; Dim = 3 }
#11: { Guid = 6067306b-dda5-4713-ad2c-9823d75f5df8; Name = LinslvPerfPar-Stokes3D_J110592; Cell Count = 110592; Dim = 3 }
#12: { Guid = dd7383b9-72a7-490e-b499-4c05c4fd346e; Name = LinslvPerfPar-Stokes3D_J32768; Cell Count = 32768; Dim = 3 }
#13: { Guid = 2a3bcf26-f9f2-4eb6-b204-00f058ca8ea8; Name = LinslvPerfPar-Stokes3D_J13824; Cell Count = 13824; Dim = 3 }
#14: { Guid = 3dd357ae-910e-4648-b6c2-f0e0060c47fb; Name = LinslvPerfPar-Stokes3D_J4096; Cell Count = 4096; Dim = 3 }
#15: { Guid = 3857c7e6-b0b6-403e-a05a-d71040221b07; Name = LinslvPerfPar-Stokes3D_J512; Cell Count = 512; Dim = 3 }
#16: { Guid = f99abf8e-46a7-4d42-a71e-524fd2e75c44; Name = LinslvPerfPar-XdgPoisson_J16777216; Cell Count = 16777216; Dim = 3 }
#17: { Guid = 101f7461-0358-4ad7-a9d5-e6547d931a03; Name = LinslvPerfPar-XdgPoisson_J2097152; Cell Count = 2097152; Dim = 3 }
#18: { Guid = bda0e12c-f0aa-4eea-a11d-6eedfbce7105; Name = LinslvPerfPar-XdgPoisson_J262144; Cell Count = 262144; Dim = 3 }
#19: { Guid = a6d8053c-a97b-4f7a-a81d-9103bfb100ef; Name = LinslvPerfPar-XdgPoisson_J32768; Cell Count = 32768; Dim = 3 }
#20: { Guid = 80bc5695-e45a-48ea-94e4-69e633417da7; Name = LinslvPerfPar-XdgPoisson_J4096; Cell Count = 4096; Dim = 3 }
#21: { Guid = 02411c0a-608a-4401-9558-aec48df4abaf; Name = LinslvPerfPar-XdgPoisson_J512; Cell Count = 512; Dim = 3 }
#22: { Guid = 3b8cf5f7-334b-4564-8e29-8e5c8bc4de75; Name = LinslvPerfPar-XdgPoisson_J64; Cell Count = 64; Dim = 3 }
#23: { Guid = 508b1d31-ffc8-4c54-ba8f-263ebbc11273; Name = LinslvPerfPar-SIP_Poisson_J83886080; Cell Count = 83886080; Dim = 3 }
#24: { Guid = aadea803-9139-4351-95f3-48abc28fe690; Name = LinslvPerfPar-SIP_Poisson_J10485760; Cell Count = 10485760; Dim = 3 }
#25: { Guid = cce7a141-c44e-4603-87bf-dc6728c27e0e; Name = LinslvPerfPar-SIP_Poisson_J1310720; Cell Count = 1310720; Dim = 3 }
#26: { Guid = d80d1e9a-b2f8-4fef-b4ba-40ae328058d2; Name = LinslvPerfPar-SIP_Poisson_J552960; Cell Count = 552960; Dim = 3 }
#27: { Guid = 411194fb-0d6a-47ea-ab88-a3bf8af40b50; Name = LinslvPerfPar-SIP_Poisson_J163840; Cell Count = 163840; Dim = 3 }
#28: { Guid = 79e98d27-1c9e-420f-9186-567fc2a6495c; Name = LinslvPerfPar-SIP_Poisson_J69120; Cell Count = 69120; Dim = 3 }
#29: { Guid = 4f114b1d-e684-4a8e-a41f-16b3a360fdcd; Name = LinslvPerfPar-SIP_Poisson_J2560; Cell Count = 2560; Dim = 3 }
#30: { Guid = 13e4b3a5-fb95-4a64-babf-9dfb2204d312; Name = LinslvPerfPar-SIP_Poisson_J20480; Cell Count = 20480; Dim = 3 }
#31: { Guid = 460b2003-5f87-460c-af5e-efb5e97d3ec6; Name = LinslvPerfPar-SIP_Poisson_J320; Cell Count = 320; Dim = 3 }

Create meshes in various resolutions:

• domain $\Omega = (0,10) \times (-1,1) \times (-1,1)$;
• a Dirichlet boundary is set for $x = 0$, everywhere else Neumann boundaries are assumed; these are typically more challenging for a linear solver than Dirichlet boundaries.
• in order to make the problem a bit challenging, a non-equidistant spacing for $y$ and $z$ coordinate are used
• the domain shape is reminiscent of a typical grid for the simulation of channel flows, with exception of the boundary conditions

Since some of the grids are quite large, it is unhandy to create them within this worksheet. Instead, the GridGen app is used to create the grids in parallel on the HPC System.

int[] Resolutions_3D = new int[] { 4, 8, 16, 24, 32, 48, 64, 128, 256 };
string[] GridNameS = new string[Resolutions_3D.Length];
var ggcS = new (GridGenControl C, int MPIsize)[Resolutions_3D.Length];

for(int cnt = 0; cnt < Resolutions_3D.Length; cnt++) {
    int Res = Resolutions_3D[cnt];    
    
    double[] _xNodes = GenericBlas.Linspace(0, 10, Res*5 + 1);
    double[] _yNodes = Utils.SinLinSpacing(-1, +1, 0.6, Res + 1);
    double[] _zNodes = Utils.SinLinSpacing(-1, +1, 0.6, Res + 1);
    int J = (_xNodes.Length - 1)*(_yNodes.Length - 1)*(_zNodes.Length - 1);
    
    string GridName = wmg.CurrentProject + "-SIP_Poisson_J" + J;
    GridNameS[cnt] = GridName;
    
    if(wmg.Grids.Where(grd => grd.Name.Contains(GridName)).Count() <= 0) {
        int NoOfProcs = (int) Math.Min(182, Math.Max(1, Math.Ceiling(J/200000.0)));
        Console.WriteLine("Must create: " + GridName + " with " + NoOfProcs + " processors.");
        
        var C = new GridGenControl();
        ggcS[cnt] = (C, NoOfProcs);
        C.SetDatabase(wmg.DefaultDatabase);
        
        C.GridName = GridName;

        C.GridBlocks = new GridGenControl.MeshBlock[] {
            new GridGenControl.Cartesian3D() {
                xNodes = _xNodes,
                yNodes = _yNodes,
                zNodes = _zNodes
            }
        };

        C.BoundaryRegions.Add((
            new BoundingBox(new double[] { -1e-6, -2, -2 }, new double[] { +1e-6, +2, +2 }), 
            BoundaryType.Dirichlet.ToString()));
        C.BoundaryRegions.Add((
            null, 
            BoundaryType.Neumann.ToString()));
        
        C.SessionName = "GridCreation-" + GridName;
    } else {
        Console.WriteLine("Found grid: " + GridName);
    }
}

Found grid: LinslvPerfPar-SIP_Poisson_J320
Found grid: LinslvPerfPar-SIP_Poisson_J2560
Found grid: LinslvPerfPar-SIP_Poisson_J20480
Found grid: LinslvPerfPar-SIP_Poisson_J69120
Found grid: LinslvPerfPar-SIP_Poisson_J163840
Found grid: LinslvPerfPar-SIP_Poisson_J552960
Found grid: LinslvPerfPar-SIP_Poisson_J1310720
Found grid: LinslvPerfPar-SIP_Poisson_J10485760
Found grid: LinslvPerfPar-SIP_Poisson_J83886080

foreach(var tt in ggcS) {
    if(tt.C != null) {
    Console.WriteLine(" Submitting: " + tt.C.SessionName); 
    var j = tt.C.CreateJob();
    j.RetryCount = 2;
    j.NumberOfMPIProcs = tt.MPIsize;
    j.Activate();
    }
}

wmg.BlockUntilAllJobsTerminate(3600*24*2); // wait at maximum two days for the jobs to finish
wmg.AllJobs

All jobs finished.

IGridInfo[] grids = new IGridInfo[Resolutions_3D.Length];
for(int cnt = 0; cnt < Resolutions_3D.Length; cnt++) {
    int Res = Resolutions_3D[cnt];    
    int J = Res*5*Res*Res;
    
    Console.WriteLine("Searching for grid with " + J + " cells");
    grids[cnt] = wmg.Grids.FirstOrDefault(grd => grd.Name.Contains(GridNameS[cnt])); // grid must be present now
    
    if(grids[cnt] != null) {
        Console.WriteLine("Found Grid: " + grids[cnt]);
        if(grids[cnt].NumberOfCells != J)
            throw new Exception("J mismatch");

        if(grids[cnt].SpatialDimension != 3)
            throw new Exception("D mismatch");
    } else {
        Console.Error.WriteLine("missing: J = " + J);
    }
}

Searching for grid with 320 cells
Found Grid: { Guid = 460b2003-5f87-460c-af5e-efb5e97d3ec6; Name = LinslvPerfPar-SIP_Poisson_J320; Cell Count = 320; Dim = 3 }
Searching for grid with 2560 cells
Found Grid: { Guid = 4f114b1d-e684-4a8e-a41f-16b3a360fdcd; Name = LinslvPerfPar-SIP_Poisson_J2560; Cell Count = 2560; Dim = 3 }
Searching for grid with 20480 cells
Found Grid: { Guid = 13e4b3a5-fb95-4a64-babf-9dfb2204d312; Name = LinslvPerfPar-SIP_Poisson_J20480; Cell Count = 20480; Dim = 3 }
Searching for grid with 69120 cells
Found Grid: { Guid = 79e98d27-1c9e-420f-9186-567fc2a6495c; Name = LinslvPerfPar-SIP_Poisson_J69120; Cell Count = 69120; Dim = 3 }
Searching for grid with 163840 cells
Found Grid: { Guid = 411194fb-0d6a-47ea-ab88-a3bf8af40b50; Name = LinslvPerfPar-SIP_Poisson_J163840; Cell Count = 163840; Dim = 3 }
Searching for grid with 552960 cells
Found Grid: { Guid = d80d1e9a-b2f8-4fef-b4ba-40ae328058d2; Name = LinslvPerfPar-SIP_Poisson_J552960; Cell Count = 552960; Dim = 3 }
Searching for grid with 1310720 cells
Found Grid: { Guid = cce7a141-c44e-4603-87bf-dc6728c27e0e; Name = LinslvPerfPar-SIP_Poisson_J1310720; Cell Count = 1310720; Dim = 3 }
Searching for grid with 10485760 cells
Found Grid: { Guid = aadea803-9139-4351-95f3-48abc28fe690; Name = LinslvPerfPar-SIP_Poisson_J10485760; Cell Count = 10485760; Dim = 3 }
Searching for grid with 83886080 cells
Found Grid: { Guid = 508b1d31-ffc8-4c54-ba8f-263ebbc11273; Name = LinslvPerfPar-SIP_Poisson_J83886080; Cell Count = 83886080; Dim = 3 }

grids

#0: { Guid = 460b2003-5f87-460c-af5e-efb5e97d3ec6; Name = LinslvPerfPar-SIP_Poisson_J320; Cell Count = 320; Dim = 3 }
#1: { Guid = 4f114b1d-e684-4a8e-a41f-16b3a360fdcd; Name = LinslvPerfPar-SIP_Poisson_J2560; Cell Count = 2560; Dim = 3 }
#2: { Guid = 13e4b3a5-fb95-4a64-babf-9dfb2204d312; Name = LinslvPerfPar-SIP_Poisson_J20480; Cell Count = 20480; Dim = 3 }
#3: { Guid = 79e98d27-1c9e-420f-9186-567fc2a6495c; Name = LinslvPerfPar-SIP_Poisson_J69120; Cell Count = 69120; Dim = 3 }
#4: { Guid = 411194fb-0d6a-47ea-ab88-a3bf8af40b50; Name = LinslvPerfPar-SIP_Poisson_J163840; Cell Count = 163840; Dim = 3 }
#5: { Guid = d80d1e9a-b2f8-4fef-b4ba-40ae328058d2; Name = LinslvPerfPar-SIP_Poisson_J552960; Cell Count = 552960; Dim = 3 }
#6: { Guid = cce7a141-c44e-4603-87bf-dc6728c27e0e; Name = LinslvPerfPar-SIP_Poisson_J1310720; Cell Count = 1310720; Dim = 3 }
#7: { Guid = aadea803-9139-4351-95f3-48abc28fe690; Name = LinslvPerfPar-SIP_Poisson_J10485760; Cell Count = 10485760; Dim = 3 }
#8: { Guid = 508b1d31-ffc8-4c54-ba8f-263ebbc11273; Name = LinslvPerfPar-SIP_Poisson_J83886080; Cell Count = 83886080; Dim = 3 }

NUnit.Framework.Assert.IsTrue(grids.All(g => g != null), "Unable to find/create some grid.");

Setup Control Object for a Solver Run

Values/Formulas for the Values of RHS and Boundary Conditions

string nl = System.Environment.NewLine;
string math_expressions_code = 
"static class InitialValues { " +
"    public static double RHS(double[] X) { " + nl +
"        return -Math.Sin(X[0]); " + nl +
"    } " + nl +
"     " + nl +
"    public static double DirichletBC(double[] X) { " + nl +
"        return 0.0; " + nl +
"    } " + nl +
"     " + nl +
"    public static double NeumannBC(double[] X) { " + nl +
"       if(Math.Abs(X[1] - 1.0) < 1.0e-8 || Math.Abs(X[1] + 1.0) < 1.0e-8) " + nl +
"           return 0; " + nl +
"       if(X.Length > 2 && (Math.Abs(X[2] - 1.0) < 1.0e-8 || Math.Abs(X[2] + 1.0) < 1.0e-8)) " + nl +
"           return 0; " + nl +
"     " + nl +
"       return Math.Cos(10.0); " + nl +
"   } " + nl +
"} " + nl;

var RHS_value = new Formula("InitialValues.RHS", false, math_expressions_code);
var NeumannBC_value = new Formula("InitialValues.NeumannBC", false, math_expressions_code);
var DirichletBC_value = new ConstantValue(0.0);

Brief test of the boundary values:

RHS_value.Evaluate(new Vector(0.2, 0.1), 0)

-0.19866933079506122

NeumannBC_value.Evaluate(new Vector(0.2, 0.1), 0)

-0.8390715290764524

Setup of Parameter Study

Polynomial degrees to test:

int[] PolyDegS = new int[] { 2, 3, 5 }; // reduced options for dev.

Number of processors:

int[] MPIsizes = new int[] { 1, 2, 4, 8, 16, 32, 64, 128 };

Loop over all combinations of parameters and define a control object for each combo:

var controls = new List<(SipControl ctrl, int NoOfProcs)>();
LinearSolverCode solver_name = LinearSolverCode.exp_Kcycle_schwarz;
foreach(int k in PolyDegS) {   
foreach(IGridInfo grd in grids) {
foreach(int MPIsize in MPIsizes) {
    if(grd == null)
        continue;

    int Np = Utils.Np(k);
    int J  = grd.NumberOfCells;
    if(J / MPIsize < 16) {
        // less than 16 cells per processor - to low for a multigrid.
        continue; 
    }
    if(J*Np/MPIsize > 500000) {
        // not interested in doing more then 500'000 DOFs per processor
        continue;
    } 
    var ctrl = new SipControl();
    controls.Add((ctrl, MPIsize));
       
    
    string caseName = string.Format("Poisson_J{0}_k{1}_Sz{2}", J, k, MPIsize);
    Console.WriteLine("setting up: " + caseName);
    ctrl.SessionName = "SIP_" + caseName;

    ctrl.SetGrid(grd);
    ctrl.savetodb = true;
    ctrl.SetDGdegree(k);

    ctrl.LinearSolver           = solver_name.GetConfig();
    var isc = ctrl.LinearSolver as IterativeSolverConfig;
    if(isc != null) {
        //Console.WriteLine(isc.ConvergenceCriterion);
        //ctrl.LinearSolver.TargetBlockSize      = Math.Min(J*Np-1,10000);
        isc.ConvergenceCriterion = 1e-8;
    }
        
        
    ctrl.InitialValues.Add("RHS", RHS_value);
    ctrl.AddBoundaryValue(BoundaryType.Dirichlet.ToString(), "T", DirichletBC_value);
    ctrl.AddBoundaryValue(BoundaryType.Neumann.ToString(), "T", NeumannBC_value);
    
    ctrl.TracingNamespaces = "*";
}
}
}

setting up: Poisson_J320_k2_Sz1
setting up: Poisson_J320_k2_Sz2
setting up: Poisson_J320_k2_Sz4
setting up: Poisson_J320_k2_Sz8
setting up: Poisson_J320_k2_Sz16
setting up: Poisson_J2560_k2_Sz1
setting up: Poisson_J2560_k2_Sz2
setting up: Poisson_J2560_k2_Sz4
setting up: Poisson_J2560_k2_Sz8
setting up: Poisson_J2560_k2_Sz16
setting up: Poisson_J2560_k2_Sz32
setting up: Poisson_J2560_k2_Sz64
setting up: Poisson_J2560_k2_Sz128
setting up: Poisson_J20480_k2_Sz1
setting up: Poisson_J20480_k2_Sz2
setting up: Poisson_J20480_k2_Sz4
setting up: Poisson_J20480_k2_Sz8
setting up: Poisson_J20480_k2_Sz16
setting up: Poisson_J20480_k2_Sz32
setting up: Poisson_J20480_k2_Sz64
setting up: Poisson_J20480_k2_Sz128
setting up: Poisson_J69120_k2_Sz2
setting up: Poisson_J69120_k2_Sz4
setting up: Poisson_J69120_k2_Sz8
setting up: Poisson_J69120_k2_Sz16
setting up: Poisson_J69120_k2_Sz32
setting up: Poisson_J69120_k2_Sz64
setting up: Poisson_J69120_k2_Sz128
setting up: Poisson_J163840_k2_Sz4
setting up: Poisson_J163840_k2_Sz8
setting up: Poisson_J163840_k2_Sz16
setting up: Poisson_J163840_k2_Sz32
setting up: Poisson_J163840_k2_Sz64
setting up: Poisson_J163840_k2_Sz128
setting up: Poisson_J552960_k2_Sz16
setting up: Poisson_J552960_k2_Sz32
setting up: Poisson_J552960_k2_Sz64
setting up: Poisson_J552960_k2_Sz128
setting up: Poisson_J1310720_k2_Sz32
setting up: Poisson_J1310720_k2_Sz64
setting up: Poisson_J1310720_k2_Sz128
setting up: Poisson_J320_k3_Sz1
setting up: Poisson_J320_k3_Sz2
setting up: Poisson_J320_k3_Sz4
setting up: Poisson_J320_k3_Sz8
setting up: Poisson_J320_k3_Sz16
setting up: Poisson_J2560_k3_Sz1
setting up: Poisson_J2560_k3_Sz2
setting up: Poisson_J2560_k3_Sz4
setting up: Poisson_J2560_k3_Sz8
setting up: Poisson_J2560_k3_Sz16
setting up: Poisson_J2560_k3_Sz32
setting up: Poisson_J2560_k3_Sz64
setting up: Poisson_J2560_k3_Sz128
setting up: Poisson_J20480_k3_Sz1
setting up: Poisson_J20480_k3_Sz2
setting up: Poisson_J20480_k3_Sz4
setting up: Poisson_J20480_k3_Sz8
setting up: Poisson_J20480_k3_Sz16
setting up: Poisson_J20480_k3_Sz32
setting up: Poisson_J20480_k3_Sz64
setting up: Poisson_J20480_k3_Sz128
setting up: Poisson_J69120_k3_Sz4
setting up: Poisson_J69120_k3_Sz8
setting up: Poisson_J69120_k3_Sz16
setting up: Poisson_J69120_k3_Sz32
setting up: Poisson_J69120_k3_Sz64
setting up: Poisson_J69120_k3_Sz128
setting up: Poisson_J163840_k3_Sz8
setting up: Poisson_J163840_k3_Sz16
setting up: Poisson_J163840_k3_Sz32
setting up: Poisson_J163840_k3_Sz64
setting up: Poisson_J163840_k3_Sz128
setting up: Poisson_J552960_k3_Sz32
setting up: Poisson_J552960_k3_Sz64
setting up: Poisson_J552960_k3_Sz128
setting up: Poisson_J1310720_k3_Sz64
setting up: Poisson_J1310720_k3_Sz128
setting up: Poisson_J320_k5_Sz1
setting up: Poisson_J320_k5_Sz2
setting up: Poisson_J320_k5_Sz4
setting up: Poisson_J320_k5_Sz8
setting up: Poisson_J320_k5_Sz16
setting up: Poisson_J2560_k5_Sz1
setting up: Poisson_J2560_k5_Sz2
setting up: Poisson_J2560_k5_Sz4
setting up: Poisson_J2560_k5_Sz8
setting up: Poisson_J2560_k5_Sz16
setting up: Poisson_J2560_k5_Sz32
setting up: Poisson_J2560_k5_Sz64
setting up: Poisson_J2560_k5_Sz128
setting up: Poisson_J20480_k5_Sz4
setting up: Poisson_J20480_k5_Sz8
setting up: Poisson_J20480_k5_Sz16
setting up: Poisson_J20480_k5_Sz32
setting up: Poisson_J20480_k5_Sz64
setting up: Poisson_J20480_k5_Sz128
setting up: Poisson_J69120_k5_Sz8
setting up: Poisson_J69120_k5_Sz16
setting up: Poisson_J69120_k5_Sz32
setting up: Poisson_J69120_k5_Sz64
setting up: Poisson_J69120_k5_Sz128
setting up: Poisson_J163840_k5_Sz32
setting up: Poisson_J163840_k5_Sz64
setting up: Poisson_J163840_k5_Sz128
setting up: Poisson_J552960_k5_Sz64
setting up: Poisson_J552960_k5_Sz128

//string path = @"C:\Users\flori\Documents\BoSSS-kummer\public\src\L4-application\ipPoisson\bin\Release\net5.0\PoissonBench";
//foreach(var ctrl in controls) {
//    ctrl.savetodb = false;
//    ctrl.SaveToFile(System.IO.Path.Combine(path, ctrl.SessionName + ".obj"));
//}

controls.Count()

107

Launch Jobs

Use the default queue defined on this machine:

ExecutionQueues

index	Username	Password	ServerName	PrivateKeyFilePath	AdditionalBatchCommands	ExecutionTime	DeploymentBaseDirectoryAtRemote	RuntimeLocation	SlurmAccount	Email	DeploymentBaseDirectory	DeployRuntime	Name	DotnetRuntime	MonoDebug	AllowedDatabasesPaths
0	fk69umer	<null>	lcluster18.hrz.tu-darmstadt.de	/home/administrator/.ssh/id_ed25519	[ #SBATCH -C avx512, #SBATCH --mem-per-cpu=8000 ]	02:00:00	/work/scratch/fk69umer/jenkins/deploy	linux/amd64-openmpi	special00006	<null>	/mnt/lb17-scratch-fk69umer/jenkins/deploy	False	Lb2-specialPrj	dotnet	False	[ /mnt/lb17-scratch-fk69umer/jenkins/databases == /work/scratch/fk69umer/jenkins/databases ]
1	fk69umer	<null>	lcluster18.hrz.tu-darmstadt.de	/home/administrator/.ssh/id_ed25519	[ #SBATCH -C avx512, #SBATCH --mem-per-cpu=8000 ]	02:00:00	/work/scratch/fk69umer/jenkins/deploy	linux/amd64-openmpi	special00006	<null>	/mnt/lb17-scratch-fk69umer/jenkins/deploy	False	Lb2-specialPrj	dotnet	False	[ /mnt/lb17-scratch-fk69umer/jenkins/databases == /work/scratch/fk69umer/jenkins/databases ]

var myBatch = GetDefaultQueue();
myBatch

Username	Password	ServerName	PrivateKeyFilePath	AdditionalBatchCommands	ExecutionTime	DeploymentBaseDirectoryAtRemote	RuntimeLocation	SlurmAccount	Email	DeploymentBaseDirectory	DeployRuntime	Name	DotnetRuntime	MonoDebug	AllowedDatabasesPaths
fk69umer	<null>	lcluster18.hrz.tu-darmstadt.de	/home/administrator/.ssh/id_ed25519	[ #SBATCH -C avx512, #SBATCH --mem-per-cpu=8000 ]	02:00:00	/work/scratch/fk69umer/jenkins/deploy	linux/amd64-openmpi	special00006	<null>	/mnt/lb17-scratch-fk69umer/jenkins/deploy	False	Lb2-specialPrj	dotnet	False	[ /mnt/lb17-scratch-fk69umer/jenkins/databases == /work/scratch/fk69umer/jenkins/databases ]

int i = 1;
foreach((var ctrl, int MPIsize) in controls) {
    Console.WriteLine($"   Submitting {i}/{controls.Count()}: " + ctrl.SessionName);
    var j = ctrl.CreateJob();
    j.RetryCount = 1;
    j.NumberOfMPIProcs = MPIsize;
    j.Activate(myBatch);
    i++;
    //ctrl.RunBatch(); // run in the default queue for this project
}

   Submitting 1/107: SIP_Poisson_J320_k2_Sz1
Deployments so far (1): (Job token: 44099893, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094420.374484' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k2_Sz1	11/01/2023 09:44:57	d1f49bc2..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 2/107: SIP_Poisson_J320_k2_Sz2
Deployments so far (1): (Job token: 44099894, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094421.832384' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k2_Sz2	11/01/2023 09:45:05	f28523e8..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 3/107: SIP_Poisson_J320_k2_Sz4
Deployments so far (1): (Job token: 44099895, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094423.289752' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k2_Sz4	11/01/2023 09:45:20	f83b1ec8..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 4/107: SIP_Poisson_J320_k2_Sz8
Deployments so far (1): (Job token: 44099896, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094424.709619' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k2_Sz8	11/01/2023 09:45:30	0df5cdf2..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 5/107: SIP_Poisson_J320_k2_Sz16
Deployments so far (1): (Job token: 44099897, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094426.150627' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k2_Sz16	11/01/2023 09:45:37	162ca768..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 6/107: SIP_Poisson_J2560_k2_Sz1
Deployments so far (1): (Job token: 44099898, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094427.638225' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k2_Sz1	11/01/2023 09:45:46	1c37731b..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 7/107: SIP_Poisson_J2560_k2_Sz2
Deployments so far (1): (Job token: 44099899, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094429.084639' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k2_Sz2	11/01/2023 09:45:51	a3fd910a..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 8/107: SIP_Poisson_J2560_k2_Sz4
Deployments so far (1): (Job token: 44099900, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094430.533351' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k2_Sz4	11/01/2023 09:45:57	8089db0f..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 9/107: SIP_Poisson_J2560_k2_Sz8
Deployments so far (1): (Job token: 44099902, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094432.031075' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k2_Sz8	11/01/2023 09:46:03	4daf6d9b..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 10/107: SIP_Poisson_J2560_k2_Sz16
Deployments so far (1): (Job token: 44099903, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094433.530887' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k2_Sz16	11/01/2023 09:45:23	0a7e2620..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 11/107: SIP_Poisson_J2560_k2_Sz32
Deployments so far (1): (Job token: 44099904, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094435.025515' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k2_Sz32	11/01/2023 09:45:26	bf0b134e..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 12/107: SIP_Poisson_J2560_k2_Sz64
Deployments so far (1): (Job token: 44099905, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094436.549784' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k2_Sz64	11/01/2023 09:45:30	74315237..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 13/107: SIP_Poisson_J2560_k2_Sz128
Deployments so far (1): (Job token: 44099906, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094438.180814' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k2_Sz128	11/01/2023 09:45:36	0977a289..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 14/107: SIP_Poisson_J20480_k2_Sz1
Deployments so far (1): (Job token: 44099907, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094439.721707' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k2_Sz1	11/01/2023 09:46:07	d37ebc80..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 15/107: SIP_Poisson_J20480_k2_Sz2
Deployments so far (1): (Job token: 44099908, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094441.281307' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k2_Sz2	11/01/2023 09:45:25	132a2daf..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 16/107: SIP_Poisson_J20480_k2_Sz4
Deployments so far (1): (Job token: 44099909, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094442.850907' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k2_Sz4	11/01/2023 09:45:29	8bfbf29e..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 17/107: SIP_Poisson_J20480_k2_Sz8
Deployments so far (1): (Job token: 44099910, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094444.448352' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k2_Sz8	11/01/2023 09:45:41	491f514f..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 18/107: SIP_Poisson_J20480_k2_Sz16
Deployments so far (1): (Job token: 44099911, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094446.025157' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k2_Sz16	11/01/2023 09:45:51	65c733e4..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 19/107: SIP_Poisson_J20480_k2_Sz32
Deployments so far (1): (Job token: 44099912, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094447.637648' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k2_Sz32	11/01/2023 09:46:02	ac67bfd4..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 20/107: SIP_Poisson_J20480_k2_Sz64
Deployments so far (1): (Job token: 44099913, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094449.246285' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k2_Sz64	11/01/2023 09:45:36	34cb991e..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 21/107: SIP_Poisson_J20480_k2_Sz128
Deployments so far (1): (Job token: 44099914, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094450.882083' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k2_Sz128	11/01/2023 09:46:28	5057b033..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 22/107: SIP_Poisson_J69120_k2_Sz2
Deployments so far (1): (Job token: 44099915, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094452.503906' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k2_Sz2	11/01/2023 09:45:38	67ae6d0a..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 23/107: SIP_Poisson_J69120_k2_Sz4
Deployments so far (1): (Job token: 44099916, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094454.149903' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k2_Sz4	11/01/2023 09:45:53	e2a3a14d..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 24/107: SIP_Poisson_J69120_k2_Sz8
Deployments so far (1): (Job token: 44099917, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094455.806886' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k2_Sz8	11/01/2023 09:45:59	67987ffa..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 25/107: SIP_Poisson_J69120_k2_Sz16
Deployments so far (1): (Job token: 44099918, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094457.519219' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k2_Sz16	11/01/2023 09:46:10	6d0956c4..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 26/107: SIP_Poisson_J69120_k2_Sz32
Deployments so far (1): (Job token: 44099919, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094459.181620' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k2_Sz32	11/01/2023 09:46:28	d5c94252..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 27/107: SIP_Poisson_J69120_k2_Sz64
Deployments so far (1): (Job token: 44099920, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094500.848850' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k2_Sz64	11/01/2023 09:46:40	d4a0aebb..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 28/107: SIP_Poisson_J69120_k2_Sz128
Deployments so far (1): (Job token: 44099921, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094502.529524' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k2_Sz128	11/01/2023 09:46:42	92e8b58c..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 29/107: SIP_Poisson_J163840_k2_Sz4
Deployments so far (1): (Job token: 44099922, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094504.223521' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k2_Sz4	11/01/2023 09:46:03	408af0eb..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 30/107: SIP_Poisson_J163840_k2_Sz8
Deployments so far (1): (Job token: 44099923, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094505.935397' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k2_Sz8	11/01/2023 09:46:45	c46a55af..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 31/107: SIP_Poisson_J163840_k2_Sz16
Deployments so far (1): (Job token: 44099924, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094507.684899' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k2_Sz16	11/01/2023 09:46:55	317d153d..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 32/107: SIP_Poisson_J163840_k2_Sz32
Deployments so far (1): (Job token: 44099925, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094509.421455' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k2_Sz32	11/01/2023 09:47:10	62c4b2fa..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 33/107: SIP_Poisson_J163840_k2_Sz64
Deployments so far (1): (Job token: 44099926, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094511.201404' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k2_Sz64	11/01/2023 09:47:35	5c380ad0..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 34/107: SIP_Poisson_J163840_k2_Sz128
Deployments so far (1): (Job token: 44099927, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094512.997204' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k2_Sz128	11/01/2023 09:47:55	70724010..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 35/107: SIP_Poisson_J552960_k2_Sz16
Deployments so far (1): (Job token: 44099928, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094514.770662' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J552960_k2_Sz16	11/01/2023 09:47:49	c2abb23f..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 36/107: SIP_Poisson_J552960_k2_Sz32
Deployments so far (1): (Job token: 44099929, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094516.567483' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J552960_k2_Sz32	11/01/2023 09:47:55	56fda482..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 37/107: SIP_Poisson_J552960_k2_Sz64
Deployments so far (1): (Job token: 44099930, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094518.450791' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J552960_k2_Sz64	11/01/2023 09:48:00	50f8df53..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 38/107: SIP_Poisson_J552960_k2_Sz128
Deployments so far (1): (Job token: 44099931, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094520.261461' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J552960_k2_Sz128	11/01/2023 09:48:51	c18f9bab..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 39/107: SIP_Poisson_J1310720_k2_Sz32
Deployments so far (1): (Job token: 44099932, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094522.062608' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 40/107: SIP_Poisson_J1310720_k2_Sz64
Deployments so far (1): (Job token: 44099933, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094523.859460' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J1310720_k2_Sz64	11/01/2023 09:49:05	617eb071..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 41/107: SIP_Poisson_J1310720_k2_Sz128
Deployments so far (1): (Job token: 44099934, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094525.716812' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J1310720_k2_Sz128	11/01/2023 09:49:29	217c2611..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 42/107: SIP_Poisson_J320_k3_Sz1
Deployments so far (1): (Job token: 44099935, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094527.550849' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k3_Sz1	11/01/2023 09:49:16	123fdecb..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 43/107: SIP_Poisson_J320_k3_Sz2
Deployments so far (1): (Job token: 44099936, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094529.419369' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k3_Sz2	11/01/2023 09:49:24	02c3b25f..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 44/107: SIP_Poisson_J320_k3_Sz4
Deployments so far (1): (Job token: 44099937, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094531.304122' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k3_Sz4	11/01/2023 09:49:26	8d91c7e5..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 45/107: SIP_Poisson_J320_k3_Sz8
Deployments so far (1): (Job token: 44099938, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094533.218761' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k3_Sz8	11/01/2023 09:49:51	6899f107..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 46/107: SIP_Poisson_J320_k3_Sz16
Deployments so far (1): (Job token: 44099939, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094535.063399' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k3_Sz16	11/01/2023 09:49:54	acf67912..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 47/107: SIP_Poisson_J2560_k3_Sz1
Deployments so far (1): (Job token: 44099940, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094536.967080' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k3_Sz1	11/01/2023 09:49:58	0a36bfde..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 48/107: SIP_Poisson_J2560_k3_Sz2
Deployments so far (1): (Job token: 44099941, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094539.205522' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k3_Sz2	11/01/2023 09:50:04	ef9717a4..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 49/107: SIP_Poisson_J2560_k3_Sz4
Deployments so far (1): (Job token: 44099942, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094541.114887' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k3_Sz4	11/01/2023 09:50:08	ba1cad85..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 50/107: SIP_Poisson_J2560_k3_Sz8
Deployments so far (1): (Job token: 44099943, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094543.020856' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k3_Sz8	11/01/2023 09:50:17	a2ffb958..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 51/107: SIP_Poisson_J2560_k3_Sz16
Deployments so far (1): (Job token: 44099944, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094544.962056' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k3_Sz16	11/01/2023 09:50:17	2453d858..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 52/107: SIP_Poisson_J2560_k3_Sz32
Deployments so far (1): (Job token: 44099945, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094546.890996' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k3_Sz32	11/01/2023 09:50:02	dd90fc61..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 53/107: SIP_Poisson_J2560_k3_Sz64
Deployments so far (1): (Job token: 44099946, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094548.866166' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k3_Sz64	11/01/2023 09:50:34	cafed5b4..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 54/107: SIP_Poisson_J2560_k3_Sz128
Deployments so far (1): (Job token: 44099947, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094550.817853' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k3_Sz128	11/01/2023 09:52:34	07481d08..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 55/107: SIP_Poisson_J20480_k3_Sz1
Deployments so far (1): (Job token: 44099948, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094552.784903' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 56/107: SIP_Poisson_J20480_k3_Sz2
Deployments so far (1): (Job token: 44099949, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094554.772828' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k3_Sz2	11/01/2023 09:50:17	f9502ce8..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 57/107: SIP_Poisson_J20480_k3_Sz4
Deployments so far (1): (Job token: 44099950, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094556.745183' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k3_Sz4	11/01/2023 09:50:22	c20192aa..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 58/107: SIP_Poisson_J20480_k3_Sz8
Deployments so far (1): (Job token: 44099951, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094558.750700' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k3_Sz8	11/01/2023 09:50:51	7240d57d..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 59/107: SIP_Poisson_J20480_k3_Sz16
Deployments so far (1): (Job token: 44099952, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094600.944697' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k3_Sz16	11/01/2023 09:51:19	1a96593d..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 60/107: SIP_Poisson_J20480_k3_Sz32
Deployments so far (1): (Job token: 44099953, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094602.972949' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k3_Sz32	11/01/2023 09:51:22	3016818b..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 61/107: SIP_Poisson_J20480_k3_Sz64
Deployments so far (1): (Job token: 44099954, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094605.023916' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k3_Sz64	11/01/2023 09:51:27	369dc51e..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 62/107: SIP_Poisson_J20480_k3_Sz128
Deployments so far (1): (Job token: 44099955, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094607.051878' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k3_Sz128	11/01/2023 09:53:12	f42ad816..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 63/107: SIP_Poisson_J69120_k3_Sz4
Deployments so far (1): (Job token: 44099956, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094609.109915' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 64/107: SIP_Poisson_J69120_k3_Sz8
Deployments so far (1): (Job token: 44099957, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094611.198277' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k3_Sz8	11/01/2023 09:52:56	94a10445..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 65/107: SIP_Poisson_J69120_k3_Sz16
Deployments so far (1): (Job token: 44099958, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094613.284520' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k3_Sz16	11/01/2023 09:53:04	195fc2f6..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 66/107: SIP_Poisson_J69120_k3_Sz32
Deployments so far (1): (Job token: 44099959, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094615.378172' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k3_Sz32	11/01/2023 09:53:07	c32e53a0..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 67/107: SIP_Poisson_J69120_k3_Sz64
Deployments so far (1): (Job token: 44099960, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094617.502141' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k3_Sz64	11/01/2023 09:52:57	7e39b11f..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 68/107: SIP_Poisson_J69120_k3_Sz128
Deployments so far (1): (Job token: 44099961, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094619.611021' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k3_Sz128	11/01/2023 09:53:44	271606d0..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 69/107: SIP_Poisson_J163840_k3_Sz8
Deployments so far (1): (Job token: 44099962, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094621.924008' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 70/107: SIP_Poisson_J163840_k3_Sz16
Deployments so far (1): (Job token: 44099963, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094624.047761' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k3_Sz16	11/01/2023 09:53:50	d2ee7d75..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 71/107: SIP_Poisson_J163840_k3_Sz32
Deployments so far (1): (Job token: 44099964, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094626.128947' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k3_Sz32	11/01/2023 09:53:57	fdc051a0..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 72/107: SIP_Poisson_J163840_k3_Sz64
Deployments so far (1): (Job token: 44099965, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094628.278199' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k3_Sz64	11/01/2023 09:54:26	45d185b0..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 73/107: SIP_Poisson_J163840_k3_Sz128
Deployments so far (1): (Job token: 44099966, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094630.402621' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k3_Sz128	11/01/2023 09:54:30	f8871b71..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 74/107: SIP_Poisson_J552960_k3_Sz32
Deployments so far (1): (Job token: 44099967, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094632.558939' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 75/107: SIP_Poisson_J552960_k3_Sz64
Deployments so far (1): (Job token: 44099968, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094634.756607' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J552960_k3_Sz64	11/01/2023 09:54:49	12de89b5..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 76/107: SIP_Poisson_J552960_k3_Sz128
Deployments so far (1): (Job token: 44099969, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094636.971706' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J552960_k3_Sz128	11/01/2023 09:54:48	188b4864..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 77/107: SIP_Poisson_J1310720_k3_Sz64
Deployments so far (1): (Job token: 44099970, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094639.298211' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 78/107: SIP_Poisson_J1310720_k3_Sz128
Deployments so far (1): (Job token: 44099971, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094641.621212' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J1310720_k3_Sz128	11/01/2023 09:55:07	07d20d6a..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 79/107: SIP_Poisson_J320_k5_Sz1
Deployments so far (1): (Job token: 44099972, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094643.827985' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k5_Sz1	11/01/2023 09:50:44	a1c10991..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 80/107: SIP_Poisson_J320_k5_Sz2
Deployments so far (1): (Job token: 44099973, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094646.011662' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k5_Sz2	11/01/2023 09:51:19	41f1484a..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 81/107: SIP_Poisson_J320_k5_Sz4
Deployments so far (1): (Job token: 44099974, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094648.239059' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k5_Sz4	11/01/2023 09:54:46	3b4fa2a3..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 82/107: SIP_Poisson_J320_k5_Sz8
Deployments so far (1): (Job token: 44099975, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094650.515680' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k5_Sz8	11/01/2023 09:54:46	18711712..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 83/107: SIP_Poisson_J320_k5_Sz16
Deployments so far (1): (Job token: 44099976, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094652.783349' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J320_k5_Sz16	11/01/2023 09:54:51	ca323963..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 84/107: SIP_Poisson_J2560_k5_Sz1
Deployments so far (1): (Job token: 44099977, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094655.038781' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 85/107: SIP_Poisson_J2560_k5_Sz2
Deployments so far (1): (Job token: 44099978, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094657.364043' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k5_Sz2	11/01/2023 09:55:01	2611e5d9..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 86/107: SIP_Poisson_J2560_k5_Sz4
Deployments so far (1): (Job token: 44099979, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094659.618743' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k5_Sz4	11/01/2023 09:55:10	188db00b..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 87/107: SIP_Poisson_J2560_k5_Sz8
Deployments so far (1): (Job token: 44099980, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094701.903583' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k5_Sz8	11/01/2023 09:54:54	c2a5644d..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 88/107: SIP_Poisson_J2560_k5_Sz16
Deployments so far (1): (Job token: 44099981, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094704.406482' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k5_Sz16	11/01/2023 09:55:02	ae412227..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 89/107: SIP_Poisson_J2560_k5_Sz32
Deployments so far (1): (Job token: 44099982, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094706.710639' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k5_Sz32	11/01/2023 09:55:08	2fd32284..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 90/107: SIP_Poisson_J2560_k5_Sz64
Deployments so far (1): (Job token: 44099983, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094709.024582' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k5_Sz64	11/01/2023 09:56:16	e4d3c82b..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 91/107: SIP_Poisson_J2560_k5_Sz128
Deployments so far (1): (Job token: 44099984, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094711.331350' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J2560_k5_Sz128	11/01/2023 09:56:28	70535364..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 92/107: SIP_Poisson_J20480_k5_Sz4
Deployments so far (1): (Job token: 44099985, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094713.683200' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 93/107: SIP_Poisson_J20480_k5_Sz8
Deployments so far (1): (Job token: 44099986, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094716.007172' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 94/107: SIP_Poisson_J20480_k5_Sz16
Deployments so far (1): (Job token: 44099987, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094718.348866' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k5_Sz16	11/01/2023 09:56:24	6501757d..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 95/107: SIP_Poisson_J20480_k5_Sz32
Deployments so far (1): (Job token: 44099988, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094720.691909' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k5_Sz32	11/01/2023 09:56:32	4343b9c0..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 96/107: SIP_Poisson_J20480_k5_Sz64
Deployments so far (1): (Job token: 44099989, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094723.051726' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k5_Sz64	11/01/2023 09:56:54	90b6faac..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 97/107: SIP_Poisson_J20480_k5_Sz128
Deployments so far (1): (Job token: 44099990, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094725.515347' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J20480_k5_Sz128	11/01/2023 09:57:41	329f27d5..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 98/107: SIP_Poisson_J69120_k5_Sz8
Deployments so far (1): (Job token: 44099991, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094727.895563' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 99/107: SIP_Poisson_J69120_k5_Sz16
Deployments so far (1): (Job token: 44099992, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094730.270099' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 100/107: SIP_Poisson_J69120_k5_Sz32
Deployments so far (1): (Job token: 44099993, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094732.679681' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 101/107: SIP_Poisson_J69120_k5_Sz64
Deployments so far (1): (Job token: 44099994, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094735.130933' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k5_Sz64	11/01/2023 09:57:51	6a02d733..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 102/107: SIP_Poisson_J69120_k5_Sz128
Deployments so far (1): (Job token: 44099995, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094737.574335' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J69120_k5_Sz128	11/01/2023 09:59:27	11772aac..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 103/107: SIP_Poisson_J163840_k5_Sz32
Deployments so far (1): (Job token: 44099996, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094740.005412' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 104/107: SIP_Poisson_J163840_k5_Sz64
Deployments so far (1): (Job token: 44099997, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094742.444888' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 105/107: SIP_Poisson_J163840_k5_Sz128
Deployments so far (1): (Job token: 44099998, FinishedSuccessful 'LinslvPerfPar-ipPoisson2023Nov01_094744.949413' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FinishedSuccessful);
Success: 1
Info: Found successful session "LinslvPerfPar	SIP_Poisson_J163840_k5_Sz128	11/01/2023 09:59:33	72323e6d..." -- job is marked as successful, no further action.
No submission, because job status is: FinishedSuccessful
   Submitting 106/107: SIP_Poisson_J552960_k5_Sz64
Deployments so far (1): (Job token: 44099999, FailedOrCanceled 'LinslvPerfPar-ipPoisson2023Nov01_094747.445306' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, FailedOrCanceled);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled
   Submitting 107/107: SIP_Poisson_J552960_k5_Sz128
Instance #1 of ssh client fk69umer@lcluster18.hrz.tu-darmstadt.de instantiated.
Deployments so far (1): (Job token: 44100000, Unknown 'LinslvPerfPar-ipPoisson2023Nov01_094749.932193' @ SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006, Unknown);
Success: 0
job submit count: 1
Note: Job has reached its maximum number of attempts to run (1) -- job is marked as fail, no further action.
Hint: you might either remove old deployments or increase the 'RetryCount'.
No submission, because job status is: FailedOrCanceled

//foreach(var j in wmg.AllJobs.Values)
//    j.DeleteOldDeploymentsAndSessions();

Wait for Completion and Check Job Status

wmg.BlockUntilAllJobsTerminate(3600*24*2); // wait at maximum two days for the jobs to finish

All jobs finished.

wmg.AllJobs

#0: SIP_Poisson_J320_k2_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k2_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#1: SIP_Poisson_J320_k2_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k2_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#2: SIP_Poisson_J320_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#3: SIP_Poisson_J320_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#4: SIP_Poisson_J320_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#5: SIP_Poisson_J2560_k2_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k2_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#6: SIP_Poisson_J2560_k2_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k2_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#7: SIP_Poisson_J2560_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#8: SIP_Poisson_J2560_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#9: SIP_Poisson_J2560_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#10: SIP_Poisson_J2560_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#11: SIP_Poisson_J2560_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#12: SIP_Poisson_J2560_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#13: SIP_Poisson_J20480_k2_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k2_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#14: SIP_Poisson_J20480_k2_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k2_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#15: SIP_Poisson_J20480_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#16: SIP_Poisson_J20480_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#17: SIP_Poisson_J20480_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#18: SIP_Poisson_J20480_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#19: SIP_Poisson_J20480_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#20: SIP_Poisson_J20480_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#21: SIP_Poisson_J69120_k2_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k2_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#22: SIP_Poisson_J69120_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#23: SIP_Poisson_J69120_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#24: SIP_Poisson_J69120_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#25: SIP_Poisson_J69120_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#26: SIP_Poisson_J69120_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#27: SIP_Poisson_J69120_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#28: SIP_Poisson_J163840_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k2_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#29: SIP_Poisson_J163840_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k2_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#30: SIP_Poisson_J163840_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#31: SIP_Poisson_J163840_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#32: SIP_Poisson_J163840_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#33: SIP_Poisson_J163840_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#34: SIP_Poisson_J552960_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J552960_k2_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#35: SIP_Poisson_J552960_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J552960_k2_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#36: SIP_Poisson_J552960_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J552960_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#37: SIP_Poisson_J552960_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J552960_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#38: SIP_Poisson_J1310720_k2_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J1310720_k2_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#39: SIP_Poisson_J1310720_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J1310720_k2_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#40: SIP_Poisson_J1310720_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J1310720_k2_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#41: SIP_Poisson_J320_k3_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k3_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#42: SIP_Poisson_J320_k3_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k3_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#43: SIP_Poisson_J320_k3_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k3_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#44: SIP_Poisson_J320_k3_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k3_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#45: SIP_Poisson_J320_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#46: SIP_Poisson_J2560_k3_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k3_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#47: SIP_Poisson_J2560_k3_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k3_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#48: SIP_Poisson_J2560_k3_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k3_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#49: SIP_Poisson_J2560_k3_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k3_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#50: SIP_Poisson_J2560_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#51: SIP_Poisson_J2560_k3_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k3_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#52: SIP_Poisson_J2560_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#53: SIP_Poisson_J2560_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#54: SIP_Poisson_J20480_k3_Sz1: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k3_Sz1: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#55: SIP_Poisson_J20480_k3_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k3_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#56: SIP_Poisson_J20480_k3_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k3_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#57: SIP_Poisson_J20480_k3_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k3_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#58: SIP_Poisson_J20480_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#59: SIP_Poisson_J20480_k3_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k3_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#60: SIP_Poisson_J20480_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#61: SIP_Poisson_J20480_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#62: SIP_Poisson_J69120_k3_Sz4: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k3_Sz4: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#63: SIP_Poisson_J69120_k3_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k3_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#64: SIP_Poisson_J69120_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#65: SIP_Poisson_J69120_k3_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k3_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#66: SIP_Poisson_J69120_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#67: SIP_Poisson_J69120_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#68: SIP_Poisson_J163840_k3_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k3_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#69: SIP_Poisson_J163840_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k3_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#70: SIP_Poisson_J163840_k3_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k3_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#71: SIP_Poisson_J163840_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#72: SIP_Poisson_J163840_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#73: SIP_Poisson_J552960_k3_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J552960_k3_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#74: SIP_Poisson_J552960_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J552960_k3_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#75: SIP_Poisson_J552960_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J552960_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#76: SIP_Poisson_J1310720_k3_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J1310720_k3_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#77: SIP_Poisson_J1310720_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J1310720_k3_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#78: SIP_Poisson_J320_k5_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k5_Sz1: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#79: SIP_Poisson_J320_k5_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k5_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#80: SIP_Poisson_J320_k5_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k5_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#81: SIP_Poisson_J320_k5_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k5_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#82: SIP_Poisson_J320_k5_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J320_k5_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#83: SIP_Poisson_J2560_k5_Sz1: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k5_Sz1: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#84: SIP_Poisson_J2560_k5_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k5_Sz2: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#85: SIP_Poisson_J2560_k5_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k5_Sz4: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#86: SIP_Poisson_J2560_k5_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k5_Sz8: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#87: SIP_Poisson_J2560_k5_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k5_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#88: SIP_Poisson_J2560_k5_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k5_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#89: SIP_Poisson_J2560_k5_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k5_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#90: SIP_Poisson_J2560_k5_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J2560_k5_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#91: SIP_Poisson_J20480_k5_Sz4: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k5_Sz4: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#92: SIP_Poisson_J20480_k5_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k5_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#93: SIP_Poisson_J20480_k5_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k5_Sz16: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#94: SIP_Poisson_J20480_k5_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k5_Sz32: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#95: SIP_Poisson_J20480_k5_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k5_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#96: SIP_Poisson_J20480_k5_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J20480_k5_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#97: SIP_Poisson_J69120_k5_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k5_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#98: SIP_Poisson_J69120_k5_Sz16: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k5_Sz16: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#99: SIP_Poisson_J69120_k5_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k5_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#100: SIP_Poisson_J69120_k5_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k5_Sz64: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#101: SIP_Poisson_J69120_k5_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J69120_k5_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#102: SIP_Poisson_J163840_k5_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k5_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#103: SIP_Poisson_J163840_k5_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k5_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#104: SIP_Poisson_J163840_k5_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J163840_k5_Sz128: FinishedSuccessful (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#105: SIP_Poisson_J552960_k5_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J552960_k5_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#106: SIP_Poisson_J552960_k5_Sz128: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)	SIP_Poisson_J552960_k5_Sz128: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)

int succJob = wmg.AllJobs.Values.Where(job => !job.Name.Contains("GridCreation") && job.Status == JobStatus.FinishedSuccessful).Count();
int succSess = wmg.Sessions.Where(Si => Si.Name.Contains("SIP_Poisson") && (Si.SuccessfulTermination == true)).Count();
(succJob, succSess)

Item1	Item2
91	100

int failJob = wmg.AllJobs.Values.Where(job => !job.Name.Contains("GridCreation") && job.Status != JobStatus.FinishedSuccessful).Count();
int failSess = wmg.Sessions.Where(Si => Si.Name.Contains("SIP_Poisson") && (Si.SuccessfulTermination == false)).Count();
(failJob, failSess)

Item1	Item2
16	16

wmg.AllJobs.Values.First().Name

SIP_Poisson_J320_k2_Sz1

NUnit.Framework.Assert.AreEqual(succJob + failJob, controls.Count);
//NUnit.Framework.Assert.AreEqual(succJob, succSess);
//NUnit.Framework.Assert.AreEqual(failJob, failSess);

wmg.Sessions.Where(sess => sess.Name.StartsWith("SIP_Poisson_J"))

#0: LinslvPerfPar	SIP_Poisson_J163840_k5_Sz128	11/01/2023 09:59:33	72323e6d...
#1: LinslvPerfPar	SIP_Poisson_J552960_k5_Sz128*	11/01/2023 10:02:38	4bfce89a...
#2: LinslvPerfPar	SIP_Poisson_J69120_k5_Sz128	11/01/2023 09:59:27	11772aac...
#3: LinslvPerfPar	SIP_Poisson_J69120_k5_Sz64	11/01/2023 09:57:51	6a02d733...
#4: LinslvPerfPar	SIP_Poisson_J1310720_k3_Sz128	11/01/2023 09:55:07	07d20d6a...
#5: LinslvPerfPar	SIP_Poisson_J552960_k5_Sz64*	11/01/2023 09:59:04	0c81c36b...
#6: LinslvPerfPar	SIP_Poisson_J552960_k3_Sz64	11/01/2023 09:54:49	12de89b5...
#7: LinslvPerfPar	SIP_Poisson_J20480_k5_Sz128	11/01/2023 09:57:41	329f27d5...
#8: LinslvPerfPar	SIP_Poisson_J20480_k5_Sz16	11/01/2023 09:56:24	6501757d...
#9: LinslvPerfPar	SIP_Poisson_J163840_k5_Sz64*	11/01/2023 09:58:01	1dd8f085...
#10: LinslvPerfPar	SIP_Poisson_J163840_k5_Sz32*	11/01/2023 09:57:58	f1d4f590...
#11: LinslvPerfPar	SIP_Poisson_J69120_k5_Sz16*	11/01/2023 09:57:43	9f5a1a2c...
#12: LinslvPerfPar	SIP_Poisson_J20480_k5_Sz64	11/01/2023 09:56:54	90b6faac...
#13: LinslvPerfPar	SIP_Poisson_J20480_k5_Sz32	11/01/2023 09:56:32	4343b9c0...
#14: LinslvPerfPar	SIP_Poisson_J69120_k5_Sz32*	11/01/2023 09:57:25	1371faa5...
#15: LinslvPerfPar	SIP_Poisson_J552960_k3_Sz128	11/01/2023 09:54:48	188b4864...
#16: LinslvPerfPar	SIP_Poisson_J2560_k5_Sz128	11/01/2023 09:56:28	70535364...
#17: LinslvPerfPar	SIP_Poisson_J2560_k5_Sz2	11/01/2023 09:55:01	2611e5d9...
#18: LinslvPerfPar	SIP_Poisson_J2560_k5_Sz64	11/01/2023 09:56:16	e4d3c82b...
#19: LinslvPerfPar	SIP_Poisson_J1310720_k3_Sz64*	11/01/2023 09:54:44	cff15425...
#20: LinslvPerfPar	SIP_Poisson_J163840_k3_Sz16	11/01/2023 09:53:50	d2ee7d75...
#21: LinslvPerfPar	SIP_Poisson_J69120_k5_Sz8*	11/01/2023 09:55:58	66387194...
#22: LinslvPerfPar	SIP_Poisson_J2560_k5_Sz4	11/01/2023 09:55:10	188db00b...
#23: LinslvPerfPar	SIP_Poisson_J20480_k5_Sz4*	11/01/2023 09:55:49	22e99efd...
#24: LinslvPerfPar	SIP_Poisson_J20480_k5_Sz8*	11/01/2023 09:55:50	c9e54b60...
#25: LinslvPerfPar	SIP_Poisson_J163840_k3_Sz64	11/01/2023 09:54:26	45d185b0...
#26: LinslvPerfPar	SIP_Poisson_J163840_k3_Sz32	11/01/2023 09:53:57	fdc051a0...
#27: LinslvPerfPar	SIP_Poisson_J552960_k3_Sz32*	11/01/2023 09:54:40	e87acf77...
#28: LinslvPerfPar	SIP_Poisson_J163840_k3_Sz128	11/01/2023 09:54:30	f8871b71...
#29: LinslvPerfPar	SIP_Poisson_J2560_k5_Sz8	11/01/2023 09:54:54	c2a5644d...
#30: LinslvPerfPar	SIP_Poisson_J2560_k5_Sz32	11/01/2023 09:55:08	2fd32284...
#31: LinslvPerfPar	SIP_Poisson_J2560_k5_Sz16	11/01/2023 09:55:02	ae412227...
#32: LinslvPerfPar	SIP_Poisson_J320_k5_Sz16	11/01/2023 09:54:51	ca323963...
#33: LinslvPerfPar	SIP_Poisson_J320_k5_Sz4	11/01/2023 09:54:46	3b4fa2a3...
#34: LinslvPerfPar	SIP_Poisson_J320_k5_Sz8	11/01/2023 09:54:46	18711712...
#35: LinslvPerfPar	SIP_Poisson_J2560_k5_Sz1*	11/01/2023 09:54:53	5889e2a2...
#36: LinslvPerfPar	SIP_Poisson_J69120_k3_Sz8	11/01/2023 09:52:56	94a10445...
#37: LinslvPerfPar	SIP_Poisson_J69120_k3_Sz128	11/01/2023 09:53:44	271606d0...
#38: LinslvPerfPar	SIP_Poisson_J69120_k3_Sz16	11/01/2023 09:53:04	195fc2f6...
#39: LinslvPerfPar	SIP_Poisson_J163840_k3_Sz8*	11/01/2023 09:53:39	a3bea28a...
#40: LinslvPerfPar	SIP_Poisson_J69120_k3_Sz32	11/01/2023 09:53:07	c32e53a0...
#41: LinslvPerfPar	SIP_Poisson_J20480_k3_Sz128	11/01/2023 09:53:12	f42ad816...
#42: LinslvPerfPar	SIP_Poisson_J69120_k3_Sz64	11/01/2023 09:52:57	7e39b11f...
#43: LinslvPerfPar	SIP_Poisson_J69120_k3_Sz4*	11/01/2023 09:52:49	c411764e...
#44: LinslvPerfPar	SIP_Poisson_J2560_k3_Sz128	11/01/2023 09:52:34	07481d08...
#45: LinslvPerfPar	SIP_Poisson_J1310720_k2_Sz64	11/01/2023 09:49:05	617eb071...
#46: LinslvPerfPar	SIP_Poisson_J20480_k3_Sz2	11/01/2023 09:50:17	f9502ce8...
#47: LinslvPerfPar	SIP_Poisson_J1310720_k2_Sz128	11/01/2023 09:49:29	217c2611...
#48: LinslvPerfPar	SIP_Poisson_J20480_k3_Sz16	11/01/2023 09:51:19	1a96593d...
#49: LinslvPerfPar	SIP_Poisson_J20480_k3_Sz64	11/01/2023 09:51:27	369dc51e...
#50: LinslvPerfPar	SIP_Poisson_J20480_k3_Sz32	11/01/2023 09:51:22	3016818b...
#51: LinslvPerfPar	SIP_Poisson_J20480_k3_Sz8	11/01/2023 09:50:51	7240d57d...
#52: LinslvPerfPar	SIP_Poisson_J320_k5_Sz2	11/01/2023 09:51:19	41f1484a...
#53: LinslvPerfPar	SIP_Poisson_J20480_k3_Sz4	11/01/2023 09:50:22	c20192aa...
#54: LinslvPerfPar	SIP_Poisson_J552960_k2_Sz16	11/01/2023 09:47:49	c2abb23f...
#55: LinslvPerfPar	SIP_Poisson_J320_k5_Sz1	11/01/2023 09:50:44	a1c10991...
#56: LinslvPerfPar	SIP_Poisson_J1310720_k2_Sz32*	11/01/2023 09:48:56	ee195044...
#57: LinslvPerfPar	SIP_Poisson_J2560_k3_Sz64	11/01/2023 09:50:34	cafed5b4...
#58: LinslvPerfPar	SIP_Poisson_J2560_k3_Sz8	11/01/2023 09:50:17	a2ffb958...
#59: LinslvPerfPar	SIP_Poisson_J2560_k3_Sz16	11/01/2023 09:50:17	2453d858...
#60: LinslvPerfPar	SIP_Poisson_J20480_k3_Sz1*	11/01/2023 09:50:13	f874acec...
#61: LinslvPerfPar	SIP_Poisson_J2560_k3_Sz1	11/01/2023 09:49:58	0a36bfde...
#62: LinslvPerfPar	SIP_Poisson_J2560_k3_Sz4	11/01/2023 09:50:08	ba1cad85...
#63: LinslvPerfPar	SIP_Poisson_J2560_k3_Sz2	11/01/2023 09:50:04	ef9717a4...
#64: LinslvPerfPar	SIP_Poisson_J552960_k2_Sz128	11/01/2023 09:48:51	c18f9bab...
#65: LinslvPerfPar	SIP_Poisson_J552960_k2_Sz32	11/01/2023 09:47:55	56fda482...
#66: LinslvPerfPar	SIP_Poisson_J2560_k3_Sz32	11/01/2023 09:50:02	dd90fc61...
#67: LinslvPerfPar	SIP_Poisson_J320_k3_Sz16	11/01/2023 09:49:54	acf67912...
#68: LinslvPerfPar	SIP_Poisson_J320_k3_Sz8	11/01/2023 09:49:51	6899f107...
#69: LinslvPerfPar	SIP_Poisson_J552960_k2_Sz64	11/01/2023 09:48:00	50f8df53...
#70: LinslvPerfPar	SIP_Poisson_J320_k3_Sz4	11/01/2023 09:49:26	8d91c7e5...
#71: LinslvPerfPar	SIP_Poisson_J320_k3_Sz2	11/01/2023 09:49:24	02c3b25f...
#72: LinslvPerfPar	SIP_Poisson_J163840_k2_Sz4	11/01/2023 09:46:03	408af0eb...
#73: LinslvPerfPar	SIP_Poisson_J320_k3_Sz1	11/01/2023 09:49:16	123fdecb...
#74: LinslvPerfPar	SIP_Poisson_J163840_k2_Sz8	11/01/2023 09:46:45	c46a55af...
#75: LinslvPerfPar	SIP_Poisson_J163840_k2_Sz128	11/01/2023 09:47:55	70724010...
#76: LinslvPerfPar	SIP_Poisson_J69120_k2_Sz2	11/01/2023 09:45:38	67ae6d0a...
#77: LinslvPerfPar	SIP_Poisson_J163840_k2_Sz64	11/01/2023 09:47:35	5c380ad0...
#78: LinslvPerfPar	SIP_Poisson_J163840_k2_Sz16	11/01/2023 09:46:55	317d153d...
#79: LinslvPerfPar	SIP_Poisson_J163840_k2_Sz32	11/01/2023 09:47:10	62c4b2fa...
#80: LinslvPerfPar	SIP_Poisson_J20480_k2_Sz1	11/01/2023 09:46:07	d37ebc80...
#81: LinslvPerfPar	SIP_Poisson_J69120_k2_Sz4	11/01/2023 09:45:53	e2a3a14d...
#82: LinslvPerfPar	SIP_Poisson_J69120_k2_Sz64	11/01/2023 09:46:40	d4a0aebb...
#83: LinslvPerfPar	SIP_Poisson_J69120_k2_Sz8	11/01/2023 09:45:59	67987ffa...
#84: LinslvPerfPar	SIP_Poisson_J69120_k2_Sz128	11/01/2023 09:46:42	92e8b58c...
#85: LinslvPerfPar	SIP_Poisson_J69120_k2_Sz32	11/01/2023 09:46:28	d5c94252...
#86: LinslvPerfPar	SIP_Poisson_J69120_k2_Sz16	11/01/2023 09:46:10	6d0956c4...
#87: LinslvPerfPar	SIP_Poisson_J20480_k2_Sz128	11/01/2023 09:46:28	5057b033...
#88: LinslvPerfPar	SIP_Poisson_J20480_k2_Sz2	11/01/2023 09:45:25	132a2daf...
#89: LinslvPerfPar	SIP_Poisson_J20480_k2_Sz32	11/01/2023 09:46:02	ac67bfd4...
#90: LinslvPerfPar	SIP_Poisson_J2560_k2_Sz8	11/01/2023 09:46:03	4daf6d9b...
#91: LinslvPerfPar	SIP_Poisson_J20480_k2_Sz16	11/01/2023 09:45:51	65c733e4...
#92: LinslvPerfPar	SIP_Poisson_J2560_k2_Sz4	11/01/2023 09:45:57	8089db0f...
#93: LinslvPerfPar	SIP_Poisson_J20480_k2_Sz8	11/01/2023 09:45:41	491f514f...
#94: LinslvPerfPar	SIP_Poisson_J20480_k2_Sz4	11/01/2023 09:45:29	8bfbf29e...
#95: LinslvPerfPar	SIP_Poisson_J2560_k2_Sz2	11/01/2023 09:45:51	a3fd910a...
#96: LinslvPerfPar	SIP_Poisson_J2560_k2_Sz1	11/01/2023 09:45:46	1c37731b...
#97: LinslvPerfPar	SIP_Poisson_J20480_k2_Sz64	11/01/2023 09:45:36	34cb991e...
#98: LinslvPerfPar	SIP_Poisson_J2560_k2_Sz128	11/01/2023 09:45:36	0977a289...
#99: LinslvPerfPar	SIP_Poisson_J320_k2_Sz16	11/01/2023 09:45:37	162ca768...
#100: LinslvPerfPar	SIP_Poisson_J320_k2_Sz8	11/01/2023 09:45:30	0df5cdf2...
#101: LinslvPerfPar	SIP_Poisson_J2560_k2_Sz64	11/01/2023 09:45:30	74315237...
#102: LinslvPerfPar	SIP_Poisson_J2560_k2_Sz32	11/01/2023 09:45:26	bf0b134e...
#103: LinslvPerfPar	SIP_Poisson_J2560_k2_Sz16	11/01/2023 09:45:23	0a7e2620...
#104: LinslvPerfPar	SIP_Poisson_J320_k2_Sz4	11/01/2023 09:45:20	f83b1ec8...
#105: LinslvPerfPar	SIP_Poisson_J320_k2_Sz2	11/01/2023 09:45:05	f28523e8...
#106: LinslvPerfPar	SIP_Poisson_J320_k2_Sz1	11/01/2023 09:44:57	d1f49bc2...

//controls.Select(tuple => tuple.ctrl)
//        .Select(ctrl => (ctrl,ctrl.GetJob()))
//        .Select(ttt => (ttt.Item1.SessionName == ttt.Item2.Name, ttt.Item1.SessionName, ttt.Item2.Name))

var NoSuccess = wmg.AllJobs.Values
                        .Where(job => job.Status != JobStatus.FinishedSuccessful)
                        .ToArray();
NoSuccess

#0: SIP_Poisson_J1310720_k2_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#1: SIP_Poisson_J20480_k3_Sz1: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#2: SIP_Poisson_J69120_k3_Sz4: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#3: SIP_Poisson_J163840_k3_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#4: SIP_Poisson_J552960_k3_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#5: SIP_Poisson_J1310720_k3_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#6: SIP_Poisson_J2560_k5_Sz1: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#7: SIP_Poisson_J20480_k5_Sz4: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#8: SIP_Poisson_J20480_k5_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#9: SIP_Poisson_J69120_k5_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#10: SIP_Poisson_J69120_k5_Sz16: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#11: SIP_Poisson_J69120_k5_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#12: SIP_Poisson_J163840_k5_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#13: SIP_Poisson_J163840_k5_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#14: SIP_Poisson_J552960_k5_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)
#15: SIP_Poisson_J552960_k5_Sz128: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006)

foreach(var fail in NoSuccess)
    Console.WriteLine(fail + ":  @" + ((fail.LatestDeployment?.DeploymentDirectory?.FullName) ?? " no deployment directory"));

SIP_Poisson_J1310720_k2_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094522.062608
SIP_Poisson_J20480_k3_Sz1: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094552.784903
SIP_Poisson_J69120_k3_Sz4: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094609.109915
SIP_Poisson_J163840_k3_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094621.924008
SIP_Poisson_J552960_k3_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094632.558939
SIP_Poisson_J1310720_k3_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094639.298211
SIP_Poisson_J2560_k5_Sz1: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094655.038781
SIP_Poisson_J20480_k5_Sz4: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094713.683200
SIP_Poisson_J20480_k5_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094716.007172
SIP_Poisson_J69120_k5_Sz8: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094727.895563
SIP_Poisson_J69120_k5_Sz16: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094730.270099
SIP_Poisson_J69120_k5_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094732.679681
SIP_Poisson_J163840_k5_Sz32: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094740.005412
SIP_Poisson_J163840_k5_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094742.444888
SIP_Poisson_J552960_k5_Sz64: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094747.445306
SIP_Poisson_J552960_k5_Sz128: FailedOrCanceled (SlurmClient Lb2-specialPrj : fk69umer@lcluster18.hrz.tu-darmstadt.de, Slurm account: special00006):  @/mnt/lb17-scratch-fk69umer/jenkins/deploy/LinslvPerfPar-ipPoisson2023Nov01_094749.932193

/*
string PathOffset = @"C:\Users\jenkinsci\Desktop\LinSlvPerfFail-20apr22";
foreach(var fail in NoSuccess) {
    var C = fail.GetControl();
    C.savetodb = false;
    C.SaveToFile(System.IO.Path.Combine(PathOffset, fail.Name + ".obj"));
    
    string Stdout = fail.Stdout;
    System.IO.File.WriteAllText(System.IO.Path.Combine(PathOffset, fail.Name + "-stdout.txt"), Stdout);
    
    string Stderr = fail.Stderr;
    System.IO.File.WriteAllText(System.IO.Path.Combine(PathOffset, fail.Name + "-stderr.txt"), Stderr);
}
*/

var FailedSessions = wmg.Sessions.Where(Si => Si.Name.Contains("SIP_Poisson") && !Si.Name.Contains("GridCreation") &&
                                        (Si.SuccessfulTermination == false
                                        || Convert.ToInt32(Si.KeysAndQueries["Conv"]) == 0));
FailedSessions

#0: LinslvPerfPar	SIP_Poisson_J552960_k5_Sz128*	11/01/2023 10:02:38	4bfce89a...
#1: LinslvPerfPar	SIP_Poisson_J552960_k5_Sz64*	11/01/2023 09:59:04	0c81c36b...
#2: LinslvPerfPar	SIP_Poisson_J163840_k5_Sz64*	11/01/2023 09:58:01	1dd8f085...
#3: LinslvPerfPar	SIP_Poisson_J163840_k5_Sz32*	11/01/2023 09:57:58	f1d4f590...
#4: LinslvPerfPar	SIP_Poisson_J69120_k5_Sz16*	11/01/2023 09:57:43	9f5a1a2c...
#5: LinslvPerfPar	SIP_Poisson_J69120_k5_Sz32*	11/01/2023 09:57:25	1371faa5...
#6: LinslvPerfPar	SIP_Poisson_J1310720_k3_Sz64*	11/01/2023 09:54:44	cff15425...
#7: LinslvPerfPar	SIP_Poisson_J69120_k5_Sz8*	11/01/2023 09:55:58	66387194...
#8: LinslvPerfPar	SIP_Poisson_J20480_k5_Sz4*	11/01/2023 09:55:49	22e99efd...
#9: LinslvPerfPar	SIP_Poisson_J20480_k5_Sz8*	11/01/2023 09:55:50	c9e54b60...
#10: LinslvPerfPar	SIP_Poisson_J552960_k3_Sz32*	11/01/2023 09:54:40	e87acf77...
#11: LinslvPerfPar	SIP_Poisson_J2560_k5_Sz1*	11/01/2023 09:54:53	5889e2a2...
#12: LinslvPerfPar	SIP_Poisson_J163840_k3_Sz8*	11/01/2023 09:53:39	a3bea28a...
#13: LinslvPerfPar	SIP_Poisson_J69120_k3_Sz4*	11/01/2023 09:52:49	c411764e...
#14: LinslvPerfPar	SIP_Poisson_J1310720_k2_Sz32*	11/01/2023 09:48:56	ee195044...
#15: LinslvPerfPar	SIP_Poisson_J20480_k3_Sz1*	11/01/2023 09:50:13	f874acec...

//foreach(var s in FailedSessions)
//    s.Delete(true);

//wmg.AllJobs["SIP_Poisson_J2560_k5_pMultigrid"].AllDeployments

//NUnit.Framework.Assert.Zero(NoSuccess.Count(), "Some Jobs Failed");

//foreach(Job jF in NoSuccess)
//   jF.DeleteOldDeploymentsAndSessions();

//NUnit.Framework.Assert.Zero(FailedSessions.Count(), "Some Sessions did not derminate successfully.");