1 % Parallel processing in Matlab workshop
   2 % Darren Price, CBU, Cambridge 2017
   3 
   4 clear
   5 close all
   6 clc
   7 
   8 % CBU Cluster Example, running independent jobs
   9 
  10 addpath /hpc-software/matlab/cbu/
  11 
  12 S = cbu_scheduler();
  13 S.NumWorkers = 2;
  14 S.SubmitArguments = '-l mem=1GB -l walltime=1:00:00';
  15 
  16 % You should first determine how much memory and how many hours you need to
  17 % run your code.
  18 
  19 %% Job1 will output 1 argument
  20 J = [];
  21 J.task = @(x) rand(x);
  22 J.n_return_values = 1;
  23 J.input_args = {5};
  24 J.depends_on = 0;
  25 
  26 cbu_qsub(J, S)
  27 
  28 % You will find all your job information in the Job folders (example
  29 % below). You might need to wait some time for the files to be saved
  30 pause(20)
  31 IN = load(sprintf('%s/Job1/Task1.in.mat',S.JobStorageLocation));
  32 OUT = load(sprintf('%s/Job1/Task1.out.mat',S.JobStorageLocation));
  33 
  34 S.JobStorageLocation = '/home/dp01/teaching/cbu_parallel_workshop/';
  35 
  36 
  37 %% Job2 will display the result with no output args
  38 
  39 % clear all job folders using a unix command
  40 !rm -Rfv /home/dp01/teaching/cbu_parallel_workshop/Job*
  41 
  42 J = [];
  43 J.task = @(x) disp(rand(x));
  44 J.n_return_values = 0; % important
  45 J.input_args = {5};
  46 J.depends_on = 0;
  47 
  48 ID = cbu_qsub(J, S);
  49 
  50 % Check the diary. This contains output normally sent to the command
  51 % window. This is useful for debugging if something goes wrong.
  52 
  53 % this is how to construct a unix string based on the job id and storage location
  54 % This is the recommended way to create strings for loading and saving files in
  55 % matlab in general see "help sprintf"
  56 [~,t] = unix(sprintf('more %s/Job%d/Task1.diary.txt', S.JobStorageLocation, ID));
  57 disp(t) % Print output to the screen
  58 
  59 
  60 
  61 %% Job3 will use an external function to save variables to specified
  62 % location. You should create a function (using a file) that takes two
  63 % input arguments (one of which being the save path)
  64 
  65 
  66 % NOTE: Here is the code for fun1.m. You need to save this to the same folder as
  67 % cbu_scheduler_example.m
  68 % 
  69 % function [x, y] = fun1(z, subjectid)
  70 % 
  71 %     x = z*2;
  72 %     y = z*3;
  73 %     
  74 % save(sprintf('/imaging/dp01/output_subject_%s.mat',subjectid),'x','y')
  75 
  76 % fun1.m cannot use plotting functions. Instead save the data to disk and
  77 % plot locally with a graphics enabled instance of matlab
  78 
  79 % Note, below we have added a addtitional paths. You can add multiple paths
  80 % in a cell array. You need to add the paths for each element of J. You
  81 % generally only need to add one path at a time.
  82 
  83 J = [];
  84 J(1).task = @fun1;
  85 J(1).AdditionalPaths = {'/home/dp01/teaching/cbu_parallel_workshop' '/path/two' '/path/three'}; 
  86 J(1).n_return_values = 2; % important
  87 J(1).input_args = {10, 'ID1'};
  88 J(1).depends_on = 0;
  89 
  90 J(2).task = @fun1;
  91 J(2).AdditionalPaths = '/home/dp01/teaching/cbu_parallel_workshop';
  92 J(2).n_return_values = 2; % important
  93 J(2).input_args = {10, 'ID2'};
  94 J(2).depends_on = 0;
  95 
  96 J(3).task = @fun1;
  97 J(3).AdditionalPaths = '/home/dp01/teaching/cbu_parallel_workshop';
  98 J(3).n_return_values = 2; % important
  99 J(3).input_args = {10, 'ID3'};
 100 J(3).depends_on = 0;
 101 
 102 ID = cbu_qsub(J, S);
 103 
 104 pause(20)
 105 out = load(sprintf('%s/Job%d/Task1.out.mat', S.JobStorageLocation, ID(1)));
 106 
 107 
 108 %% Create J in a loop with additional paths
 109 
 110 IDs = {'ID1' 'ID2' 'ID3'};
 111 for ii = 1:3
 112     J(ii).task = @fun1;
 113     J(ii).AdditionalPaths = {'/home/dp01/teaching/cbu_parallel_workshop'};
 114     J(ii).n_return_values = 2; % important
 115     J(ii).input_args = {10, IDs{ii}};
 116     J(ii).depends_on = 0;
 117 end
 118 
 119 ID = cbu_qsub(J, S);
 120 
 121 % Load 1 of the job output .mat files
 122 pause(20)
 123 out = load(sprintf('%s/Job%d/Task1.out.mat', S.JobStorageLocation, ID(1)))
 124 
 125 
 126 %% parfor loop (recommended for smaller jobs)
 127 clear 
 128 
 129 % cbupool is a wrapper for parpool with corrected settings
 130 delete(gcp)
 131 cbupool(3)
 132 parfor ii = 1:3
 133   % build magic squares in parallel
 134   q{ii} = magic(ii + 2);
 135 end
 136 
 137 for ii=1:length(q)
 138   % plot each magic square
 139   figure, imagesc(q{ii});
 140 end
 141 
 142 
 143 % Things that won't work
 144 
 145 % 1. Index must be consecutive integers
 146 parfor ii = [1:2:10]
 147     n = f(ii);
 148   % build magic squares in parallel
 149   q{ii} = magic(n + 2);
 150 end
 151 
 152 % Instead, use 
 153 f = 1:2:10;
 154 parfor ii = 1:5
 155     n = f(ii);
 156   % build magic squares in parallel
 157   q{ii} = magic(n + 2);
 158 end
 159 
 160 
 161 % Using an index of an index is BAD, and the error message is quite
 162 % unambiguous. "Error: The variable q in a parfor cannot be classified."
 163 cheatindex = [10:-1:1];
 164 parfor ii = 1:10
 165   % build magic squares in parallel
 166   q{cheatindex(ii)} = magic(ii + 2);
 167   disp(labindex)
 168 end
 169 
 170 
 171 % For most jobs taking a few hours, you can use parfor to run a function
 172 % taht contains your entire analysis code.
 173 IDlist = {'ID1' 'ID2' 'ID3'};
 174 NumericalInput = [1 4 6]; % just some arbitrary numbers
 175 parfor ii = 1:3
 176     [x(ii), y(ii)] = fun1(NumericalInput(ii), IDlist{ii}) % this was also save variables to disk
 177 end
 178 
 179 disp(x)
 180 disp(y)
 181 ls /imaging/dp01/output_subject_*
 182 % results in
 183 % /imaging/dp01/output_subject_ID1.mat  
 184 % /imaging/dp01/output_subject_ID2.mat
 185 % /imaging/dp01/output_subject_ID3.mat
 186 
 187 
 188 %% cbupool with Arguments (example for a very large job)
 189 % You should not request large amounts of resources unless you need them.
 190 % This increases the chances of your job crashing unexpectedly and also
 191 % takes longer to start.
 192 delete(gcp)
 193 P=cbupool(96);
 194 P.ResourceTemplate='-l nodes=^N^,mem=196GB,walltime=96:00:00';
 195 parpool(P) 
 196 
 197 
 198 % For smaller jobs (i.e. 20 subjects taking 2 hours each) use this. 2GB per worker is
 199 % usually enough, but you should calculate this before hand to ensure you
 200 % request enough memory.
 201 
 202 delete(gcp)
 203 P=cbupool(20);
 204 P.ResourceTemplate='-l nodes=^N^,mem=40GB,walltime=4:00:00';
 205 parpool(P) 
 206 
 207 % put your parfor loop here
 208 
 209 delete(gcp) % Important to delete it when you are finished to free up resources for other users
 210 
 211 %% Inspecting gcp
 212 % You can check it's methods
 213 methods(gcp)
 214 
 215 % addAttachedFiles       disp                   listAutoAttachedFiles  parfevalOnAll          
 216 % delete                 display                parfeval               updateAttachedFiles    
 217 
 218 % in order to run any methods of gcp you need to assign it to a variable
 219 
 220 g = gcp;
 221 
 222 g.listAutoAttachedFiles
 223 
 224 
 225 %% Other examples from the intranet page http://intranet.mrc-cbu.cam.ac.uk/computing/cluster-demo
 226 cd /hpc-software/example_scripts/workshop/
 227 edit matlab_scheduler.m
 228 edit matlab_analysis.m