Test Function Definitions

Himmelblau

f[{x_,y_}]=((((x^2)+y-11))^2)+(((x+(y^2)-7))^2)+1.4((((x+3.77931))^2)+(((y+3.283186))^2));
g[{x_,y_}]=D[f[{x,y}],{{x,y}}];

Table[
NMinimize[f[{x,y}],{x,y},Method->method][[1]],
{method,{"NelderMead","DifferentialEvolution","SimulatedAnnealing","RandomSearch"}}
]

SIAM

siam[{x_,y_}]:=Exp[Sin[50 x]]+Sin[60 Exp[y]]+Sin[70 Sin[x]]+Sin[Sin[80 y]]-Sin[10 (x+y)]+1/4 (x^2+y^2);
dsiam[{x_,y_}]=D[siam[{x,y}],{{x,y}}];
Table[
NMinimize[siam[{x,y}],{x,y},Method->method][[1]],
{method,{"NelderMead","DifferentialEvolution","SimulatedAnnealing","RandomSearch"}}
]

Double-SIAM

siam4[{x1_,x2_,x3_,x4_}]:=siam[{x1,x2}]+siam[{x3,x4}];
dsiam4[{x1_,x2_,x3_,x4_}]=D[siam4[{x1,x2,x3,x4}],{{x1,x2,x3,x4}}];
Table[
NMinimize[siam4[{x1,x2,x3,x4}],{x1,x2,x3,x4},Method->method][[1]],
{method,{"NelderMead","DifferentialEvolution","SimulatedAnnealing","RandomSearch"}}
]

Rosenbrock

rosen[{x_,y_}]:=(((1-x))^2)+100(((y-(x^2)))^2);
drosen[{x_,y_}]=D[rosen[{x,y}],{{x,y}}];
Table[
NMinimize[rosen[{x,y}],{x,y},Method->method][[1]],
{method,{"NelderMead","DifferentialEvolution","SimulatedAnnealing","RandomSearch"}}
]

100D Rosenbrock

r100[x_?VectorQ]:=Sum[rosen[{(Subscript[x, [[i]]]),(Subscript[x, [[i+1]]])}],{i,1,50}]
dr100[x_?VectorQ]:=Join@@Table[drosen[{(Subscript[x, [[i]]]),(Subscript[x, [[i+1]]])}],{i,1,50}];

x100=Array[Subscript[x,#]&,{100}];

Table[
NMinimize[r100[x100],x100,Method->method],
{method,{"NelderMead","DifferentialEvolution","SimulatedAnnealing","RandomSearch"}}
]

27D Himmelblau

RandomSPDMat[n_,Dist_: UniformDistribution[{1,10}]]:=Module[{Q=RandomReal[{-1,1},{n,n}],λs=Abs[RandomReal[Dist,n]]},Q=Eigenvectors[Q+Transpose[Q]];
Q=Transpose[Q].DiagonalMatrix[λs].Q;Q+Transpose[Q]]

(*A=RandomSPDMat[25];*)

A={{13.89726190512564`,0.3482649313643883`,-0.6910470007156779`,0.5798210396761876`,0.0023352052981060467`,0.09673499254024745`,-0.6210688171339451`,-0.7176175319552658`,0.6282537681943126`,0.6525197317640975`,-0.3085551389555893`,0.15768710664360733`,-0.4301810261369343`,-0.09245584772456239`,-0.1644127799851987`,0.5067277066348563`,-0.49670589530341813`,-0.119862414257359`,0.20497025837868077`,0.2261290574951787`,-0.4497840203442449`,0.7616705046437113`,-0.6221084362155207`,0.9799943293327735`,-0.3795301831686881`},{0.3482649313643883`,12.899263878189076`,0.11683606922159646`,-1.2551221219763928`,-0.10303702989922733`,0.18929821849789688`,0.536598552436704`,-1.0379172846717184`,0.5937289392348152`,-0.8533322943441363`,1.369260548872386`,-0.06619358751005755`,0.010083214820820863`,0.09608527428067787`,0.9207283473211643`,0.21326320193247236`,-1.0714921160894297`,0.5609755948340734`,0.3737260317577771`,0.7280806936527096`,-0.08053469750134548`,0.6108450437915754`,0.42852879659954213`,-0.8646028391112704`,0.07063020627629735`},{-0.6910470007156779`,0.11683606922159646`,11.21057213591209`,-0.15813064495170537`,0.9421964363088764`,0.18943944389397233`,0.3765390994418923`,-1.5057271640118657`,-0.13552920310063632`,1.2212263319411578`,1.010083457380646`,0.35312192975615864`,-0.9832791960615875`,0.0018971451383281435`,1.9789331354357764`,-0.1565559314599871`,0.82795475364942`,0.27370553150521404`,-0.5758219149273268`,0.841557654150308`,-0.35707701332426767`,0.8131551797035699`,0.3455574401338749`,-0.9439028551995281`,0.179229585125128`},{0.5798210396761876`,-1.2551221219763928`,-0.15813064495170537`,9.53940072959331`,0.1850143649210092`,0.5742218079272654`,0.9120556220957425`,-0.8057841023536798`,0.5265343423199155`,-1.04481687035828`,1.4219787204083445`,1.4875718461027045`,0.7194891867763553`,-0.8804691623283036`,0.9314878385216616`,-1.2829091249685396`,-0.720013819147013`,0.8593806911770043`,1.0458671205953263`,1.5611175521614513`,0.5573691497856466`,1.7261877994179484`,0.3073249685544708`,0.29984285109765296`,0.9313721032926443`},{0.0023352052981060467`,-0.10303702989922733`,0.9421964363088764`,0.1850143649210092`,10.906412086655164`,0.7833113306085026`,0.7680045267845189`,-0.07070741366791825`,-0.07405899474065028`,-0.032847750233845355`,1.1331600858647102`,-1.2291228561607457`,-1.2239499843096375`,-0.015552625854301949`,-2.8224827157930297`,0.4027385869090996`,0.6138506383922778`,-0.2637848974403651`,-0.7824587153908373`,-0.12929244688412245`,0.3850098817385861`,-0.9168496945586802`,-0.7665032825859001`,0.5510197821466766`,-0.6914584317379879`},{0.09673499254024745`,0.18929821849789688`,0.18943944389397233`,0.5742218079272654`,0.7833113306085026`,11.816980203601407`,-0.8856655185360808`,1.7662028752743084`,0.07274038100501841`,1.169047766131691`,-0.8628983424003266`,-0.9625914083380849`,0.7192307236801769`,1.3732069570697307`,0.5409585693805511`,0.7379351046114203`,-0.9062258727085969`,0.06723597298365735`,0.1123748694805697`,-0.7346164145491358`,-1.021184689407603`,-1.1184040560854847`,0.5369756473613102`,-0.4100317576606746`,0.8960632834030966`},{-0.6210688171339451`,0.536598552436704`,0.3765390994418923`,0.9120556220957425`,0.7680045267845189`,-0.8856655185360808`,14.742473548228105`,0.6032651248866887`,0.9120935238353534`,-0.14997916815732082`,-0.5122987234241294`,0.6452549768694071`,-0.9457195351694212`,-0.08211215831858071`,-0.3415586983033469`,-0.5627544716272552`,-0.22686911459923886`,-0.9818560798896419`,-1.3722841354119382`,0.31153864836203715`,1.0671495141910858`,-0.3502245858350409`,0.6481905639354351`,-1.03145123211441`,-0.5852750436753487`},{-0.7176175319552658`,-1.0379172846717184`,-1.5057271640118657`,-0.8057841023536798`,-0.07070741366791825`,1.7662028752743084`,0.6032651248866887`,12.304021101039588`,-0.32106779182841405`,0.18421284574468955`,1.7771089126959811`,-0.26562129000311335`,-1.9715713059227986`,-0.010759020598698238`,0.6179343724895187`,-1.3093010984865456`,0.5085037103867744`,0.25530187523095993`,-0.43368554880203014`,1.3321838062602336`,0.02267580120265008`,1.1912979805290234`,-0.047091691102669544`,0.310112437117536`,0.012670392666358922`},{0.6282537681943126`,0.5937289392348152`,-0.13552920310063632`,0.5265343423199155`,-0.07405899474065028`,0.07274038100501841`,0.9120935238353534`,-0.32106779182841405`,11.56046616391612`,-2.5024901472759535`,0.22666002808822866`,-0.21998747287916487`,-0.7324421178545226`,-0.672705251201227`,0.09684706611938439`,-0.33508942738068503`,0.48552900474186084`,0.38065762830533173`,-0.10885431557391592`,0.10504120391261546`,1.6133417327377937`,-1.8889308710270918`,0.32125877311191686`,0.6646497919814012`,1.0040805512560453`},{0.6525197317640975`,-0.8533322943441363`,1.2212263319411578`,-1.04481687035828`,-0.032847750233845355`,1.169047766131691`,-0.14997916815732082`,0.18421284574468955`,-2.5024901472759535`,11.403990937770402`,0.47234713344353935`,-0.232847400282298`,0.5829956329906145`,0.30044295119241615`,-0.3370258588685918`,0.7195409288089112`,0.7202905961431815`,0.7634358862284198`,0.13087337462956522`,1.0522283434411996`,-1.28573969836082`,0.4888388693630384`,1.0323952686478446`,1.0203910855883311`,2.3241375002191518`},{-0.3085551389555893`,1.369260548872386`,1.010083457380646`,1.4219787204083445`,1.1331600858647102`,-0.8628983424003266`,-0.5122987234241294`,1.7771089126959811`,0.22666002808822866`,0.47234713344353935`,11.581658232460052`,-0.3207732080020295`,0.8425017305350632`,0.7891316576452637`,0.46142022429696333`,0.44717726265893976`,0.9810011247006662`,-0.6482374332615386`,0.5650276857244085`,0.8536587696150216`,-1.8578273441008561`,-0.5649978917698454`,0.8905509457508535`,0.000349730605474996`,0.4952405103896693`},{0.15768710664360733`,-0.06619358751005755`,0.35312192975615864`,1.4875718461027045`,-1.2291228561607457`,-0.9625914083380849`,0.6452549768694071`,-0.26562129000311335`,-0.21998747287916487`,-0.232847400282298`,-0.3207732080020295`,11.393290352702659`,-0.24005709892079863`,-0.0195808303602506`,1.080502665471252`,0.6900933097952088`,-0.7437176526804155`,-0.3579118325906516`,-0.07894380024657444`,1.2716222846649494`,1.189086223093317`,-0.6797833580032627`,1.8295992287562903`,-0.05975357843135015`,0.34106117051724577`},{-0.4301810261369343`,0.010083214820820863`,-0.9832791960615875`,0.7194891867763553`,-1.2239499843096375`,0.7192307236801769`,-0.9457195351694212`,-1.9715713059227986`,-0.7324421178545226`,0.5829956329906145`,0.8425017305350632`,-0.24005709892079863`,11.667020673760163`,0.6645305295390342`,0.35077836801773876`,0.692362874401988`,0.3164287664178375`,-1.7751448678405588`,-0.6305061805439651`,0.8781053795672137`,-0.5058970886781344`,-0.7312488373065615`,0.02614265482799595`,-0.9196047321636129`,-1.6476166012177114`},{-0.09245584772456239`,0.09608527428067787`,0.0018971451383281435`,-0.8804691623283036`,-0.015552625854301949`,1.3732069570697307`,-0.08211215831858071`,-0.010759020598698238`,-0.672705251201227`,0.30044295119241615`,0.7891316576452637`,-0.0195808303602506`,0.6645305295390342`,13.120797544860633`,-0.47788330699179704`,0.1999256388559263`,-0.4669225472685878`,0.7943278900572133`,-0.4090858639416285`,-0.979656551873587`,0.43333672114230515`,-0.05952715424153546`,-0.06393543963791211`,-0.4510043142345952`,-0.03306188152505929`},{-0.1644127799851987`,0.9207283473211643`,1.9789331354357764`,0.9314878385216616`,-2.8224827157930297`,0.5409585693805511`,-0.3415586983033469`,0.6179343724895187`,0.09684706611938439`,-0.3370258588685918`,0.46142022429696333`,1.080502665471252`,0.35077836801773876`,-0.47788330699179704`,9.679671527627129`,0.6032545981373839`,0.3653300695278915`,0.7444126402543172`,0.47499811939685366`,-0.9069123408508362`,-1.2021594159631026`,0.7344941559935054`,-1.3575873259504352`,-0.44600638943511894`,-0.25663394554339175`},{0.5067277066348563`,0.21326320193247236`,-0.1565559314599871`,-1.2829091249685396`,0.4027385869090996`,0.7379351046114203`,-0.5627544716272552`,-1.3093010984865456`,-0.33508942738068503`,0.7195409288089112`,0.44717726265893976`,0.6900933097952088`,0.692362874401988`,0.1999256388559263`,0.6032545981373839`,12.34096699372077`,-0.562014624657286`,-0.008947597104436489`,0.5410246395957865`,-0.3346152431418912`,-0.44011589201264895`,0.00139671112716086`,-0.9132708145490138`,0.03475441618595046`,0.45895262920074276`},{-0.49670589530341813`,-1.0714921160894297`,0.82795475364942`,-0.720013819147013`,0.6138506383922778`,-0.9062258727085969`,-0.22686911459923886`,0.5085037103867744`,0.48552900474186084`,0.7202905961431815`,0.9810011247006662`,-0.7437176526804155`,0.3164287664178375`,-0.4669225472685878`,0.3653300695278915`,-0.562014624657286`,14.11550034418582`,1.2650881639356082`,0.31968867981401533`,-0.502081828621134`,-1.0103552125024642`,1.0126450417594954`,-0.20845956538982416`,-1.2025986656314422`,0.24498799580380057`},{-0.119862414257359`,0.5609755948340734`,0.27370553150521404`,0.8593806911770043`,-0.2637848974403651`,0.06723597298365735`,-0.9818560798896419`,0.25530187523095993`,0.38065762830533173`,0.7634358862284198`,-0.6482374332615386`,-0.3579118325906516`,-1.7751448678405588`,0.7943278900572133`,0.7444126402543172`,-0.008947597104436489`,1.2650881639356082`,12.141409051876234`,-0.39250435762497204`,-0.27680769321906123`,-0.15644739895314796`,-1.2189779163212493`,-1.3249541664403939`,0.16050925704018595`,-1.3064353219134428`},{0.20497025837868077`,0.3737260317577771`,-0.5758219149273268`,1.0458671205953263`,-0.7824587153908373`,0.1123748694805697`,-1.3722841354119382`,-0.43368554880203014`,-0.10885431557391592`,0.13087337462956522`,0.5650276857244085`,-0.07894380024657444`,-0.6305061805439651`,-0.4090858639416285`,0.47499811939685366`,0.5410246395957865`,0.31968867981401533`,-0.39250435762497204`,13.133780098532144`,0.8947033521249208`,-0.10264568978443303`,0.300031773551141`,-0.618275751607956`,-0.6963190452925361`,-1.3443638006058352`},{0.2261290574951787`,0.7280806936527096`,0.841557654150308`,1.5611175521614513`,-0.12929244688412245`,-0.7346164145491358`,0.31153864836203715`,1.3321838062602336`,0.10504120391261546`,1.0522283434411996`,0.8536587696150216`,1.2716222846649494`,0.8781053795672137`,-0.979656551873587`,-0.9069123408508362`,-0.3346152431418912`,-0.502081828621134`,-0.27680769321906123`,0.8947033521249208`,13.455491314951479`,0.9862873436434384`,0.7210964433460432`,-0.03554420703925042`,0.29315517945201497`,-0.19345837334183613`},{-0.4497840203442449`,-0.08053469750134548`,-0.35707701332426767`,0.5573691497856466`,0.3850098817385861`,-1.021184689407603`,1.0671495141910858`,0.02267580120265008`,1.6133417327377937`,-1.28573969836082`,-1.8578273441008561`,1.189086223093317`,-0.5058970886781344`,0.43333672114230515`,-1.2021594159631026`,-0.44011589201264895`,-1.0103552125024642`,-0.15644739895314796`,-0.10264568978443303`,0.9862873436434384`,15.424788637630998`,-0.3691887032218304`,-0.7932863581237442`,-0.4411659146198088`,0.12304174320848649`},{0.7616705046437113`,0.6108450437915754`,0.8131551797035699`,1.7261877994179484`,-0.9168496945586802`,-1.1184040560854847`,-0.3502245858350409`,1.1912979805290234`,-1.8889308710270918`,0.4888388693630384`,-0.5649978917698454`,-0.6797833580032627`,-0.7312488373065615`,-0.05952715424153546`,0.7344941559935054`,0.00139671112716086`,1.0126450417594954`,-1.2189779163212493`,0.300031773551141`,0.7210964433460432`,-0.3691887032218304`,12.386778812701333`,-0.39856755966707297`,-0.28960871390764054`,-0.7320797021131773`},{-0.6221084362155207`,0.42852879659954213`,0.3455574401338749`,0.3073249685544708`,-0.7665032825859001`,0.5369756473613102`,0.6481905639354351`,-0.047091691102669544`,0.32125877311191686`,1.0323952686478446`,0.8905509457508535`,1.8295992287562903`,0.02614265482799595`,-0.06393543963791211`,-1.3575873259504352`,-0.9132708145490138`,-0.20845956538982416`,-1.3249541664403939`,-0.618275751607956`,-0.03554420703925042`,-0.7932863581237442`,-0.39856755966707297`,13.907096380424862`,-0.11262436521888787`,1.2454479071839057`},{0.9799943293327735`,-0.8646028391112704`,-0.9439028551995281`,0.29984285109765296`,0.5510197821466766`,-0.4100317576606746`,-1.03145123211441`,0.310112437117536`,0.6646497919814012`,1.0203910855883311`,0.000349730605474996`,-0.05975357843135015`,-0.9196047321636129`,-0.4510043142345952`,-0.44600638943511894`,0.03475441618595046`,-1.2025986656314422`,0.16050925704018595`,-0.6963190452925361`,0.29315517945201497`,-0.4411659146198088`,-0.28960871390764054`,-0.11262436521888787`,11.621858623034289`,-0.7264020688230566`},{-0.3795301831686881`,0.07063020627629735`,0.179229585125128`,0.9313721032926443`,-0.6914584317379879`,0.8960632834030966`,-0.5852750436753487`,0.012670392666358922`,1.0040805512560453`,2.3241375002191518`,0.4952405103896693`,0.34106117051724577`,-1.6476166012177114`,-0.03306188152505929`,-0.25663394554339175`,0.45895262920074276`,0.24498799580380057`,-1.3064353219134428`,-1.3443638006058352`,-0.19345837334183613`,0.12304174320848649`,-0.7320797021131773`,1.2454479071839057`,-0.7264020688230566`,10.075322429838133`}};

(*b=RandomReal[{0,10},25]*)

b={5.911966321238196`,1.9979483000644271`,5.837155786039809`,4.397637968627144`,4.183296472544624`,3.272630353650563`,5.136528353500047`,9.056516819107127`,3.672789295402792`,4.728136250553598`,2.4827587868061185`,8.754712668701657`,4.502884886622636`,7.847573626633735`,5.350106352920676`,8.332652508965495`,9.686219531122735`,1.2450788780014688`,7.999815422938708`,4.621163770266252`,0.7653400775553432`,8.617970174144343`,5.120185552946732`,2.668975779984808`,3.2198360967144346`};

sol=LinearSolve[A,-b];

bigSol={-3.779310247014282,-3.283185989579471}~Join~sol

Clear[f27,g27]
f27[x_?VectorQ]:=f[(Subscript[x, [[1;;2]]])]+(1/2)(Subscript[x, [[3;;27]]]).A.(Subscript[x, [[3;;27]]])+b.(Subscript[x, [[3;;27]]])
g27[x_?VectorQ]:=g[(Subscript[x, [[1;;2]]])]~Join~(A.(Subscript[x, [[3;;27]]])+b);

sol.A.sol+b.sol
f27[bigSol]

x27=Array[Subscript[x,#]&,{27}];

Table[
NMinimize[f27[x27],x27,Method->method][[1]],
{method,{"NelderMead","DifferentialEvolution","SimulatedAnnealing","RandomSearch"}}
]

PSO Algorithm

Driver

PSO[{f_,g_},{lo_,hi_},{dim_,n_,iters_},P_,ZeroQ_:False]:=
Module[
{S,H,p,x,v,b,i,j,best,swarmBest=Null},

(* Random start, velocity, and initialize personal best*)
S=Table[
x=RandomReal[{lo,hi},dim];
If[ZeroQ,
v=Table[0.0,{dim}];,
v=RandomReal[{-Abs[hi-lo],Abs[hi-lo]},dim];
];
b={x,f[x]};
If[swarmBest==Null,
swarmBest=b,
If[f[x]<swarmBest[[2]],
swarmBest=b;
];
];

{x,v,b},{n}
];

(* Initialize history *)
H = Table[S,{iters}];

Do[
For[p=1,p≤n,p++,
(* Move the marble *)
{S[[p]],swarmBest}=P[S[[p]],{f,g},{lo,hi},dim,swarmBest];
];
H[[i]]=S;,

{i,2,iters}
];
{swarmBest,S,H}
]

Driver No history

PSOFast[{f_,g_},{lo_,hi_},{dim_,n_,iters_},P_,ZeroQ_:False]:=
Module[
{S,H,p,x,v,b,i,j,best,swarmBest=Null},

(* Random start, velocity, and initialize personal best*)
S=Table[
(* Random starting point *)
x=RandomReal[{lo,hi},dim]; 

If[ZeroQ,
v=Table[0.0,{dim}];,
v=RandomReal[{-Abs[hi-lo],Abs[hi-lo]},dim];
];

(* Initialize personal bests *)
b={x,f[x]};

(* Initalize Swarm best *)
If[swarmBest==Null,
swarmBest=b,
If[f[x]<swarmBest[[2]],
swarmBest=b;
];
];

{x,v,b},{n}
];

Do[
For[p=1,p≤n,p++,
(* Move the marble *)
{S[[p]],swarmBest}=P[S[[p]],{f,g},{lo,hi},dim,swarmBest];
];,

{i,2,iters}
];

swarmBest[[2]]
]

Swarm

Swarm[p_,{F_,G_},{lo_,hi_},dim_,swarmBest_]:=
Module[
{
(* Particle position velocity and best *)
x=p[[1]],v=p[[2]],best=p[[3]],
(* global/swarm best *)
g=swarmBest,
ω=.729,
rp,rg,
ϕp=1.49445,ϕg=1.49445
},

(* Pick random weights for personal best and swarm best *)
{rp,rg}=RandomReal[{0,1},2];


v=ω v+ϕp rp(best[[1]]-x)+ϕg rg(g[[1]]-x);
x=x+v;

If[F[x]<best[[2]],best={x,F[x]}];

If[F[x]<g[[2]],g={x,F[x]}];

{{x,v,best},g}
];

Random Swarm

RandomSwarm[p_,{F_,G_},{lo_,hi_},dim_,swarmBest_]:=
Module[
{
(* Particle position velocity and best *)
x=p[[1]],v=p[[2]],best=p[[3]],
(* global/swarm best *)
g=swarmBest,
ω=.729,
rp,rg,
rand,
prob=.05,
ϕp=1.49445,ϕg=1.49445
},

rand=RandomReal[];

If[rand<prob,
v=RandomReal[{-Abs[hi-lo],Abs[hi-lo]},dim];
x+=v;
,
(* Pick random weights for personal best and swarm best *)
{rp,rg}=RandomReal[{0,1},2];


v=ω v+ϕp rp(best[[1]]-x)+ϕg rg(g[[1]]-x);
x=x+v;

];

If[F[x]<best[[2]],best={x,F[x]}];

If[F[x]<g[[2]],g={x,F[x]}];

{{x,v,best},g}
];

Gradient Shotgun

GradientShotgun[p_,{F_,G_},{lo_,hi_},dim_,swarmBest_]:=
Module[
{
x=p[[1]],(* particle's position *)
v=p[[2]],(* particle's velocity *) 
best=p[[3]], (* particles best known position *)
g=swarmBest,(* swarm's best known position *)

gradf,(* gradient direction *)

ω=.5, (* weight on gradient-like velocity *)

u,(* random component added to gradient *)

rp,rg,(* random weights for direction of particle best, swarm best *)
ϕp=.2,ϕg=.8 (* hard weights for particle/swarm best directions *)
},

{rp,rg}=RandomReal[{0,1},2];

(* Compute gradient stuff *)
gradf=-G[x];

u=Sign[gradf]*RandomReal[{0,ω Abs[hi-lo]},dim];


v=u+ ϕp rp(best[[1]]-x)+ϕg rg(g[[1]]-x);
x=x+v;

(* Updat bests *)
If[F[x]<best[[2]],best={x,F[x]}];
If[F[x]<g[[2]],g={x,F[x]}];

{{x,v,best},g}
];

Gradient Shotgun 2

GradientShotgun2[p_,{F_,G_},{lo_,hi_},dim_,swarmBest_]:=
Module[
{
x=p[[1]],(* particle's position *)
v=p[[2]],(* particle's velocity *) 
best=p[[3]], (* particles best known position *)
g=swarmBest,(* swarm's best known position *)
δ,
gradf,(* gradient direction *)
norm, (* gradient norm *)

ω=.2, (* weight on gradient-like velocity *)

ϵ=.2, (* Spread factor *)
L, (* length scaling *)

u,(* random component added to gradient *)

rp,rg,(* random weights for direction of particle best, swarm best *)
ϕp=.2,ϕg=.8 (* hard weights for particle/swarm best directions *)
},

{rp,rg}=RandomReal[{0,1},2];

(* Compute gradient stuff *)
gradf=-G[x];
norm=Norm[gradf];
L=(1/norm)RandomReal[{0,Abs[hi-lo]}];

u=L ω gradf*RandomReal[{1+ϵ,1-ϵ},dim];


v=ω u+ ϕp rp(best[[1]]-x)+ϕg rg(g[[1]]-x);
x=x+v;

(* Updat bests *)
If[F[x]<best[[2]],best={x,F[x]}];
If[F[x]<g[[2]],g={x,F[x]}];

{{x,v,best},g}
];

Gravity Marbles v1

GravityMarbles[p_,{F_,G_},{lo_,hi_},dim_,swarmBest_]:=
Module[
{
x=p[[1]],(* particle's position *)
v=p[[2]],(* particle's velocity *) 
χ,
bestP=p[[3]], (* particles best known stuff *)
bestS=swarmBest,(* swarm's best known stuff *)

xs,bp,bs,f,
k=100.0,(* scaling constant for attraction term *)
g,g2,ng,ng2,(* gradient direction *)
h=.0001 (* time step *)
},
{xs,bs}=bestS;
bp=bestP[[2]];

(* Compute gradient *)
g=G[x];ng=Norm[g];
If[ng>1000,g*=100./ng];

v= h(-g+k(3+(bs-bp))(x-xs));
x=x+v;

(* Updat bests *)
f=F[x];
If[f<bp,bestP={x,f}];
If[f<bs,bestS={x,f}];

{{x,v,bestP},bestS}
];

Gravity Marbles v2

GravityMarbles2[p_,{F_,G_},{lo_,hi_},dim_,swarmBest_]:=
Module[
{
x=p[[1]],(* particle's position *)
v=p[[2]],(* particle's velocity *) 
bestP=p[[3]], (* particles best known stuff *)
bestS=swarmBest,(* swarm's best known stuff *)

xs,bp,bs,f,
d=10.0,(* damping derm*)
k=33.,(* attractive force toward swarm best *)
g,(* gradient direction *)
h=.001 (* time step *)
},
{xs,bs}=bestS;
bp=bestP[[2]];

(* Compute gradient *)
g=G[x];

(* Move particle *)
x=x+h v;
(* Update velocity *)
v+= h(-g-d v+k(xs-x));


(* Updat bests *)
f=F[x];
If[f<bp,bestP={x,f}];
If[f<bs,bestS={x,f}];

{{x,v,bestP},bestS}
];

Testing

4D SIAM Variant

Clear[best,S,H]
{best,S,H}=PSO[{siam4,dsiam4},{-1,1},{4,128,500},RandomSwarm];
best

NMinimize[siam4[{x1,x2,x3,x4}],{x1,x2,x3,x4},Method->Automatic]

-6.6137372

Modified Himmelblau

Clear[best,S,H]
{best,S,H}=PSO[{f,g},{-5,5},{2,16,200},GravityMarbles2];
best

2D Siam

Clear[best,S,H]
{best,S,H}=PSO[{siam,dsiam},{-1,1},{2,64,500},Swarm];
NumberForm[best,20]

Rosenbrock

Clear[best,S,H]
{best,S,H}=PSO[{rosen,drosen},{-3,3},{2,128,100},Swarm];
best

100 Dimensional Rosenbrock

Clear[best,S,H]
{best,S}=PSOFast[{r100,dr100},{-2,2},{100,512,500},GravityMarbles2,True];
best

27 Dimensional (Himmelblau + Quadratic)

Clear[best,S]
{best,S}=PSOFast[{f27,g27},{-5,5},{27,512,500},GravityMarbles2];
{bigSol,f27[bigSol]}
best

Visualization

2D Visualization

vectorize2d[P_,V_]:=
Arrow/@Partition[Riffle[P,P+.1V],2];

Animate[
Show[
plot,
Epilog->{
Red,PointSize[.01],Point[H[[i,All,1]]],
Orange,Arrowheads[Small],
vectorize2d[H[[i,All,1]],H[[i,All,2]]]
}
],
{i,1,200,1},
SynchronousUpdating->False,
AnimationRate->5,
AnimationRunning->False,

Initialization:>(
plot=DensityPlot[
f[{x,y}],{x,-5,5},{y,-5,5},
ColorFunction->"BlueGreenYellow",
PlotPoints->30,
AxesLabel->{x,y},
ImageSize->400
];
)
]

4D Visualization

Animate[
GraphicsRow[{
Show[
plot,
Epilog->{
Red,Point[H[[i,All,1,1;;2]]](*,
Orange,Arrowheads[Small],
vectorize2d[H[[i,All,1,1;;2]],H[[i,All,2,1;;2]]]*)
}
],
Show[
plot,
Epilog->{
Red,Point[H[[i,All,1,3;;4]]](*,
Orange,Arrowheads[Small],
vectorize2d[H[[i,All,1,3;;4]],H[[i,All,2,3;;4]]]*)
}
]},
ImageSize->1000
]
,
{i,1,500,1},
SynchronousUpdating->False,
AnimationRate->10,
AnimationRunning->False,

Initialization:>(
plot=DensityPlot[
siam[{x,y}],{x,-1,1},{y,-1,1},
ColorFunction->"BlueGreenYellow",
PlotPoints->30,
AxesLabel->{x,y},
ImageSize->500
];
)
]

Backup Visualization (NO MOVING PICTURES)

DensityPlot[
f[{x,y}],{x,-5,5},{y,-5,5},
ColorFunction->"BlueGreenYellow",AxesLabel->{x,y},
ImageSize->300,
Epilog->{Pink,Point[H[[#,All,1]]]}
]&/@{1,10,20,60,100,150,175,200}