\[ \left (b^2-a^2\right ) y(x)+2 a \cot (a x) y'(x)+y''(x)=0 \] ✓ Mathematica : cpu = 0.061108 (sec), leaf count = 44
DSolve[(-a^2 + b^2)*y[x] + 2*a*Cot[a*x]*Derivative[1][y][x] + Derivative[2][y][x] == 0,y[x],x]
\[\left \{\left \{y(x)\to c_1 e^{-i b x} \csc (a x)-\frac {i c_2 e^{i b x} \csc (a x)}{2 b}\right \}\right \}\] ✓ Maple : cpu = 0.07 (sec), leaf count = 24
dsolve(diff(diff(y(x),x),x)+2*a*diff(y(x),x)*cot(a*x)+(-a^2+b^2)*y(x)=0,y(x))
\[y \left (x \right ) = \frac {c_{2} \cos \left (b x \right )+c_{1} \sin \left (b x \right )}{\sin \left (a x \right )}\]