\[ y(x) \left (-a^2 \cos ^2(x)-\frac {b^2}{(2 a-3)^2}-3 a-b \cos (x)-2\right )+\sin ^2(x) y''(x)=0 \] ✓ Mathematica : cpu = 4.11309 (sec), leaf count = 4128
DSolve[(-2 - 3*a - b^2/(-3 + 2*a)^2 - b*Cos[x] - a^2*Cos[x]^2)*y[x] + Sin[x]^2*Derivative[2][y][x] == 0,y[x],x]
\[\left \{\left \{y(x)\to \frac {c_1 (\cos (x)+1)^{\frac {1}{2} \left (-\frac {8 a^2}{-16 a^2+48 a-36}+\frac {24 a}{-16 a^2+48 a-36}+a-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \left (-8 a+\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}+\sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}\right )+\frac {\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}}{4 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}-\frac {18}{-16 a^2+48 a-36}+1\right )} \, _2F_1\left (-\frac {8 a^2}{-16 a^2+48 a-36}+\frac {24 a}{-16 a^2+48 a-36}+a+\frac {\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}}{4 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}-\frac {18}{-16 a^2+48 a-36},\frac {1}{8} \left (-8 a+\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}+\sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}\right );\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )};\frac {1}{2} (1-\cos (x))\right ) (\cos (x)-1)^{\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{4 \left (-16 a^2+48 a-36\right )}}}{\sqrt [4]{\cos ^2(x)-1}}+\frac {\left (-\frac {1}{2}\right )^{1-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}} c_2 (1-\cos (x))^{1-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}} (\cos (x)+1)^{\frac {1}{2} \left (-\frac {8 a^2}{-16 a^2+48 a-36}+\frac {24 a}{-16 a^2+48 a-36}+a-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \left (-8 a+\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}+\sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}\right )+\frac {\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}}{4 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}-\frac {18}{-16 a^2+48 a-36}+1\right )} \, _2F_1\left (-\frac {8 a^2}{-16 a^2+48 a-36}+\frac {24 a}{-16 a^2+48 a-36}+a-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}+\frac {\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}}{4 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}-\frac {18}{-16 a^2+48 a-36}+1,-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )}+\frac {1}{8} \left (-8 a+\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}+\sqrt {\left (8 a-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right )}{-16 a^2+48 a-36}\right )^2-16 \left (4 a^2-\frac {2 \left (-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72\right ) a}{-16 a^2+48 a-36}+2 b+\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{-16 a^2+48 a-36}-1\right )}\right )+1;2-\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{2 \left (-16 a^2+48 a-36\right )};\frac {1}{2} (1-\cos (x))\right ) (\cos (x)-1)^{\frac {-32 a^2+96 a+\sqrt {\left (32 a^2-96 a+72\right )^2-4 \left (-16 a^2+48 a-36\right ) \left (16 a^4+16 b a^2-88 a^2-48 b a+48 a+4 b^2+36 b+45\right )}-72}{4 \left (-16 a^2+48 a-36\right )}}}{\sqrt [4]{\cos ^2(x)-1}}\right \}\right \}\] ✓ Maple : cpu = 0.515 (sec), leaf count = 549
dsolve(sin(x)^2*diff(diff(y(x),x),x)-(a^2*cos(x)^2+b*cos(x)+b^2/(2*a-3)^2+3*a+2)*y(x),y(x))
\[y \left (x \right ) = \frac {\left (\frac {\cos \left (x \right )}{2}-\frac {1}{2}\right )^{\frac {4 a -6+\sqrt {4 b^{2}+16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}}{8 a -12}} \left (\hypergeom \left (\left [\frac {8 a^{2}+\sqrt {4 b^{2}+16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}-\sqrt {4 b^{2}-16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}-8 a -6}{8 a -12}, \frac {-8 a^{2}+\sqrt {4 b^{2}+16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}-\sqrt {4 b^{2}-16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}+16 a -6}{8 a -12}\right ], \left [\frac {4 a -6-\sqrt {4 b^{2}-16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}}{4 a -6}\right ], \frac {\cos \left (x \right )}{2}+\frac {1}{2}\right ) \left (2 \cos \left (x \right )+2\right )^{\frac {4 a -6-\sqrt {4 b^{2}-16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}}{8 a -12}} c_{1}+\hypergeom \left (\left [\frac {8 a^{2}+\sqrt {4 b^{2}+16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}+\sqrt {4 b^{2}-16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}-8 a -6}{8 a -12}, \frac {-8 a^{2}+\sqrt {4 b^{2}+16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}+\sqrt {4 b^{2}-16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}+16 a -6}{8 a -12}\right ], \left [\frac {4 a -6+\sqrt {4 b^{2}-16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}}{4 a -6}\right ], \frac {\cos \left (x \right )}{2}+\frac {1}{2}\right ) \left (2 \cos \left (x \right )+2\right )^{\frac {4 a -6+\sqrt {4 b^{2}-16 \left (a -\frac {3}{2}\right )^{2} b +16 a^{4}-72 a^{2}+81}}{8 a -12}} c_{2}\right )}{\sqrt {\sin \left (x \right )}}\]