\[ b \sqrt {\left (1-y(x)^2\right ) \left (1-a^2 y(x)^2\right )} y'(x)^2+\left (y(x)^2-1\right ) \left (a^2 y(x)^2-1\right ) y''(x)+y(x) \left (-2 a^2 y(x)^2+a^2+1\right ) y'(x)^2=0 \] ✓ Mathematica : cpu = 1.01734 (sec), leaf count = 124
DSolve[y[x]*(1 + a^2 - 2*a^2*y[x]^2)*Derivative[1][y][x]^2 + b*Sqrt[(1 - y[x]^2)*(1 - a^2*y[x]^2)]*Derivative[1][y][x]^2 + (-1 + y[x]^2)*(-1 + a^2*y[x]^2)*Derivative[2][y][x] == 0,y[x],x]
\[\left \{\left \{y(x)\to \text {InverseFunction}\left [\int _1^{\text {$\#$1}}\frac {\exp \left (\frac {b \sqrt {1-K[1]^2} \sqrt {1-a^2 K[1]^2} F\left (\sin ^{-1}(K[1])|a^2\right )}{\sqrt {\left (K[1]^2-1\right ) \left (a^2 K[1]^2-1\right )}}+\frac {1}{2} (-\log (1-K[1])-\log (K[1]+1)-\log (1-a K[1])-\log (a K[1]+1))\right )}{c_1}dK[1]\& \right ][x+c_2]\right \}\right \}\] ✓ Maple : cpu = 0.123 (sec), leaf count = 72
dsolve((y(x)^2-1)*(a^2*y(x)^2-1)*diff(diff(y(x),x),x)+b*((1-y(x)^2)*(1-a^2*y(x)^2))^(1/2)*diff(y(x),x)^2+(1+a^2-2*a^2*y(x)^2)*y(x)*diff(y(x),x)^2=0,y(x))
\[\int _{}^{y \left (x \right )}{\mathrm e}^{\int \frac {-2 \textit {\_b}^{3} a^{2}+\textit {\_b} \,a^{2}+b \sqrt {\left (\textit {\_b}^{2}-1\right ) \left (\textit {\_b}^{2} a^{2}-1\right )}+\textit {\_b}}{\left (\textit {\_b}^{2}-1\right ) \left (\textit {\_b}^{2} a^{2}-1\right )}d \textit {\_b}}d \textit {\_b} -c_{1} x -c_{2} = 0\]