\[ y''(x)^2 \left (a^2 y(x)^2-b^2\right )+y'(x)^2 \left (a^2 y'(x)^2-1\right )-2 a^2 y(x) y'(x)^2 y''(x)=0 \] ✓ Mathematica : cpu = 118.439 (sec), leaf count = 451
DSolve[Derivative[1][y][x]^2*(-1 + a^2*Derivative[1][y][x]^2) - 2*a^2*y[x]*Derivative[1][y][x]^2*Derivative[2][y][x] + (-b^2 + a^2*y[x]^2)*Derivative[2][y][x]^2 == 0,y[x],x]
\[\left \{\left \{y(x)\to \frac {b \left (e^{\frac {\sqrt {-1+a^2 c_1{}^2} (x+c_2)}{b}}-c_1\right )}{\sqrt {-1+a^2 c_1{}^2}}\right \},\left \{y(x)\to c_1 e^{c_2 x}-\frac {\sqrt {b^2+\frac {1}{c_2{}^2}}}{a}\right \},\left \{y(x)\to \frac {\exp \left (-\frac {i (x+c_2) \left (4 a^2 \cos \left (\frac {c_1}{b}\right )+4 i a^2 \sin \left (\frac {c_1}{b}\right )+b^4 \cos \left (\frac {c_1}{b}\right )-i b^4 \sin \left (\frac {c_1}{b}\right )\right )}{b \left (-4 a^2 \cos \left (\frac {c_1}{b}\right )-4 i a^2 \sin \left (\frac {c_1}{b}\right )+b^4 \cos \left (\frac {c_1}{b}\right )-i b^4 \sin \left (\frac {c_1}{b}\right )\right )}\right )-4 b^3 \cos \left (\frac {c_1}{b}\right )-4 i b^3 \sin \left (\frac {c_1}{b}\right )}{4 a^2 \cos \left (\frac {2 c_1}{b}\right )+4 i a^2 \sin \left (\frac {2 c_1}{b}\right )+b^4}\right \},\left \{y(x)\to \frac {\exp \left (-\frac {i (x+c_2) \left (4 a^2 \cos \left (\frac {c_1}{b}\right )+4 i a^2 \sin \left (\frac {c_1}{b}\right )+b^4 \cos \left (\frac {c_1}{b}\right )-i b^4 \sin \left (\frac {c_1}{b}\right )\right )}{b \left (-4 a^2 \cos \left (\frac {c_1}{b}\right )-4 i a^2 \sin \left (\frac {c_1}{b}\right )+b^4 \cos \left (\frac {c_1}{b}\right )-i b^4 \sin \left (\frac {c_1}{b}\right )\right )}\right )+4 b^3 \cos \left (\frac {c_1}{b}\right )+4 i b^3 \sin \left (\frac {c_1}{b}\right )}{4 a^2 \cos \left (\frac {2 c_1}{b}\right )+4 i a^2 \sin \left (\frac {2 c_1}{b}\right )+b^4}\right \}\right \}\] ✓ Maple : cpu = 6.695 (sec), leaf count = 162
dsolve((a^2*y(x)^2-b^2)*diff(diff(y(x),x),x)^2-2*a^2*y(x)*diff(y(x),x)^2*diff(diff(y(x),x),x)+(a^2*diff(y(x),x)^2-1)*diff(y(x),x)^2=0,y(x))
\[y \left (x \right ) = \frac {\tan \left (\frac {\sqrt {a^{2}}\, \left (-x +c_{1}\right )}{a b}\right ) b}{\sqrt {\tan ^{2}\left (\frac {\sqrt {a^{2}}\, \left (-x +c_{1}\right )}{a b}\right )+1}\, a}\]