\[ \int \frac {1}{\left (-b x+a^2 x^2\right )^{3/2} \left (a x^2+x \sqrt {-b x+a^2 x^2}\right )^{3/2}} \, dx \]
Optimal antiderivative \[ \frac {2 \sqrt {a^{2} x^{2}-b x}\, \left (1601 a^{6} x^{3}-456 a^{4} b \,x^{2}-200 a^{2} b^{2} x +210 b^{3}\right ) \sqrt {x \left (a x +\sqrt {a^{2} x^{2}-b x}\right )}}{1155 b^{5} x^{4} \left (-a^{2} x +b \right )}+\sqrt {x \left (a x +\sqrt {a^{2} x^{2}-b x}\right )}\, \left (-\frac {4 \left (2533 a^{5} x^{2}+461 a^{3} b x +245 a \,b^{2}\right )}{1155 b^{5} x^{3}}-\frac {6 a^{\frac {11}{2}} \sqrt {-a x +\sqrt {a^{2} x^{2}-b x}}\, \arctan \left (\frac {\sqrt {a}\, \sqrt {-a x +\sqrt {a^{2} x^{2}-b x}}}{\sqrt {b}}\right )}{b^{\frac {11}{2}} x}\right ) \]
command
Integrate[1/((-(b*x) + a^2*x^2)^(3/2)*(a*x^2 + x*Sqrt[-(b*x) + a^2*x^2])^(3/2)),x]
Mathematica 13.1 output
\[ -\frac {2 \sqrt {x \left (a x+\sqrt {x \left (-b+a^2 x\right )}\right )} \left (\sqrt {b} \left (210 b^3+10 a b^2 \left (-20 a x+49 \sqrt {x \left (-b+a^2 x\right )}\right )+2 a^3 b x \left (-228 a x+461 \sqrt {x \left (-b+a^2 x\right )}\right )+a^5 x^2 \left (1601 a x+5066 \sqrt {x \left (-b+a^2 x\right )}\right )\right )+3465 a^{11/2} x^2 \sqrt {x \left (-b+a^2 x\right )} \sqrt {-a x+\sqrt {x \left (-b+a^2 x\right )}} \text {ArcTan}\left (\frac {\sqrt {a} \sqrt {-a x+\sqrt {x \left (-b+a^2 x\right )}}}{\sqrt {b}}\right )\right )}{1155 b^{11/2} x^3 \sqrt {x \left (-b+a^2 x\right )}} \]
Mathematica 12.3 output
\[ \int \frac {1}{\left (-b x+a^2 x^2\right )^{3/2} \left (a x^2+x \sqrt {-b x+a^2 x^2}\right )^{3/2}} \, dx \]________________________________________________________________________________________