\[ \int \frac {\sqrt {-b+a^2 x^2} \sqrt {a x+\sqrt {-b+a^2 x^2}}}{\sqrt {c+\sqrt {a x+\sqrt {-b+a^2 x^2}}}} \, dx \]
Optimal antiderivative \[ \frac {\left (-2560 a^{3} c^{4} x^{3}-2048 a^{2} c^{6} x^{2}+1920 a b \,c^{4} x +1024 b \,c^{6}-1575 a \,b^{2} x -840 b^{2} c^{2}\right ) \sqrt {c +\sqrt {a x +\sqrt {a^{2} x^{2}-b}}}+\left (2240 a^{3} c^{3} x^{3}+1536 a^{2} c^{5} x^{2}+2048 a \,c^{7} x -11760 a b \,c^{3} x -768 b \,c^{5}+1050 b^{2} c \right ) \sqrt {a x +\sqrt {a^{2} x^{2}-b}}\, \sqrt {c +\sqrt {a x +\sqrt {a^{2} x^{2}-b}}}+\sqrt {a^{2} x^{2}-b}\, \left (\left (-2560 a^{2} c^{4} x^{2}-2048 a \,c^{6} x +640 b \,c^{4}-1575 b^{2}\right ) \sqrt {c +\sqrt {a x +\sqrt {a^{2} x^{2}-b}}}+\left (2240 a^{2} c^{3} x^{2}+1536 a \,c^{5} x +2048 c^{7}-10640 b \,c^{3}\right ) \sqrt {a x +\sqrt {a^{2} x^{2}-b}}\, \sqrt {c +\sqrt {a x +\sqrt {a^{2} x^{2}-b}}}\right )}{5040 a \,c^{3} \left (a x +\sqrt {a^{2} x^{2}-b}\right )^{\frac {3}{2}}}+\frac {5 b^{2} \arctanh \left (\frac {\sqrt {c +\sqrt {a x +\sqrt {a^{2} x^{2}-b}}}}{\sqrt {c}}\right )}{16 a \,c^{\frac {7}{2}}} \]
command
Integrate[(Sqrt[-b + a^2*x^2]*Sqrt[a*x + Sqrt[-b + a^2*x^2]])/Sqrt[c + Sqrt[a*x + Sqrt[-b + a^2*x^2]]],x]
Mathematica 13.1 output
\[ \frac {\frac {\sqrt {c} \sqrt {c+\sqrt {a x+\sqrt {-b+a^2 x^2}}} \left (-105 b^2 \left (8 c^2-10 c \sqrt {a x+\sqrt {-b+a^2 x^2}}+15 \left (a x+\sqrt {-b+a^2 x^2}\right )\right )-16 b c^3 \left (-64 c^3+48 c^2 \sqrt {a x+\sqrt {-b+a^2 x^2}}-40 c \left (3 a x+\sqrt {-b+a^2 x^2}\right )+35 \sqrt {a x+\sqrt {-b+a^2 x^2}} \left (21 a x+19 \sqrt {-b+a^2 x^2}\right )\right )+64 c^3 \left (a x+\sqrt {-b+a^2 x^2}\right ) \left (32 c^4 \sqrt {a x+\sqrt {-b+a^2 x^2}}+8 a c^2 x \left (-4 c+3 \sqrt {a x+\sqrt {-b+a^2 x^2}}\right )+5 a^2 x^2 \left (-8 c+7 \sqrt {a x+\sqrt {-b+a^2 x^2}}\right )\right )\right )}{\left (a x+\sqrt {-b+a^2 x^2}\right )^{3/2}}+1575 b^2 \tanh ^{-1}\left (\frac {\sqrt {c+\sqrt {a x+\sqrt {-b+a^2 x^2}}}}{\sqrt {c}}\right )}{5040 a c^{7/2}} \]
Mathematica 12.3 output
\[ \int \frac {\sqrt {-b+a^2 x^2} \sqrt {a x+\sqrt {-b+a^2 x^2}}}{\sqrt {c+\sqrt {a x+\sqrt {-b+a^2 x^2}}}} \, dx \]________________________________________________________________________________________