\[ \int \frac {\sin ^2(x)}{a \cos (x)+b \sin (x)} \, dx \]
Optimal antiderivative \[ -\frac {a^{2} \arctanh \left (\frac {b \cos \left (x \right )-a \sin \left (x \right )}{\sqrt {a^{2}+b^{2}}}\right )}{\left (a^{2}+b^{2}\right )^{\frac {3}{2}}}-\frac {b \cos \left (x \right )}{a^{2}+b^{2}}-\frac {a \sin \left (x \right )}{a^{2}+b^{2}} \]
command
integrate(sin(x)**2/(a*cos(x)+b*sin(x)),x)
Sympy 1.10.1 under Python 3.10.4 output
\[ \begin {cases} \tilde {\infty } \cos {\left (x \right )} & \text {for}\: a = 0 \wedge b = 0 \\- \frac {\cos {\left (x \right )}}{b} & \text {for}\: a = 0 \\- \frac {\sin ^{2}{\left (x \right )}}{3 i b \sin {\left (x \right )} + 3 b \cos {\left (x \right )}} - \frac {2 i \sin {\left (x \right )} \cos {\left (x \right )}}{3 i b \sin {\left (x \right )} + 3 b \cos {\left (x \right )}} - \frac {2 \cos ^{2}{\left (x \right )}}{3 i b \sin {\left (x \right )} + 3 b \cos {\left (x \right )}} & \text {for}\: a = - i b \\- \frac {\sin ^{2}{\left (x \right )}}{- 3 i b \sin {\left (x \right )} + 3 b \cos {\left (x \right )}} + \frac {2 i \sin {\left (x \right )} \cos {\left (x \right )}}{- 3 i b \sin {\left (x \right )} + 3 b \cos {\left (x \right )}} - \frac {2 \cos ^{2}{\left (x \right )}}{- 3 i b \sin {\left (x \right )} + 3 b \cos {\left (x \right )}} & \text {for}\: a = i b \\- \frac {a^{2} \log {\left (\tan {\left (\frac {x}{2} \right )} - \frac {b}{a} - \frac {\sqrt {a^{2} + b^{2}}}{a} \right )} \tan ^{2}{\left (\frac {x}{2} \right )}}{a^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + a^{2} \sqrt {a^{2} + b^{2}} + b^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {a^{2} + b^{2}}} - \frac {a^{2} \log {\left (\tan {\left (\frac {x}{2} \right )} - \frac {b}{a} - \frac {\sqrt {a^{2} + b^{2}}}{a} \right )}}{a^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + a^{2} \sqrt {a^{2} + b^{2}} + b^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {a^{2} + b^{2}}} + \frac {a^{2} \log {\left (\tan {\left (\frac {x}{2} \right )} - \frac {b}{a} + \frac {\sqrt {a^{2} + b^{2}}}{a} \right )} \tan ^{2}{\left (\frac {x}{2} \right )}}{a^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + a^{2} \sqrt {a^{2} + b^{2}} + b^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {a^{2} + b^{2}}} + \frac {a^{2} \log {\left (\tan {\left (\frac {x}{2} \right )} - \frac {b}{a} + \frac {\sqrt {a^{2} + b^{2}}}{a} \right )}}{a^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + a^{2} \sqrt {a^{2} + b^{2}} + b^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {a^{2} + b^{2}}} - \frac {2 a \sqrt {a^{2} + b^{2}} \tan {\left (\frac {x}{2} \right )}}{a^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + a^{2} \sqrt {a^{2} + b^{2}} + b^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {a^{2} + b^{2}}} - \frac {2 b \sqrt {a^{2} + b^{2}}}{a^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + a^{2} \sqrt {a^{2} + b^{2}} + b^{2} \sqrt {a^{2} + b^{2}} \tan ^{2}{\left (\frac {x}{2} \right )} + b^{2} \sqrt {a^{2} + b^{2}}} & \text {otherwise} \end {cases} \]
Sympy 1.8 under Python 3.8.8 output
\[ \text {Timed out} \]________________________________________________________________________________________