\[ \boxed { \left \{ {\frac {{\rm d}^{2}}{{\rm d}{t}^{2}}}x \left ( t \right ) = \left ( 3\, \left ( \cos \left ( at+b \right ) \right ) ^{2}-1 \right ) {c}^{2}x \left ( t \right ) +3/2\,{c}^{2}y \left ( t \right ) \sin \left ( 2\,atb \right ) ,{\frac {{\rm d}^{2}}{{\rm d}{t}^{2}}}y \left ( t \right ) = \left ( 3\, \left ( \sin \left ( at+b \right ) \right ) ^{2}-1 \right ) {c}^{2}y \left ( t \right ) +3/2\,{c}^{2}x \left ( t \right ) \sin \left ( 2\,atb \right ) \right \} } \]
Mathematica: cpu = 0.009501 (sec), leaf count = 86 \[ \text {DSolve}\left [\left \{x''(t)=c^2 x(t) \left (3 \cos ^2(a t+b)-1\right )+\frac {3}{2} c^2 y(t) \sin (2 a b t),y''(t)=\frac {3}{2} c^2 x(t) \sin (2 a b t)+c^2 y(t) \left (3 \sin ^2(a t+b)-1\right )\right \},\{x(t),y(t)\},t\right ] \]
Maple: cpu = 13.666 (sec), leaf count = 1445 \[ \left \{ \left \{ x \left ( t \right ) ={\it DESol} \left ( \left \{ { \frac {{\rm d}^{4}}{{\rm d}{t}^{4}}}{\it \_Y} \left ( t \right ) + \left ( 2\,{\frac {\sin \left ( atb \right ) ab}{\cos \left ( atb \right ) }}-2\,{\frac {ab\cos \left ( atb \right ) }{\sin \left ( atb \right ) }} \right ) {\frac {{\rm d}^{3}}{{\rm d}{t}^{3}}}{\it \_Y} \left ( t \right ) + \left ( 2\,{c}^{2}-3\, \left ( \sin \left ( at \right ) \right ) ^{2}{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{ 2}-3\, \left ( \cos \left ( at \right ) \right ) ^{2}{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}-3\, \left ( \sin \left ( at \right ) \right ) ^{2}{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}-3\, \left ( \cos \left ( at \right ) \right ) ^{2}{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}+2\,{\frac { \left ( \sin \left ( atb \right ) \right ) ^{2}{a}^{2}{b}^{2}}{ \left ( \cos \left ( atb \right ) \right ) ^{2}}}+2\,{\frac { \left ( \cos \left ( atb \right ) \right ) ^{ 2}{a}^{2}{b}^{2}}{ \left ( \sin \left ( atb \right ) \right ) ^{2}}} \right ) {\frac {{\rm d}^{2}}{{\rm d}{t}^{2}}}{\it \_Y} \left ( t \right ) + \left ( 2\,{\frac {\sin \left ( atb \right ) ab{c}^{2}}{\cos \left ( atb \right ) }}-2\,{\frac {ab\cos \left ( atb \right ) {c}^{2}}{ \sin \left ( atb \right ) }}+12\,\sin \left ( at \right ) \cos \left ( at \right ) a{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}-12\, \left ( \sin \left ( at \right ) \right ) ^{2}a{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) +12\, \left ( \cos \left ( at \right ) \right ) ^{2}a{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) -12\, \sin \left ( at \right ) \cos \left ( at \right ) a{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}-6\,{\frac {\sin \left ( atb \right ) \left ( \cos \left ( at \right ) \right ) ^{2}ab{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}}{\cos \left ( atb \right ) }}+6\,{\frac {\cos \left ( atb \right ) \left ( \cos \left ( at \right ) \right ) ^{2}a b{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}}{\sin \left ( atb \right ) }}-6\,{\frac {\sin \left ( atb \right ) \left ( \sin \left ( at \right ) \right ) ^{2}ab{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}}{\cos \left ( atb \right ) }}+6\,{\frac {\cos \left ( atb \right ) \left ( \sin \left ( at \right ) \right ) ^{2}ab{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}}{\sin \left ( atb \right ) }}+12\,{ \frac {\sin \left ( atb \right ) \sin \left ( at \right ) \cos \left ( at \right ) ab{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) }{\cos \left ( atb \right ) }}-12\,{\frac {\cos \left ( atb \right ) \sin \left ( at \right ) \cos \left ( at \right ) ab{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) }{\sin \left ( atb \right ) }} \right ) { \frac {\rm d}{{\rm d}t}}{\it \_Y} \left ( t \right ) + \left ( {c}^{4}-6 \, \left ( \sin \left ( at \right ) \right ) ^{2}{a}^{2}{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}+6\, \left ( \cos \left ( at \right ) \right ) ^{2}{a}^{2}{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2} +6\, \left ( \sin \left ( at \right ) \right ) ^{2}{a}^{2}{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}-6\, \left ( \cos \left ( at \right ) \right ) ^{2}{a}^{2}{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2} +9\, \left ( \sin \left ( at \right ) \right ) ^{2} \left ( \cos \left ( at \right ) \right ) ^{2}{c}^{4} \left ( \cos \left ( b \right ) \right ) ^{ 4}+9\, \left ( \sin \left ( at \right ) \right ) ^{4}{c}^{4} \left ( \cos \left ( b \right ) \right ) ^{2} \left ( \sin \left ( b \right ) \right ) ^{2}+9\, \left ( \cos \left ( at \right ) \right ) ^{4}{c}^{4} \left ( \cos \left ( b \right ) \right ) ^{2} \left ( \sin \left ( b \right ) \right ) ^{2}+9\, \left ( \sin \left ( at \right ) \right ) ^{2} \left ( \cos \left ( at \right ) \right ) ^{2}{c}^{4} \left ( \sin \left ( b \right ) \right ) ^{4}+12\,{\frac {\cos \left ( atb \right ) \left ( \sin \left ( at \right ) \right ) ^{2}{a}^{2}b{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) }{\sin \left ( atb \right ) }}-12\,{ \frac {\cos \left ( atb \right ) \left ( \cos \left ( at \right ) \right ) ^{2}{a}^{2}b{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) }{\sin \left ( atb \right ) }}-12\,{\frac {\sin \left ( atb \right ) \sin \left ( at \right ) \cos \left ( at \right ) {a}^{2}b{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}}{\cos \left ( atb \right ) } }+12\,{\frac {\cos \left ( atb \right ) \sin \left ( at \right ) \cos \left ( at \right ) {a}^{2}b{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}}{\sin \left ( atb \right ) }}+12\,{\frac {\sin \left ( atb \right ) \sin \left ( at \right ) \cos \left ( at \right ) {a}^{2}b{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}}{\cos \left ( atb \right ) } }-12\,{\frac {\cos \left ( atb \right ) \sin \left ( at \right ) \cos \left ( at \right ) {a}^{2}b{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}}{\sin \left ( atb \right ) }}-12\,{\frac {\sin \left ( atb \right ) \left ( \sin \left ( at \right ) \right ) ^{2}{a}^{2}b{c}^{2} \cos \left ( b \right ) \sin \left ( b \right ) }{\cos \left ( atb \right ) }}+12\,{\frac {\sin \left ( atb \right ) \left ( \cos \left ( at \right ) \right ) ^{2}{a}^{2}b{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) }{\cos \left ( atb \right ) }}+2\,{\frac { \left ( \sin \left ( a tb \right ) \right ) ^{2}{a}^{2}{b}^{2}{c}^{2}}{ \left ( \cos \left ( atb \right ) \right ) ^{2}}}+2\,{\frac { \left ( \cos \left ( atb \right ) \right ) ^{2}{a}^{2}{b}^{2}{c}^{2}}{ \left ( \sin \left ( atb \right ) \right ) ^{2}}}+18\, \left ( \sin \left ( at \right ) \right ) ^{3}\cos \left ( at \right ) {c}^{4}\cos \left ( b \right ) \left ( \sin \left ( b \right ) \right ) ^{3}-18\,\sin \left ( at \right ) \left ( \cos \left ( at \right ) \right ) ^{3}{c}^{4}\cos \left ( b \right ) \left ( \sin \left ( b \right ) \right ) ^{3}-18\, \left ( \sin \left ( at \right ) \right ) ^{3}\cos \left ( at \right ) {c}^{4} \left ( \cos \left ( b \right ) \right ) ^{3}\sin \left ( b \right ) +18\,\sin \left ( at \right ) \left ( \cos \left ( at \right ) \right ) ^{3}{c}^{4} \left ( \cos \left ( b \right ) \right ) ^{3}\sin \left ( b \right ) -36\, \left ( \sin \left ( at \right ) \right ) ^{2} \left ( \cos \left ( at \right ) \right ) ^{2}{c}^{4} \left ( \cos \left ( b \right ) \right ) ^{2} \left ( \sin \left ( b \right ) \right ) ^{2}-6\,{\frac { \left ( \sin \left ( atb \right ) \right ) ^{2} \left ( \cos \left ( at \right ) \right ) ^{2}{a}^{2}{b}^{2}{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}}{ \left ( \cos \left ( atb \right ) \right ) ^{2}}}-6\,{ \frac { \left ( \cos \left ( atb \right ) \right ) ^{2} \left ( \cos \left ( at \right ) \right ) ^{2}{a}^{2}{b}^{2}{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}}{ \left ( \sin \left ( atb \right ) \right ) ^{2}}}-6\,{\frac { \left ( \sin \left ( atb \right ) \right ) ^{ 2} \left ( \sin \left ( at \right ) \right ) ^{2}{a}^{2}{b}^{2}{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}}{ \left ( \cos \left ( atb \right ) \right ) ^{2}}}-6\,{\frac { \left ( \cos \left ( atb \right ) \right ) ^{2} \left ( \sin \left ( at \right ) \right ) ^{2}{a}^{2}{b}^{2 }{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}}{ \left ( \sin \left ( atb \right ) \right ) ^{2}}}-3\, \left ( \cos \left ( at \right ) \right ) ^{2}{c}^{4} \left ( \sin \left ( b \right ) \right ) ^{2}-3\, \left ( \sin \left ( at \right ) \right ) ^{2}{c}^{4} \left ( \cos \left ( b \right ) \right ) ^{2}-3\, \left ( \cos \left ( at \right ) \right ) ^{2}{c}^{4} \left ( \cos \left ( b \right ) \right ) ^{2}-3\, \left ( \sin \left ( at \right ) \right ) ^{2}{c}^{4} \left ( \sin \left ( b \right ) \right ) ^{2}-9\, \left ( \sin \left ( atb \right ) \right ) ^{2} \left ( \cos \left ( atb \right ) \right ) ^{2}{c}^{4}-24\, \sin \left ( at \right ) \cos \left ( at \right ) {a}^{2}{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) +12\,{\frac { \left ( \sin \left ( atb \right ) \right ) ^{2}\sin \left ( at \right ) \cos \left ( at \right ) {a}^{2}{b}^{2}{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) }{ \left ( \cos \left ( atb \right ) \right ) ^{2}}}+12\,{\frac { \left ( \cos \left ( atb \right ) \right ) ^{2}\sin \left ( at \right ) \cos \left ( at \right ) {a}^{2}{b}^{2}{c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) }{ \left ( \sin \left ( atb \right ) \right ) ^{2}}} \right ) {\it \_Y} \left ( t \right ) \right \} , \left \{ {\it \_Y} \left ( t \right ) \right \} \right ) ,y \left ( t \right ) ={\frac {-3\, x \left ( t \right ) \left ( \sin \left ( at \right ) \right ) ^{2}{c}^{2} \left ( \sin \left ( b \right ) \right ) ^{2}+6\,x \left ( t \right ) \sin \left ( at \right ) \cos \left ( at \right ) {c}^{2}\cos \left ( b \right ) \sin \left ( b \right ) -3\,x \left ( t \right ) \left ( \cos \left ( at \right ) \right ) ^{2}{c}^{2} \left ( \cos \left ( b \right ) \right ) ^{2}+{c}^{2}x \left ( t \right ) +{\frac {{\rm d}^{2}}{{\rm d}{ t}^{2}}}x \left ( t \right ) }{3\,\cos \left ( atb \right ) {c}^{2}\sin \left ( atb \right ) }} \right \} \right \} \]