DSolve[{Derivative[2][x][t] == c^2*(-1 + 3*Cos[b + a*t]^2)*x[t] + (3*c^2*Sin[2*a*b*t]*y[t])/2, Derivative[2][y][t] == (3*c^2*Sin[2*a*b*t]*x[t])/2 + c^2*(-1 + 3*Sin[b + a*t]^2)*y[t]},{x[t], y[t]},t]