just sharing my solution, passed all testcases with 0 or 0.01s on first submisson.

my approach is to construct a linear array (insted of handlling a 2d array) from each layer of the matrix, once you have the linear array, rotating it is easy by just finding the new starting position and fill the original matrix layer again starting from that position.

void dorotate (long r, long **matrix, int m, int n) {
    int k=m;
    long *flatten;

    if (n<m) k=n;
    flatten = malloc ( (m*2+(n-2)*2) * sizeof (long) );


    for (int i=0; i<k; i++) {

        if ((i+1)*2 > k) break;

        int ptr=0;

        //top row, left to right
        for (int j=i; j<n-i; j++) {
            flatten[ptr++] = matrix[i][j];
        }

        // right column, top to bottom
        for (int j=i+1; j<m-i; j++) {
            flatten[ptr++] = matrix[j][n-1-i];
        }

        // bottom row, right to left
        for (int j=n-2-i; j>=i; j--) {
            flatten[ptr++] = matrix[m-1-i][j];
        }

        // left column, bottom to top
        for (int j=m-2-i; j>i; j--) {
            flatten[ptr++] = matrix[j][i];
        }

        //printf ("%d - ", ptr);
        //for (int p=0; p<ptr; p++) {
            //printf ("%ld ", flatten[p]);
        //}
        //printf ("\n\n");

        int new_start_pos = r%ptr;
        //printf ("new_start_pos = %d ", new_start_pos);
        //printf ("\n\n");

        if (new_start_pos>0) {
            //top row, left to right
            for (int j=i; j<n-i; j++) {
                matrix[i][j] = flatten[new_start_pos];
                new_start_pos = (new_start_pos+1) % ptr;
            }

            // right column, top to bottom
            for (int j=i+1; j<m-i; j++) {
                matrix[j][n-1-i] = flatten[new_start_pos];
                new_start_pos = (new_start_pos+1) % ptr;
            }

            // bottom row, right to left
            for (int j=n-2-i; j>=i; j--) {
                matrix[m-1-i][j] = flatten[new_start_pos];
                new_start_pos = (new_start_pos+1) % ptr;
            }

            // left column, bottom to top
            for (int j=m-2-i; j>i; j--) {
                matrix[j][i] = flatten[new_start_pos];
                new_start_pos = (new_start_pos+1) % ptr;
            }
        }

        //display_matrix (matrix, m, n);
        //printf ("\n\n");

    }

    if (!flatten) free(flatten);
}