It feels like the key leaks a lot into the encoded sequence. Just try encoding a sequence of 0x00 0x00 0x00 0x00...

        #include <stdio.h>

        int main() {
                char key[] = {0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08};
                int key_size = sizeof(key) / sizeof(key[0]);

                int pos_a = key[0];
                int pos_b = key[1];
                int pos_c = key[2];

                char encrypted[100], decrypted[100];

                for(int i = 0; i< 100; i++) encrypted[i] = 0;

                for(int i = 0; i < 100; i++)
                {
                        int old_a = pos_a;
                        pos_a = key[pos_b] % key_size;
                        pos_b = (pos_a + 1 + key[pos_c] % (key_size - 1)) % key_size;
                        pos_c = (pos_a + 1 + key[old_a] % (key_size - 1)) % key_size;
                        decrypted[i] = encrypted[i] ^ key[pos_a] ^ key[pos_b];
                        key[pos_c] = (key[pos_c] << 31) | (key[pos_c] >> 1);
                        key[pos_a] ^= key[pos_c] ^ i ^ decrypted[i];
                }

                for(int i = 0; i < 100; i++) printf("%02x ", decrypted[i]);

        }