I want to code in c itself. Any way out ?

Well you could do that, but you don't need to. I submitted a solution using dynamic programming, and one using just regular recursion. The difference in running time is so small that it prints the same running times, even though one runs in linear time and the other one in exponential time. They should increase the test size and limit the amount of running time so that without the use of dynamic programming you can't solve the problem imo.

True, I switched from JavaScript just to get it done.. did find a pitfall though.

third = math.pow(second, 2) + first

will produce weird numbers from the 8th iteration.

instead must use: second*second + first

AFAIK, there's no easy way to do this problem in C++ with the libraries provided. In real life you'd use a bignum library like GNU MP or Boost.Multiprecision, but those libraries aren't available here.

You can certainly roll your own, but that seems a little extreme.

I've done all the other algorithm problems in C++, but for this one I switched to Java so I could use java.math.BigInteger.

Here is how I do it. You can actually make it faster by playing with Base.

#include <cmath>
#include <cstdio>
#include <vector>
#include <iostream>
#include <algorithm>
using namespace std;

namespace
{
    char toChar(char d) {
        return d | 0x30;
    }
}

constexpr unsigned Base = 100;

class BigNumber
{
private:
    string m_num;

public:
    BigNumber(int num = 0);
    BigNumber(const BigNumber & other);
    BigNumber(BigNumber && other);
    BigNumber &operator+=(const BigNumber & num2);
    void swap(BigNumber &num);

    BigNumber &operator=(BigNumber num);

    friend BigNumber operator*(const BigNumber & num1, const BigNumber & num2);
    friend BigNumber operator+(BigNumber num1, const BigNumber & num2);
    friend ostream & operator<<(ostream & outstream, const BigNumber & num);

private:
    void add(const BigNumber &num, int exp);
};

BigNumber::BigNumber(int num)
{
    for (; num; num /= Base) {
        m_num.push_back(num % Base);
    }
    if (m_num.empty()) {
        m_num.push_back(0);
    }
}

BigNumber::BigNumber(const BigNumber & other)
    : m_num(other.m_num)
{ }

BigNumber::BigNumber(BigNumber && other)
    : m_num(std::move(other.m_num))
{ }

BigNumber operator*(const BigNumber & num1, const BigNumber & num2)
{
    BigNumber precomp[Base];
    for (int i = 1; i < Base; ++i) {
        precomp[i] = precomp[i - 1] + num1;
    }

    BigNumber result;
    int exp = 0;
    for_each(num2.m_num.begin(), num2.m_num.end(), [&result, &precomp, &exp](char c) {
        result.add(precomp[c], exp++);
    });
    return result;
}

void BigNumber::add(const BigNumber & num, int exp)
{
    if (exp > m_num.size()) {
        m_num.append(exp - m_num.size(), 0).append(num.m_num.begin(), num.m_num.end());
    } else {
        auto b1 = m_num.begin() + exp, e1 = m_num.end();
        auto b2 = num.m_num.begin(), e2 = num.m_num.end();

        int r = 0;
        for (; b1 != e1 && b2 != e2; ++b1, ++b2, r/= Base) {
            r += *b1 + *b2;
            *b1 = r % Base;
        }

        for (; r && b1 != e1; ++b1, r /= Base) {
            r += *b1;
            *b1 = r % Base;
        }

        for (; r && b2 != e2; ++b2, r /= Base) {
            r += *b2;
            m_num.push_back(r % Base);
        }

        if (b2 != e2) {
            m_num.append(b2, e2);
        }

        if (r) {
            m_num.push_back(r);
        }
    }
}

BigNumber &BigNumber::operator+=(const BigNumber & num2)
{
    add(num2, 0);
    return *this;
}

BigNumber operator+(BigNumber num1, const BigNumber & num2)
{
    num1.add(num2, 0);
    return num1;
}

ostream & operator<<(ostream & outstream, const BigNumber & num)
{
    string x(num.m_num.size() * 2, 0);
    for (int i = 0; i < num.m_num.size(); ++i) {
        auto d = num.m_num[i];
        x[2 * i] = toChar(d % 10);
        x[2 * i + 1] = toChar(d / 10);
    }
    if (x.back() == '0') {
        x.erase(x.size() - 1);
    }
    reverse(x.begin(), x.end());
    return outstream << x;
}

BigNumber & BigNumber::operator=(BigNumber num) {
    m_num = std::move(num.m_num);
    return *this;
}

void BigNumber::swap(BigNumber & num)
{
    m_num.swap(num.m_num);
}

int main() {
    int _a, _b, n;
    cin >> _a >> _b >> n;

    BigNumber a(_a), b(_b);
    while (n-- > 2) {
        a += b * b;
        a.swap(b);
    }

    cout << b;
    return 0;
}

I passed all tests by using the Karatsuba multiplication algorithm with my own Big Integer implementation. Very dumb by the way, but fast enough to find the 24 modified Fibonacci number in a half of second (0 1 24). Nothing special work was done about working with number representation – store it as is in a base 10 in a vector. If it is still relevant for you, email me (taenaru@gmail.com) and I'll gladly explain you how it might work.

There is not! And this is the challenge. You will need to cut the number in pieces to work with it.